diff --git a/clang-tools-extra/clang-tidy/ExpandModularHeadersPPCallbacks.h b/clang-tools-extra/clang-tidy/ExpandModularHeadersPPCallbacks.h
index 95216368492ca..d72d021f44838 100644
--- a/clang-tools-extra/clang-tidy/ExpandModularHeadersPPCallbacks.h
+++ b/clang-tools-extra/clang-tidy/ExpandModularHeadersPPCallbacks.h
@@ -137,7 +137,7 @@ class ExpandModularHeadersPPCallbacks : public PPCallbacks {
   std::unique_ptr<Preprocessor> PP;
   bool EnteredMainFile = false;
   bool StartedLexing = false;
-  Token CurrentToken;
+  Token CurrentToken = Token();
 };
 
 } // namespace tooling
diff --git a/clang-tools-extra/clang-tidy/bugprone/FloatLoopCounterCheck.cpp b/clang-tools-extra/clang-tidy/bugprone/FloatLoopCounterCheck.cpp
index adf2d2b4bcc07..38a0234337756 100644
--- a/clang-tools-extra/clang-tidy/bugprone/FloatLoopCounterCheck.cpp
+++ b/clang-tools-extra/clang-tidy/bugprone/FloatLoopCounterCheck.cpp
@@ -31,6 +31,7 @@ void FloatLoopCounterCheck::registerMatchers(MatchFinder *Finder) {
 
 void FloatLoopCounterCheck::check(const MatchFinder::MatchResult &Result) {
   const auto *FS = Result.Nodes.getNodeAs<ForStmt>("for");
+  assert(FS && "FS should not be null");
 
   diag(FS->getInc()->getBeginLoc(), "loop induction expression should not have "
                                     "floating-point type")
diff --git a/clang-tools-extra/clang-tidy/cppcoreguidelines/ProBoundsAvoidUncheckedContainerAccessCheck.cpp b/clang-tools-extra/clang-tidy/cppcoreguidelines/ProBoundsAvoidUncheckedContainerAccessCheck.cpp
index 54c4692923949..cf4b445a554e8 100644
--- a/clang-tools-extra/clang-tidy/cppcoreguidelines/ProBoundsAvoidUncheckedContainerAccessCheck.cpp
+++ b/clang-tools-extra/clang-tidy/cppcoreguidelines/ProBoundsAvoidUncheckedContainerAccessCheck.cpp
@@ -176,7 +176,7 @@ void ProBoundsAvoidUncheckedContainerAccessCheck::check(
     }
   } else if (const auto *MCE = dyn_cast<CXXMemberCallExpr>(MatchedExpr)) {
     // Case: a.operator[](i) or a->operator[](i)
-    const auto *Callee = dyn_cast<MemberExpr>(MCE->getCallee());
+    const auto *Callee = cast<MemberExpr>(MCE->getCallee());
 
     if (FixMode == At) {
       // Cases: a.operator[](i) => a.at(i) and a->operator[](i) => a->at(i)
diff --git a/clang/docs/LanguageExtensions.rst b/clang/docs/LanguageExtensions.rst
index 4d9dc817c4b52..d1303b1eb3b1b 100644
--- a/clang/docs/LanguageExtensions.rst
+++ b/clang/docs/LanguageExtensions.rst
@@ -1827,23 +1827,6 @@ Builtin type aliases
 
 Clang provides a few builtin aliases to improve the throughput of certain metaprogramming facilities.
 
-__builtin_common_reference
---------------------------
-
-.. code-block:: c++
-
-  template <template <class, class, template <class> class, template <class> class> class BasicCommonReferenceT,
-            template <class... Args> CommonTypeT,
-            template <class> HasTypeMember,
-            class HasNoTypeMember,
-            class... Ts>
-  using __builtin_common_reference = ...;
-
-This alias is used for implementing ``std::common_reference``. If ``std::common_reference`` should contain a ``type``
-member, it is an alias to ``HasTypeMember<TheCommonReference>``. Otherwse it is an alias to ``HasNoTypeMember``. The
-``CommonTypeT`` is usually ``std::common_type_t``. ``BasicCommonReferenceT`` is usually an alias template to
-``basic_common_reference<T, U, TX, UX>::type``.
-
 __builtin_common_type
 ---------------------
 
diff --git a/clang/include/clang/AST/ASTConsumer.h b/clang/include/clang/AST/ASTConsumer.h
index 447f2592d2359..a1ef187ee2069 100644
--- a/clang/include/clang/AST/ASTConsumer.h
+++ b/clang/include/clang/AST/ASTConsumer.h
@@ -27,6 +27,7 @@ namespace clang {
   class VarDecl;
   class FunctionDecl;
   class ImportDecl;
+  class OpenACCRoutineDecl;
 
 /// ASTConsumer - This is an abstract interface that should be implemented by
 /// clients that read ASTs.  This abstraction layer allows the client to be
@@ -116,6 +117,11 @@ class ASTConsumer {
   // variable has been instantiated.
   virtual void HandleCXXStaticMemberVarInstantiation(VarDecl *D) {}
 
+  /// Callback to handle the end-of-translation unit attachment of OpenACC
+  /// routine declaration information.
+  virtual void HandleOpenACCRoutineReference(const FunctionDecl *FD,
+                                             const OpenACCRoutineDecl *RD) {}
+
   /// Callback involved at the end of a translation unit to
   /// notify the consumer that a vtable for the given C++ class is
   /// required.
diff --git a/clang/include/clang/AST/OperationKinds.def b/clang/include/clang/AST/OperationKinds.def
index c2dca895e8411..8a13ad988403b 100644
--- a/clang/include/clang/AST/OperationKinds.def
+++ b/clang/include/clang/AST/OperationKinds.def
@@ -364,6 +364,9 @@ CAST_OPERATION(IntToOCLSampler)
 // Truncate a vector type by dropping elements from the end (HLSL only).
 CAST_OPERATION(HLSLVectorTruncation)
 
+// Truncate a matrix type by dropping elements from the end (HLSL only).
+CAST_OPERATION(HLSLMatrixTruncation)
+
 // Non-decaying array RValue cast (HLSL only).
 CAST_OPERATION(HLSLArrayRValue)
 
diff --git a/clang/include/clang/Basic/BuiltinTemplates.td b/clang/include/clang/Basic/BuiltinTemplates.td
index 804871de2bdab..504405acbdc78 100644
--- a/clang/include/clang/Basic/BuiltinTemplates.td
+++ b/clang/include/clang/Basic/BuiltinTemplates.td
@@ -10,11 +10,11 @@ class TemplateArg<string name> {
   string Name = name;
 }
 
-class Template<list<TemplateArg> args, string name = ""> : TemplateArg<name> {
+class Template<list<TemplateArg> args, string name> : TemplateArg<name> {
   list<TemplateArg> Args = args;
 }
 
-class Class<string name = "", bit is_variadic = 0> : TemplateArg<name> {
+class Class<string name, bit is_variadic = 0> : TemplateArg<name> {
   bit IsVariadic = is_variadic;
 }
 
@@ -56,32 +56,6 @@ def __builtin_common_type : CPlusPlusBuiltinTemplate<
    Class<"HasNoTypeMember">,
    Class<"Ts", /*is_variadic=*/1>]>;
 
-// template <template <class,"
-//                     class,"
-//                     template <class> class,"
-//                     template <class> class> class BasicCommonReferenceT,"
-//           template <class... Args> class CommonTypeT,"
-//           template <class> class HasTypeMember,"
-//           class HasNoTypeMember,"
-//           class... Ts>"
-def __builtin_common_reference : CPlusPlusBuiltinTemplate<
-            [Template<[Class<>,
-                       Class<>,
-                       Template<[Class<>]>,
-                       Template<[Class<>]>], "BasicCommonReferenceT">,
-             Template<[Class<"Args", /*is_variadic=*/1>], "CommonTypeT">,
-             Template<[Class<>], "HasTypeMember">,
-             Class<"HasNoTypeMember">,
-             Class<"Ts", /*is_variadic=*/1>]>;
-
-foreach Ref = ["", "lvalue", "rvalue"] in {
-  foreach Const = ["", "const"] in {
-    foreach Volatile = ["", "volatile"] in {
-      def __clang_internal_xref_#Ref#Const#Volatile : CPlusPlusBuiltinTemplate<[Class<>]>;
-    }
-  }
-}
-
 // template <uint32_t Opcode,
 //           uint32_t Size,
 //           uint32_t Alignment,
diff --git a/clang/include/clang/Basic/DiagnosticGroups.td b/clang/include/clang/Basic/DiagnosticGroups.td
index 71f35b597d965..eadd724950a2c 100644
--- a/clang/include/clang/Basic/DiagnosticGroups.td
+++ b/clang/include/clang/Basic/DiagnosticGroups.td
@@ -1059,6 +1059,7 @@ def SuperSubClassMismatch : DiagGroup<"super-class-method-mismatch">;
 def OverridingMethodMismatch : DiagGroup<"overriding-method-mismatch">;
 def VariadicMacros : DiagGroup<"variadic-macros">;
 def VectorConversion : DiagGroup<"vector-conversion">;      // clang specific
+def MatrixConversion : DiagGroup<"matrix-conversion">;      // clang specific
 def VexingParse : DiagGroup<"vexing-parse">;
 def VLAUseStaticAssert : DiagGroup<"vla-extension-static-assert">;
 def VLACxxExtension : DiagGroup<"vla-cxx-extension", [VLAUseStaticAssert]>;
@@ -1333,6 +1334,8 @@ def : DiagGroup<"int-conversions",
                 [IntConversion]>; // -Wint-conversions = -Wint-conversion
 def : DiagGroup<"vector-conversions",
                 [VectorConversion]>; // -Wvector-conversions = -Wvector-conversion
+def : DiagGroup<"matrix-conversions",
+                [MatrixConversion]>; // -Wmatrix-conversions = -Wmatrix-conversion
 def : DiagGroup<"unused-local-typedefs", [UnusedLocalTypedef]>;
                 // -Wunused-local-typedefs = -Wunused-local-typedef
 
diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td
index 91eb021e21ca1..0cb02f6598952 100644
--- a/clang/include/clang/Basic/DiagnosticSemaKinds.td
+++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td
@@ -4352,6 +4352,9 @@ def warn_param_typestate_mismatch : Warning<
 def warn_unknown_sanitizer_ignored : Warning<
   "unknown sanitizer '%0' ignored">, InGroup<UnknownSanitizers>;
 
+def warn_impcast_matrix_scalar : Warning<
+  "implicit conversion turns matrix to scalar: %0 to %1">,
+  InGroup<MatrixConversion>;
 def warn_impcast_vector_scalar : Warning<
   "implicit conversion turns vector to scalar: %0 to %1">,
   InGroup<Conversion>, DefaultIgnore;
@@ -13282,7 +13285,10 @@ def err_hlsl_builtin_scalar_vector_mismatch
           "vector type with matching scalar element type%diff{: $ vs $|}2,3">;
 
 def warn_hlsl_impcast_vector_truncation : Warning<
-  "implicit conversion truncates vector: %0 to %1">, InGroup<Conversion>;
+  "implicit conversion truncates vector: %0 to %1">, InGroup<VectorConversion>;
+
+def warn_hlsl_impcast_matrix_truncation : Warning<
+  "implicit conversion truncates matrix: %0 to %1">, InGroup<MatrixConversion>;
 
 def warn_hlsl_availability : Warning<
   "%0 is only available %select{|in %4 environment }3on %1 %2 or newer">,
diff --git a/clang/include/clang/Basic/arm_mve.td b/clang/include/clang/Basic/arm_mve.td
index 51b7dd16e5195..77531c31538c1 100644
--- a/clang/include/clang/Basic/arm_mve.td
+++ b/clang/include/clang/Basic/arm_mve.td
@@ -783,15 +783,15 @@ let params = T.Unsigned in {
 }
 let params = T.Float in {
   def vminnmq: Intrinsic<Vector, (args Vector:$a, Vector:$b),
-                                 (IRIntBase<"minnum", [Vector]> $a, $b)>;
+                                 (fminnm $a, $b)>;
   def vmaxnmq: Intrinsic<Vector, (args Vector:$a, Vector:$b),
-                                 (IRIntBase<"maxnum", [Vector]> $a, $b)>;
+                                 (fmaxnm $a, $b)>;
   def vminnmaq: Intrinsic<Vector, (args Vector:$a, Vector:$b),
-                                  (IRIntBase<"minnum", [Vector]>
+                                  (fminnm
                                    (IRIntBase<"fabs", [Vector]> $a),
                                    (IRIntBase<"fabs", [Vector]> $b))>;
   def vmaxnmaq: Intrinsic<Vector, (args Vector:$a, Vector:$b),
-                                  (IRIntBase<"maxnum", [Vector]>
+                                  (fmaxnm
                                    (IRIntBase<"fabs", [Vector]> $a),
                                    (IRIntBase<"fabs", [Vector]> $b))>;
 }
diff --git a/clang/include/clang/Basic/arm_mve_defs.td b/clang/include/clang/Basic/arm_mve_defs.td
index 3714262898476..3210549d0cb56 100644
--- a/clang/include/clang/Basic/arm_mve_defs.td
+++ b/clang/include/clang/Basic/arm_mve_defs.td
@@ -589,6 +589,10 @@ def fsub: strictFPAlt<fsub_node,
                       IRInt<"vsub", [Vector]>>;
 def fmul: strictFPAlt<fmul_node,
                       IRInt<"vmul", [Vector]>>;
+def fminnm : strictFPAlt<IRIntBase<"minnum", [Vector]>,
+                         IRInt<"vminnm", [Vector]>>;
+def fmaxnm : strictFPAlt<IRIntBase<"maxnum", [Vector]>,
+                         IRInt<"vmaxnm", [Vector]>>;
 
 // -----------------------------------------------------------------------------
 // Convenience lists of parameter types. 'T' is just a container record, so you
diff --git a/clang/include/clang/CIR/CIRGenerator.h b/clang/include/clang/CIR/CIRGenerator.h
index 5ea11463ffa9f..31dead2d7b585 100644
--- a/clang/include/clang/CIR/CIRGenerator.h
+++ b/clang/include/clang/CIR/CIRGenerator.h
@@ -81,6 +81,9 @@ class CIRGenerator : public clang::ASTConsumer {
   void HandleTagDeclDefinition(clang::TagDecl *d) override;
   void HandleTagDeclRequiredDefinition(const clang::TagDecl *D) override;
   void HandleCXXStaticMemberVarInstantiation(clang::VarDecl *D) override;
+  void
+  HandleOpenACCRoutineReference(const clang::FunctionDecl *FD,
+                                const clang::OpenACCRoutineDecl *RD) override;
   void CompleteTentativeDefinition(clang::VarDecl *d) override;
   void HandleVTable(clang::CXXRecordDecl *rd) override;
 
diff --git a/clang/include/clang/CIR/Dialect/IR/CIROps.td b/clang/include/clang/CIR/Dialect/IR/CIROps.td
index 4b64fc56c57ad..ae199f35cb10e 100644
--- a/clang/include/clang/CIR/Dialect/IR/CIROps.td
+++ b/clang/include/clang/CIR/Dialect/IR/CIROps.td
@@ -1173,6 +1173,35 @@ def CIR_SwitchOp : CIR_Op<"switch", [
   let hasLLVMLowering = false;
 }
 
+//===----------------------------------------------------------------------===//
+// IsConstantOp
+//===----------------------------------------------------------------------===//
+
+def CIR_IsConstantOp : CIR_Op<"is_constant", [Pure]> {
+  let summary = "Test for manifest compile-time constant";
+  let description = [{
+    Returns `true` if the argument is known to be a manifest compile-time
+    constant otherwise returns `false`. If the argument is a constant expression
+    which refers to a global (the address of which _is_ a constant, but not
+    manifest during the compile), then the intrinsic evaluates to `false`.
+
+    This is used to represent `__builtin_constant_p` in cases where the argument
+    isn't known to be constant during initial translation of the source code but
+    might be proven to be constant after later optimizations.
+
+    Example:
+    ```
+    %1 = cir.is_constant %2 : !s32i -> !cir.bool
+    ```
+  }];
+  let arguments = (ins CIR_AnyType:$val);
+  let results = (outs CIR_BoolType:$result);
+
+  let assemblyFormat = [{
+    $val `:` qualified(type($val)) `->` qualified(type($result)) attr-dict
+  }];
+}
+
 //===----------------------------------------------------------------------===//
 // SwitchFlatOp
 //===----------------------------------------------------------------------===//
diff --git a/clang/include/clang/Options/Options.td b/clang/include/clang/Options/Options.td
index 28c609bb8524d..84e3b4d4e59e2 100644
--- a/clang/include/clang/Options/Options.td
+++ b/clang/include/clang/Options/Options.td
@@ -4870,25 +4870,25 @@ def ggdb3 : Flag<["-"], "ggdb3">, Group<ggdbN_Group>;
 def glldb : Flag<["-"], "glldb">, Group<gTune_Group>;
 def gsce : Flag<["-"], "gsce">, Group<gTune_Group>;
 def gdbx : Flag<["-"], "gdbx">, Group<gTune_Group>;
-// Equivalent to our default dwarf version. Forces usual dwarf emission when
+// Equivalent to our default DWARF version. Forces usual DWARF emission when
 // CodeView is enabled.
 def gdwarf : Flag<["-"], "gdwarf">, Group<g_Group>,
   Visibility<[ClangOption, CLOption, DXCOption, FlangOption]>,
-  HelpText<"Generate source-level debug information with the default dwarf version">;
+  HelpText<"Generate source-level debug information with the default DWARF version">;
 
 let Visibility = [ClangOption, FlangOption] in {
 def gdwarf_2 : Flag<["-"], "gdwarf-2">, Group<g_Group>,
-  HelpText<"Generate source-level debug information with dwarf version 2">;
+  HelpText<"Generate source-level debug information with DWARF version 2">;
 def gdwarf_3 : Flag<["-"], "gdwarf-3">, Group<g_Group>,
-  HelpText<"Generate source-level debug information with dwarf version 3">;
+  HelpText<"Generate source-level debug information with DWARF version 3">;
 def gdwarf_4 : Flag<["-"], "gdwarf-4">, Group<g_Group>,
-  HelpText<"Generate source-level debug information with dwarf version 4">;
+  HelpText<"Generate source-level debug information with DWARF version 4">;
 def gdwarf_5 : Flag<["-"], "gdwarf-5">, Group<g_Group>,
-  HelpText<"Generate source-level debug information with dwarf version 5">;
+  HelpText<"Generate source-level debug information with DWARF version 5">;
 def gdwarf_6
     : Flag<["-"], "gdwarf-6">,
       Group<g_Group>,
-      HelpText<"Generate source-level debug information with dwarf version 6">;
+      HelpText<"Generate source-level debug information with DWARF version 6">;
 }
 def gdwarf64 : Flag<["-"], "gdwarf64">, Group<g_Group>,
   Visibility<[ClangOption, CC1Option, CC1AsOption]>,
@@ -4915,7 +4915,7 @@ def gno_heterogeneous_dwarf : Flag<["-"], "gno-heterogeneous-dwarf">,
   HelpText<"Disable DWARF extensions for heterogeneous debugging">,
   Alias<gheterogeneous_dwarf_EQ>, AliasArgs<["disabled"]>;
 
-def gcodeview : Flag<["-"], "gcodeview">,
+def gcodeview : Flag<["-"], "gcodeview">, Group<g_Group>,
   HelpText<"Generate CodeView debug information">,
   Visibility<[ClangOption, CC1Option, CC1AsOption, CLOption, DXCOption]>,
   MarshallingInfoFlag<CodeGenOpts<"EmitCodeView">>;
@@ -4923,17 +4923,20 @@ defm codeview_ghash : BoolOption<"g", "codeview-ghash",
   CodeGenOpts<"CodeViewGHash">, DefaultFalse,
   PosFlag<SetTrue, [], [ClangOption, CC1Option],
           "Emit type record hashes in a .debug$H section">,
-  NegFlag<SetFalse>, BothFlags<[], [ClangOption, CLOption, DXCOption]>>;
+  NegFlag<SetFalse>, BothFlags<[], [ClangOption, CLOption, DXCOption]>>,
+  Group<g_flags_Group>;
 defm codeview_command_line : BoolOption<"g", "codeview-command-line",
   CodeGenOpts<"CodeViewCommandLine">, DefaultTrue,
   PosFlag<SetTrue, [], [ClangOption], "Emit compiler path and command line into CodeView debug information">,
   NegFlag<SetFalse, [], [ClangOption], "Don't emit compiler path and command line into CodeView debug information">,
-  BothFlags<[], [ClangOption, CLOption, DXCOption, CC1Option]>>;
+  BothFlags<[], [ClangOption, CLOption, DXCOption, CC1Option]>>,
+  Group<g_flags_Group>;
 defm inline_line_tables : BoolGOption<"inline-line-tables",
   CodeGenOpts<"NoInlineLineTables">, DefaultFalse,
   NegFlag<SetTrue, [], [ClangOption, CC1Option],
           "Don't emit inline line tables.">,
-  PosFlag<SetFalse>, BothFlags<[], [ClangOption, CLOption, DXCOption]>>;
+  PosFlag<SetFalse>, BothFlags<[], [ClangOption, CLOption, DXCOption]>>,
+  Group<g_flags_Group>;
 
 def gfull : Flag<["-"], "gfull">, Group<g_Group>;
 def gused : Flag<["-"], "gused">, Group<g_Group>;
@@ -4958,7 +4961,8 @@ defm strict_dwarf : BoolOption<"g", "strict-dwarf",
 defm omit_unreferenced_methods : BoolGOption<"omit-unreferenced-methods",
   CodeGenOpts<"DebugOmitUnreferencedMethods">, DefaultFalse,
   NegFlag<SetFalse>,
-  PosFlag<SetTrue, [], [CC1Option]>, BothFlags<[], [ClangOption, CLOption, DXCOption]>>;
+  PosFlag<SetTrue, [], [CC1Option]>, BothFlags<[], [ClangOption, CLOption, DXCOption]>>,
+  Group<g_flags_Group>;
 defm column_info : BoolOption<"g", "column-info",
   CodeGenOpts<"DebugColumnInfo">, DefaultTrue,
   NegFlag<SetFalse, [], [ClangOption, CC1Option]>,
@@ -5027,6 +5031,7 @@ defm structor_decl_linkage_names
                           "Attach linkage names to C++ constructor/destructor "
                           "declarations in DWARF.">,
                   BothFlags<[], [ClangOption, CLOption, CC1Option]>>,
+                  Group<g_flags_Group>,
                   DocBrief<[{On some ABIs (e.g., Itanium), constructors and destructors may have multiple variants. Historically, when generating DWARF, Clang did not attach ``DW_AT_linkage_name`` to structor DIEs because there were multiple possible manglings (depending on the structor variant) that could be used. With ``-gstructor-decl-linkage-names``, for ABIs with structor variants, we attach a "unified" mangled name to structor declarations DIEs which debuggers can use to look up all the definitions for a structor declaration. E.g., a "unified" mangled name ``_ZN3FooC4Ev`` may have multiple definitions associated with it such as ``_ZN3FooC1Ev`` and ``_ZN3FooC2Ev``.
 
 Enabling this flag results in a better interactive debugging experience (both GDB and LLDB have support for understanding these "unified" linkage names). However, it comes with a significant increase in debug-info size (particularly the `.debug_str` section). As an escape hatch, users can disable this feature using ``-gno-structor-decl-linkage-names``.}]>;
@@ -5035,7 +5040,8 @@ defm key_instructions : BoolGOption<"key-instructions",
     NegFlag<SetFalse>, PosFlag<SetTrue, [], [],
         "Enable Key Instructions, which reduces the jumpiness of debug stepping in optimized C/C++ code"
         " in some debuggers. DWARF only.">,
-    BothFlags<[], [ClangOption, CLOption, CC1Option]>>;
+    BothFlags<[], [ClangOption, CLOption, CC1Option]>>,
+  Group<g_flags_Group>;
 def headerpad__max__install__names : Joined<["-"], "headerpad_max_install_names">;
 def help : Flag<["-", "--"], "help">,
     Visibility<[ClangOption, CC1Option, CC1AsOption,
@@ -8690,7 +8696,7 @@ def main_file_name : Separate<["-"], "main-file-name">,
   Visibility<[CC1Option, CC1AsOption]>,
   MarshallingInfoString<CodeGenOpts<"MainFileName">>;
 def split_dwarf_output : Separate<["-"], "split-dwarf-output">,
-  HelpText<"File name to use for split dwarf debug info output">,
+  HelpText<"File name to use for split DWARF debug info output">,
   Visibility<[CC1Option, CC1AsOption, FC1Option]>,
   MarshallingInfoString<CodeGenOpts<"SplitDwarfOutput">>;
 
@@ -8724,7 +8730,7 @@ def dependent_lib : Joined<["--"], "dependent-lib=">,
   MarshallingInfoStringVector<CodeGenOpts<"DependentLibraries">>;
 
 def split_dwarf_file : Separate<["-"], "split-dwarf-file">,
-  HelpText<"Name of the split dwarf debug info file to encode in the object file">,
+  HelpText<"Name of the split DWARF debug info file to encode in the object file">,
   MarshallingInfoString<CodeGenOpts<"SplitDwarfFile">>;
 
 } // let Visibility = [CC1Option, FC1Option]
diff --git a/clang/include/clang/Sema/Overload.h b/clang/include/clang/Sema/Overload.h
index 59bbd0fbd9e95..ab45328ee8ab7 100644
--- a/clang/include/clang/Sema/Overload.h
+++ b/clang/include/clang/Sema/Overload.h
@@ -198,6 +198,9 @@ class Sema;
     /// HLSL vector truncation.
     ICK_HLSL_Vector_Truncation,
 
+    /// HLSL Matrix truncation.
+    ICK_HLSL_Matrix_Truncation,
+
     /// HLSL non-decaying array rvalue cast.
     ICK_HLSL_Array_RValue,
 
diff --git a/clang/include/clang/Sema/Sema.h b/clang/include/clang/Sema/Sema.h
index 19cca691e9859..78ecbccbe4efc 100644
--- a/clang/include/clang/Sema/Sema.h
+++ b/clang/include/clang/Sema/Sema.h
@@ -15322,17 +15322,6 @@ class Sema final : public SemaBase {
   QualType BuiltinDecay(QualType BaseType, SourceLocation Loc);
   QualType BuiltinAddReference(QualType BaseType, UTTKind UKind,
                                SourceLocation Loc);
-
-  QualType BuiltinAddRValueReference(QualType BaseType, SourceLocation Loc) {
-    return BuiltinAddReference(BaseType, UnaryTransformType::AddRvalueReference,
-                               Loc);
-  }
-
-  QualType BuiltinAddLValueReference(QualType BaseType, SourceLocation Loc) {
-    return BuiltinAddReference(BaseType, UnaryTransformType::AddLvalueReference,
-                               Loc);
-  }
-
   QualType BuiltinRemoveExtent(QualType BaseType, UTTKind UKind,
                                SourceLocation Loc);
   QualType BuiltinRemoveReference(QualType BaseType, UTTKind UKind,
@@ -15347,9 +15336,6 @@ class Sema final : public SemaBase {
   QualType BuiltinChangeSignedness(QualType BaseType, UTTKind UKind,
                                    SourceLocation Loc);
 
-  bool BuiltinIsConvertible(QualType From, QualType To, SourceLocation Loc,
-                            bool CheckNothrow = false);
-
   bool BuiltinIsBaseOf(SourceLocation RhsTLoc, QualType LhsT, QualType RhsT);
 
   /// Ensure that the type T is a literal type.
diff --git a/clang/include/clang/Sema/SemaOpenACC.h b/clang/include/clang/Sema/SemaOpenACC.h
index f751e985ae0ff..b5e3ecab36d22 100644
--- a/clang/include/clang/Sema/SemaOpenACC.h
+++ b/clang/include/clang/Sema/SemaOpenACC.h
@@ -37,8 +37,16 @@ class Scope;
 class SemaOpenACC : public SemaBase {
 public:
   using DeclGroupPtrTy = OpaquePtr<DeclGroupRef>;
+  using RoutineRefListTy = std::pair<FunctionDecl *, OpenACCRoutineDecl *>;
 
 private:
+  // We save a list of routine clauses that refer to a different function(that
+  // is, routine-with-a-name) so that we can do the emission at the 'end'.  We
+  // have to do this, since functions can be emitted before they are referenced,
+  // and the OpenACCRoutineDecl isn't necessarily emitted, as it might be in a
+  // function/etc. So we do these emits at the end of the TU.
+  llvm::SmallVector<RoutineRefListTy> RoutineRefList;
+
   struct ComputeConstructInfo {
     /// Which type of compute construct we are inside of, which we can use to
     /// determine whether we should add loops to the above collection.  We can
@@ -752,6 +760,7 @@ class SemaOpenACC : public SemaBase {
   };
 
   SemaOpenACC(Sema &S);
+  void ActOnEndOfTranslationUnit(TranslationUnitDecl *TU);
 
   // Called when we encounter a 'while' statement, before looking at its 'body'.
   void ActOnWhileStmt(SourceLocation WhileLoc);
diff --git a/clang/lib/AST/Expr.cpp b/clang/lib/AST/Expr.cpp
index 1f405920ce6b5..ca7f3e16a9276 100644
--- a/clang/lib/AST/Expr.cpp
+++ b/clang/lib/AST/Expr.cpp
@@ -1934,6 +1934,7 @@ bool CastExpr::CastConsistency() const {
   case CK_FixedPointToBoolean:
   case CK_HLSLArrayRValue:
   case CK_HLSLVectorTruncation:
+  case CK_HLSLMatrixTruncation:
   case CK_HLSLElementwiseCast:
   case CK_HLSLAggregateSplatCast:
   CheckNoBasePath:
diff --git a/clang/lib/AST/ExprConstant.cpp b/clang/lib/AST/ExprConstant.cpp
index 3eedf0d6795d8..2966066c8b6e5 100644
--- a/clang/lib/AST/ExprConstant.cpp
+++ b/clang/lib/AST/ExprConstant.cpp
@@ -11770,6 +11770,10 @@ bool VectorExprEvaluator::VisitCastExpr(const CastExpr *E) {
       Elements.push_back(Val.getVectorElt(I));
     return Success(Elements, E);
   }
+  case CK_HLSLMatrixTruncation: {
+    // TODO: See #168935. Add matrix truncation support to expr constant.
+    return Error(E);
+  }
   case CK_HLSLAggregateSplatCast: {
     APValue Val;
     QualType ValTy;
@@ -18440,6 +18444,10 @@ bool IntExprEvaluator::VisitCastExpr(const CastExpr *E) {
       return Error(E);
     return Success(Val.getVectorElt(0), E);
   }
+  case CK_HLSLMatrixTruncation: {
+    // TODO: See #168935. Add matrix truncation support to expr constant.
+    return Error(E);
+  }
   case CK_HLSLElementwiseCast: {
     SmallVector<APValue> SrcVals;
     SmallVector<QualType> SrcTypes;
@@ -19033,6 +19041,10 @@ bool FloatExprEvaluator::VisitCastExpr(const CastExpr *E) {
       return Error(E);
     return Success(Val.getVectorElt(0), E);
   }
+  case CK_HLSLMatrixTruncation: {
+    // TODO: See #168935. Add matrix truncation support to expr constant.
+    return Error(E);
+  }
   case CK_HLSLElementwiseCast: {
     SmallVector<APValue> SrcVals;
     SmallVector<QualType> SrcTypes;
@@ -19190,6 +19202,7 @@ bool ComplexExprEvaluator::VisitCastExpr(const CastExpr *E) {
   case CK_IntegralToFixedPoint:
   case CK_MatrixCast:
   case CK_HLSLVectorTruncation:
+  case CK_HLSLMatrixTruncation:
   case CK_HLSLElementwiseCast:
   case CK_HLSLAggregateSplatCast:
     llvm_unreachable("invalid cast kind for complex value");
diff --git a/clang/lib/Basic/Targets/AMDGPU.h b/clang/lib/Basic/Targets/AMDGPU.h
index a51d8d2375cfe..8dcf1d1c9561a 100644
--- a/clang/lib/Basic/Targets/AMDGPU.h
+++ b/clang/lib/Basic/Targets/AMDGPU.h
@@ -84,6 +84,18 @@ class LLVM_LIBRARY_VISIBILITY AMDGPUTargetInfo final : public TargetInfo {
     return TT.getArch() == llvm::Triple::r600;
   }
 
+  bool hasFlatSupport() const {
+    if (GPUKind >= llvm::AMDGPU::GK_GFX700)
+      return true;
+
+    // Dummy target is assumed to be gfx700+ for amdhsa.
+    if (GPUKind == llvm::AMDGPU::GK_NONE &&
+        getTriple().getOS() == llvm::Triple::AMDHSA)
+      return true;
+
+    return false;
+  }
+
 public:
   AMDGPUTargetInfo(const llvm::Triple &Triple, const TargetOptions &Opts);
 
@@ -316,14 +328,16 @@ class LLVM_LIBRARY_VISIBILITY AMDGPUTargetInfo final : public TargetInfo {
       Opts["cl_amd_media_ops"] = true;
       Opts["cl_amd_media_ops2"] = true;
 
+      // FIXME: Check subtarget for image support.
       Opts["__opencl_c_images"] = true;
       Opts["__opencl_c_3d_image_writes"] = true;
+      Opts["__opencl_c_read_write_images"] = true;
       Opts["cl_khr_3d_image_writes"] = true;
       Opts["__opencl_c_program_scope_global_variables"] = true;
       Opts["__opencl_c_atomic_order_seq_cst"] = true;
       Opts["__opencl_c_atomic_scope_all_devices"] = true;
 
-      if (GPUKind >= llvm::AMDGPU::GK_GFX700) {
+      if (hasFlatSupport()) {
         Opts["__opencl_c_generic_address_space"] = true;
         Opts["__opencl_c_device_enqueue"] = true;
       }
diff --git a/clang/lib/CIR/CodeGen/CIRGenBuiltin.cpp b/clang/lib/CIR/CodeGen/CIRGenBuiltin.cpp
index e14b5f8aac337..12d93cf4c73c6 100644
--- a/clang/lib/CIR/CodeGen/CIRGenBuiltin.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenBuiltin.cpp
@@ -542,6 +542,45 @@ RValue CIRGenFunction::emitBuiltinExpr(const GlobalDecl &gd, unsigned builtinID,
     return emitCall(e->getCallee()->getType(), CIRGenCallee::forDirect(fnOp), e,
                     returnValue);
   }
+
+  case Builtin::BI__builtin_constant_p: {
+    mlir::Type resultType = convertType(e->getType());
+
+    const Expr *arg = e->getArg(0);
+    QualType argType = arg->getType();
+    // FIXME: The allowance for Obj-C pointers and block pointers is historical
+    // and likely a mistake.
+    if (!argType->isIntegralOrEnumerationType() && !argType->isFloatingType() &&
+        !argType->isObjCObjectPointerType() && !argType->isBlockPointerType()) {
+      // Per the GCC documentation, only numeric constants are recognized after
+      // inlining.
+      return RValue::get(
+          builder.getConstInt(getLoc(e->getSourceRange()),
+                              mlir::cast<cir::IntType>(resultType), 0));
+    }
+
+    if (arg->HasSideEffects(getContext())) {
+      // The argument is unevaluated, so be conservative if it might have
+      // side-effects.
+      return RValue::get(
+          builder.getConstInt(getLoc(e->getSourceRange()),
+                              mlir::cast<cir::IntType>(resultType), 0));
+    }
+
+    mlir::Value argValue = emitScalarExpr(arg);
+    if (argType->isObjCObjectPointerType()) {
+      cgm.errorNYI(e->getSourceRange(),
+                   "__builtin_constant_p: Obj-C object pointer");
+      return {};
+    }
+    argValue = builder.createBitcast(argValue, convertType(argType));
+
+    mlir::Value result = cir::IsConstantOp::create(
+        builder, getLoc(e->getSourceRange()), argValue);
+    // IsConstantOp returns a bool, but __builtin_constant_p returns an int.
+    result = builder.createBoolToInt(result, resultType);
+    return RValue::get(result);
+  }
   case Builtin::BI__builtin_dynamic_object_size:
   case Builtin::BI__builtin_object_size: {
     unsigned type =
diff --git a/clang/lib/CIR/CodeGen/CIRGenBuiltinX86.cpp b/clang/lib/CIR/CodeGen/CIRGenBuiltinX86.cpp
index 8d01b7dbd15f6..2d503ddb08a7d 100644
--- a/clang/lib/CIR/CodeGen/CIRGenBuiltinX86.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenBuiltinX86.cpp
@@ -11,10 +11,14 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "CIRGenBuilder.h"
 #include "CIRGenFunction.h"
 #include "CIRGenModule.h"
+#include "mlir/IR/Location.h"
+#include "mlir/IR/ValueRange.h"
 #include "clang/Basic/Builtins.h"
 #include "clang/Basic/TargetBuiltins.h"
+#include "clang/CIR/Dialect/IR/CIRTypes.h"
 #include "clang/CIR/MissingFeatures.h"
 
 using namespace clang;
@@ -85,6 +89,69 @@ static mlir::Value getMaskVecValue(CIRGenBuilderTy &builder, mlir::Location loc,
   return maskVec;
 }
 
+// Builds the VecShuffleOp for pshuflw and pshufhw x86 builtins.
+//
+// The vector is split into lanes of 8 word elements (16 bits). The lower or
+// upper half of each lane, controlled by `isLow`, is shuffled in the following
+// way: The immediate is truncated to 8 bits, separated into 4 2-bit fields. The
+// i-th field's value represents the resulting index of the i-th element in the
+// half lane after shuffling. The other half of the lane remains unchanged.
+static cir::VecShuffleOp emitPshufWord(CIRGenBuilderTy &builder,
+                                       const mlir::Value vec,
+                                       const mlir::Value immediate,
+                                       const mlir::Location loc,
+                                       const bool isLow) {
+  uint32_t imm = CIRGenFunction::getZExtIntValueFromConstOp(immediate);
+
+  auto vecTy = cast<cir::VectorType>(vec.getType());
+  unsigned numElts = vecTy.getSize();
+
+  unsigned firstHalfStart = isLow ? 0 : 4;
+  unsigned secondHalfStart = 4 - firstHalfStart;
+
+  // Splat the 8-bits of immediate 4 times to help the loop wrap around.
+  imm = (imm & 0xff) * 0x01010101;
+
+  int64_t indices[32];
+  for (unsigned l = 0; l != numElts; l += 8) {
+    for (unsigned i = firstHalfStart; i != firstHalfStart + 4; ++i) {
+      indices[l + i] = l + (imm & 3) + firstHalfStart;
+      imm >>= 2;
+    }
+    for (unsigned i = secondHalfStart; i != secondHalfStart + 4; ++i)
+      indices[l + i] = l + i;
+  }
+
+  return builder.createVecShuffle(loc, vec, ArrayRef(indices, numElts));
+}
+
+// Builds the shuffle mask for pshufd and shufpd/shufps x86 builtins.
+// The shuffle mask is written to outIndices.
+static void
+computeFullLaneShuffleMask(CIRGenFunction &cgf, const mlir::Value vec,
+                           uint32_t imm, const bool isShufP,
+                           llvm::SmallVectorImpl<int64_t> &outIndices) {
+  auto vecTy = cast<cir::VectorType>(vec.getType());
+  unsigned numElts = vecTy.getSize();
+  unsigned numLanes = cgf.cgm.getDataLayout().getTypeSizeInBits(vecTy) / 128;
+  unsigned numLaneElts = numElts / numLanes;
+
+  // Splat the 8-bits of immediate 4 times to help the loop wrap around.
+  imm = (imm & 0xff) * 0x01010101;
+
+  for (unsigned l = 0; l != numElts; l += numLaneElts) {
+    for (unsigned i = 0; i != numLaneElts; ++i) {
+      uint32_t idx = imm % numLaneElts;
+      imm /= numLaneElts;
+      if (isShufP && i >= (numLaneElts / 2))
+        idx += numElts;
+      outIndices[l + i] = l + idx;
+    }
+  }
+
+  outIndices.resize(numElts);
+}
+
 static mlir::Value emitX86MaskAddLogic(CIRGenBuilderTy &builder,
                                        mlir::Location loc,
                                        const std::string &intrinsicName,
@@ -100,6 +167,44 @@ static mlir::Value emitX86MaskAddLogic(CIRGenBuilderTy &builder,
   return builder.createBitcast(resVec, ops[0].getType());
 }
 
+static mlir::Value emitX86MaskUnpack(CIRGenBuilderTy &builder,
+                                     mlir::Location loc,
+                                     const std::string &intrinsicName,
+                                     SmallVectorImpl<mlir::Value> &ops) {
+  unsigned numElems = cast<cir::IntType>(ops[0].getType()).getWidth();
+
+  // Convert both operands to mask vectors.
+  mlir::Value lhs = getMaskVecValue(builder, loc, ops[0], numElems);
+  mlir::Value rhs = getMaskVecValue(builder, loc, ops[1], numElems);
+
+  mlir::Type i32Ty = builder.getSInt32Ty();
+
+  // Create indices for extracting the first half of each vector.
+  SmallVector<mlir::Attribute, 32> halfIndices;
+  for (auto i : llvm::seq<unsigned>(0, numElems / 2))
+    halfIndices.push_back(cir::IntAttr::get(i32Ty, i));
+
+  // Extract first half of each vector. This gives better codegen than
+  // doing it in a single shuffle.
+  mlir::Value lhsHalf = builder.createVecShuffle(loc, lhs, lhs, halfIndices);
+  mlir::Value rhsHalf = builder.createVecShuffle(loc, rhs, rhs, halfIndices);
+
+  // Create indices for concatenating the vectors.
+  // NOTE: Operands are swapped to match the intrinsic definition.
+  // After the half extraction, both vectors have numElems/2 elements.
+  // In createVecShuffle(rhsHalf, lhsHalf, indices), indices [0..numElems/2-1]
+  // select from rhsHalf, and indices [numElems/2..numElems-1] select from
+  // lhsHalf.
+  SmallVector<mlir::Attribute, 64> concatIndices;
+  for (auto i : llvm::seq<unsigned>(0, numElems))
+    concatIndices.push_back(cir::IntAttr::get(i32Ty, i));
+
+  // Concat the vectors (RHS first, then LHS).
+  mlir::Value res =
+      builder.createVecShuffle(loc, rhsHalf, lhsHalf, concatIndices);
+  return builder.createBitcast(res, ops[0].getType());
+}
+
 static mlir::Value emitX86MaskLogic(CIRGenBuilderTy &builder,
                                     mlir::Location loc,
                                     cir::BinOpKind binOpKind,
@@ -130,6 +235,40 @@ static mlir::Value emitVecInsert(CIRGenBuilderTy &builder, mlir::Location loc,
   return cir::VecInsertOp::create(builder, loc, vec, value, indexVal);
 }
 
+static mlir::Value emitX86FunnelShift(CIRGenBuilderTy &builder,
+                                      mlir::Location location, mlir::Value &op0,
+                                      mlir::Value &op1, mlir::Value &amt,
+                                      bool isRight) {
+  mlir::Type op0Ty = op0.getType();
+
+  // Amount may be scalar immediate, in which case create a splat vector.
+  // Funnel shifts amounts are treated as modulo and types are all power-of-2
+  // so we only care about the lowest log2 bits anyway.
+  if (amt.getType() != op0Ty) {
+    auto vecTy = mlir::cast<cir::VectorType>(op0Ty);
+    uint64_t numElems = vecTy.getSize();
+
+    auto amtTy = mlir::cast<cir::IntType>(amt.getType());
+    auto vecElemTy = mlir::cast<cir::IntType>(vecTy.getElementType());
+
+    // If signed, cast to the same width but unsigned first to
+    // ensure zero-extension when casting to a bigger unsigned `vecElemeTy`.
+    if (amtTy.isSigned()) {
+      cir::IntType unsignedAmtTy = builder.getUIntNTy(amtTy.getWidth());
+      amt = builder.createIntCast(amt, unsignedAmtTy);
+    }
+    cir::IntType unsignedVecElemType = builder.getUIntNTy(vecElemTy.getWidth());
+    amt = builder.createIntCast(amt, unsignedVecElemType);
+    amt = cir::VecSplatOp::create(
+        builder, location, cir::VectorType::get(unsignedVecElemType, numElems),
+        amt);
+  }
+
+  const StringRef intrinsicName = isRight ? "fshr" : "fshl";
+  return emitIntrinsicCallOp(builder, location, intrinsicName, op0Ty,
+                             mlir::ValueRange{op0, op1, amt});
+}
+
 mlir::Value CIRGenFunction::emitX86BuiltinExpr(unsigned builtinID,
                                                const CallExpr *expr) {
   if (builtinID == Builtin::BI__builtin_cpu_is) {
@@ -232,9 +371,7 @@ mlir::Value CIRGenFunction::emitX86BuiltinExpr(unsigned builtinID,
   case X86::BI__builtin_ia32_vec_ext_v4di: {
     unsigned numElts = cast<cir::VectorType>(ops[0].getType()).getSize();
 
-    uint64_t index =
-        ops[1].getDefiningOp<cir::ConstantOp>().getIntValue().getZExtValue();
-
+    uint64_t index = getZExtIntValueFromConstOp(ops[1]);
     index &= numElts - 1;
 
     cir::ConstantOp indexVal =
@@ -257,7 +394,15 @@ mlir::Value CIRGenFunction::emitX86BuiltinExpr(unsigned builtinID,
     return emitVecInsert(builder, getLoc(expr->getExprLoc()), ops[0], ops[1],
                          ops[2]);
   }
-
+  case X86::BI__builtin_ia32_kunpckhi:
+    return emitX86MaskUnpack(builder, getLoc(expr->getExprLoc()),
+                             "x86.avx512.kunpackb", ops);
+  case X86::BI__builtin_ia32_kunpcksi:
+    return emitX86MaskUnpack(builder, getLoc(expr->getExprLoc()),
+                             "x86.avx512.kunpackw", ops);
+  case X86::BI__builtin_ia32_kunpckdi:
+    return emitX86MaskUnpack(builder, getLoc(expr->getExprLoc()),
+                             "x86.avx512.kunpackd", ops);
   case X86::BI_mm_setcsr:
   case X86::BI__builtin_ia32_ldmxcsr: {
     mlir::Location loc = getLoc(expr->getExprLoc());
@@ -682,12 +827,20 @@ mlir::Value CIRGenFunction::emitX86BuiltinExpr(unsigned builtinID,
   case X86::BI__builtin_ia32_pblendw256:
   case X86::BI__builtin_ia32_pblendd128:
   case X86::BI__builtin_ia32_pblendd256:
+    cgm.errorNYI(expr->getSourceRange(),
+                 std::string("unimplemented X86 builtin call: ") +
+                     getContext().BuiltinInfo.getName(builtinID));
+    return {};
   case X86::BI__builtin_ia32_pshuflw:
   case X86::BI__builtin_ia32_pshuflw256:
   case X86::BI__builtin_ia32_pshuflw512:
+    return emitPshufWord(builder, ops[0], ops[1], getLoc(expr->getExprLoc()),
+                         true);
   case X86::BI__builtin_ia32_pshufhw:
   case X86::BI__builtin_ia32_pshufhw256:
   case X86::BI__builtin_ia32_pshufhw512:
+    return emitPshufWord(builder, ops[0], ops[1], getLoc(expr->getExprLoc()),
+                         false);
   case X86::BI__builtin_ia32_pshufd:
   case X86::BI__builtin_ia32_pshufd256:
   case X86::BI__builtin_ia32_pshufd512:
@@ -696,13 +849,28 @@ mlir::Value CIRGenFunction::emitX86BuiltinExpr(unsigned builtinID,
   case X86::BI__builtin_ia32_vpermilpd256:
   case X86::BI__builtin_ia32_vpermilps256:
   case X86::BI__builtin_ia32_vpermilpd512:
-  case X86::BI__builtin_ia32_vpermilps512:
+  case X86::BI__builtin_ia32_vpermilps512: {
+    const uint32_t imm = getSExtIntValueFromConstOp(ops[1]);
+
+    llvm::SmallVector<int64_t, 16> mask(16);
+    computeFullLaneShuffleMask(*this, ops[0], imm, false, mask);
+
+    return builder.createVecShuffle(getLoc(expr->getExprLoc()), ops[0], mask);
+  }
   case X86::BI__builtin_ia32_shufpd:
   case X86::BI__builtin_ia32_shufpd256:
   case X86::BI__builtin_ia32_shufpd512:
   case X86::BI__builtin_ia32_shufps:
   case X86::BI__builtin_ia32_shufps256:
-  case X86::BI__builtin_ia32_shufps512:
+  case X86::BI__builtin_ia32_shufps512: {
+    const uint32_t imm = getZExtIntValueFromConstOp(ops[2]);
+
+    llvm::SmallVector<int64_t, 16> mask(16);
+    computeFullLaneShuffleMask(*this, ops[0], imm, true, mask);
+
+    return builder.createVecShuffle(getLoc(expr->getExprLoc()), ops[0], ops[1],
+                                    mask);
+  }
   case X86::BI__builtin_ia32_permdi256:
   case X86::BI__builtin_ia32_permdf256:
   case X86::BI__builtin_ia32_permdi512:
@@ -796,12 +964,16 @@ mlir::Value CIRGenFunction::emitX86BuiltinExpr(unsigned builtinID,
   case X86::BI__builtin_ia32_prolq128:
   case X86::BI__builtin_ia32_prolq256:
   case X86::BI__builtin_ia32_prolq512:
+    return emitX86FunnelShift(builder, getLoc(expr->getExprLoc()), ops[0],
+                              ops[0], ops[1], false);
   case X86::BI__builtin_ia32_prord128:
   case X86::BI__builtin_ia32_prord256:
   case X86::BI__builtin_ia32_prord512:
   case X86::BI__builtin_ia32_prorq128:
   case X86::BI__builtin_ia32_prorq256:
   case X86::BI__builtin_ia32_prorq512:
+    return emitX86FunnelShift(builder, getLoc(expr->getExprLoc()), ops[0],
+                              ops[0], ops[1], true);
   case X86::BI__builtin_ia32_selectb_128:
   case X86::BI__builtin_ia32_selectb_256:
   case X86::BI__builtin_ia32_selectb_512:
@@ -947,9 +1119,6 @@ mlir::Value CIRGenFunction::emitX86BuiltinExpr(unsigned builtinID,
         getMaskVecValue(builder, getLoc(expr->getExprLoc()), ops[0], numElts);
     return builder.createBitcast(resVec, ops[0].getType());
   }
-  case X86::BI__builtin_ia32_kunpckdi:
-  case X86::BI__builtin_ia32_kunpcksi:
-  case X86::BI__builtin_ia32_kunpckhi:
   case X86::BI__builtin_ia32_sqrtsh_round_mask:
   case X86::BI__builtin_ia32_sqrtsd_round_mask:
   case X86::BI__builtin_ia32_sqrtss_round_mask:
diff --git a/clang/lib/CIR/CodeGen/CIRGenClass.cpp b/clang/lib/CIR/CodeGen/CIRGenClass.cpp
index c98d9bb0724f6..ca9fe939139cd 100644
--- a/clang/lib/CIR/CodeGen/CIRGenClass.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenClass.cpp
@@ -126,8 +126,7 @@ static void emitMemberInitializer(CIRGenFunction &cgf,
                             lhs.isVolatileQualified());
       // Ensure that we destroy the objects if an exception is thrown later in
       // the constructor.
-      QualType::DestructionKind dtorKind = fieldType.isDestructedType();
-      assert(!cgf.needsEHCleanup(dtorKind) &&
+      assert(!cgf.needsEHCleanup(fieldType.isDestructedType()) &&
              "Arrays of non-record types shouldn't need EH cleanup");
       return;
     }
diff --git a/clang/lib/CIR/CodeGen/CIRGenDeclOpenACC.cpp b/clang/lib/CIR/CodeGen/CIRGenDeclOpenACC.cpp
index d52986db49ea6..a5322ac4e1930 100644
--- a/clang/lib/CIR/CodeGen/CIRGenDeclOpenACC.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenDeclOpenACC.cpp
@@ -287,9 +287,92 @@ void CIRGenModule::emitGlobalOpenACCDeclareDecl(const OpenACCDeclareDecl *d) {
 }
 
 void CIRGenFunction::emitOpenACCRoutine(const OpenACCRoutineDecl &d) {
-  getCIRGenModule().errorNYI(d.getSourceRange(), "OpenACC Routine Construct");
+  // Do nothing here. The OpenACCRoutineDeclAttr handles the implicit name
+  // cases, and the end-of-TU handling manages the named cases. This is
+  // necessary because these references aren't necessarily emitted themselves,
+  // but can be named anywhere.
 }
 
 void CIRGenModule::emitGlobalOpenACCRoutineDecl(const OpenACCRoutineDecl *d) {
-  errorNYI(d->getSourceRange(), "OpenACC Global Routine Construct");
+  // Do nothing here. The OpenACCRoutineDeclAttr handles the implicit name
+  // cases, and the end-of-TU handling manages the named cases. This is
+  // necessary because these references aren't necessarily emitted themselves,
+  // but can be named anywhere.
+}
+
+namespace {
+class OpenACCRoutineClauseEmitter final
+    : public OpenACCClauseVisitor<OpenACCRoutineClauseEmitter> {
+  CIRGen::CIRGenBuilderTy &builder;
+  mlir::acc::RoutineOp routineOp;
+  llvm::SmallVector<mlir::acc::DeviceType> lastDeviceTypeValues;
+
+public:
+  OpenACCRoutineClauseEmitter(CIRGen::CIRGenBuilderTy &builder,
+                              mlir::acc::RoutineOp routineOp)
+      : builder(builder), routineOp(routineOp) {}
+
+  void emitClauses(ArrayRef<const OpenACCClause *> clauses) {
+    this->VisitClauseList(clauses);
+  }
+
+  void VisitClause(const OpenACCClause &clause) {
+    llvm_unreachable("Invalid OpenACC clause on routine");
+  }
+
+  void VisitSeqClause(const OpenACCSeqClause &clause) {
+    routineOp.addSeq(builder.getContext(), lastDeviceTypeValues);
+  }
+  void VisitWorkerClause(const OpenACCWorkerClause &clause) {
+    routineOp.addWorker(builder.getContext(), lastDeviceTypeValues);
+  }
+  void VisitVectorClause(const OpenACCVectorClause &clause) {
+    routineOp.addVector(builder.getContext(), lastDeviceTypeValues);
+  }
+
+  void VisitNoHostClause(const OpenACCNoHostClause &clause) {
+    routineOp.setNohost(/*attrValue=*/true);
+  }
+};
+} // namespace
+
+void CIRGenModule::emitOpenACCRoutineDecl(
+    const clang::FunctionDecl *funcDecl, cir::FuncOp func,
+    SourceLocation pragmaLoc, ArrayRef<const OpenACCClause *> clauses) {
+  mlir::OpBuilder::InsertionGuard guardCase(builder);
+  // These need to appear at the global module.
+  builder.setInsertionPointToEnd(&getModule().getBodyRegion().front());
+
+  mlir::Location routineLoc = getLoc(pragmaLoc);
+
+  std::stringstream routineNameSS;
+  // This follows the same naming format as Flang.
+  routineNameSS << "acc_routine_" << routineCounter++;
+  std::string routineName = routineNameSS.str();
+
+  // There isn't a good constructor for RoutineOp that just takes a location +
+  // name + function, so we use one that creates an otherwise RoutineOp and
+  // count on the visitor/emitter to fill these in.
+  auto routineOp = mlir::acc::RoutineOp::create(
+      builder, routineLoc, routineName,
+      mlir::SymbolRefAttr::get(builder.getContext(), func.getName()),
+      /*implicit=*/false);
+
+  // We have to add a pointer going the other direction via an acc.routine_info,
+  // from the func to the routine.
+  llvm::SmallVector<mlir::SymbolRefAttr> funcRoutines;
+  if (auto routineInfo =
+          func.getOperation()->getAttrOfType<mlir::acc::RoutineInfoAttr>(
+              mlir::acc::getRoutineInfoAttrName()))
+    funcRoutines.append(routineInfo.getAccRoutines().begin(),
+                        routineInfo.getAccRoutines().end());
+
+  funcRoutines.push_back(
+      mlir::SymbolRefAttr::get(builder.getContext(), routineName));
+  func.getOperation()->setAttr(
+      mlir::acc::getRoutineInfoAttrName(),
+      mlir::acc::RoutineInfoAttr::get(func.getContext(), funcRoutines));
+
+  OpenACCRoutineClauseEmitter emitter{builder, routineOp};
+  emitter.emitClauses(clauses);
 }
diff --git a/clang/lib/CIR/CodeGen/CIRGenExpr.cpp b/clang/lib/CIR/CodeGen/CIRGenExpr.cpp
index 1c2b83e6094f3..5d509e37f4621 100644
--- a/clang/lib/CIR/CodeGen/CIRGenExpr.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenExpr.cpp
@@ -188,6 +188,7 @@ Address CIRGenFunction::emitPointerWithAlignment(const Expr *expr,
     case CK_HLSLArrayRValue:
     case CK_HLSLElementwiseCast:
     case CK_HLSLVectorTruncation:
+    case CK_HLSLMatrixTruncation:
     case CK_IntToOCLSampler:
     case CK_IntegralCast:
     case CK_IntegralComplexCast:
@@ -1323,6 +1324,7 @@ LValue CIRGenFunction::emitCastLValue(const CastExpr *e) {
   case CK_IntegralToFixedPoint:
   case CK_MatrixCast:
   case CK_HLSLVectorTruncation:
+  case CK_HLSLMatrixTruncation:
   case CK_HLSLArrayRValue:
   case CK_HLSLElementwiseCast:
   case CK_HLSLAggregateSplatCast:
diff --git a/clang/lib/CIR/CodeGen/CIRGenExprComplex.cpp b/clang/lib/CIR/CodeGen/CIRGenExprComplex.cpp
index 9ed920085c8c6..fe06f8cc2c430 100644
--- a/clang/lib/CIR/CodeGen/CIRGenExprComplex.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenExprComplex.cpp
@@ -534,6 +534,7 @@ mlir::Value ComplexExprEmitter::emitCast(CastKind ck, Expr *op,
   case CK_IntegralToFixedPoint:
   case CK_MatrixCast:
   case CK_HLSLVectorTruncation:
+  case CK_HLSLMatrixTruncation:
   case CK_HLSLArrayRValue:
   case CK_HLSLElementwiseCast:
   case CK_HLSLAggregateSplatCast:
diff --git a/clang/lib/CIR/CodeGen/CIRGenExprConstant.cpp b/clang/lib/CIR/CodeGen/CIRGenExprConstant.cpp
index 66f8ef9b05913..329fd08bc8914 100644
--- a/clang/lib/CIR/CodeGen/CIRGenExprConstant.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenExprConstant.cpp
@@ -1012,6 +1012,7 @@ class ConstExprEmitter
     case CK_MatrixCast:
     case CK_HLSLArrayRValue:
     case CK_HLSLVectorTruncation:
+    case CK_HLSLMatrixTruncation:
     case CK_HLSLElementwiseCast:
     case CK_HLSLAggregateSplatCast:
       return {};
diff --git a/clang/lib/CIR/CodeGen/CIRGenFunction.h b/clang/lib/CIR/CodeGen/CIRGenFunction.h
index b6926bb88ac85..ada16ff187430 100644
--- a/clang/lib/CIR/CodeGen/CIRGenFunction.h
+++ b/clang/lib/CIR/CodeGen/CIRGenFunction.h
@@ -203,6 +203,22 @@ class CIRGenFunction : public CIRGenTypeCache {
     return convertType(getContext().getTypeDeclType(t));
   }
 
+  /// Get integer from a mlir::Value that is an int constant or a constant op.
+  static int64_t getSExtIntValueFromConstOp(mlir::Value val) {
+    auto constOp = val.getDefiningOp<cir::ConstantOp>();
+    assert(constOp && "getIntValueFromConstOp call with non ConstantOp");
+    return constOp.getIntValue().getSExtValue();
+  }
+
+  /// Get zero-extended integer from a mlir::Value that is an int constant or a
+  /// constant op.
+  static int64_t getZExtIntValueFromConstOp(mlir::Value val) {
+    auto constOp = val.getDefiningOp<cir::ConstantOp>();
+    assert(constOp &&
+           "getZeroExtendedIntValueFromConstOp call with non ConstantOp");
+    return constOp.getIntValue().getZExtValue();
+  }
+
   ///  Return the cir::TypeEvaluationKind of QualType \c type.
   static cir::TypeEvaluationKind getEvaluationKind(clang::QualType type);
 
@@ -1816,7 +1832,7 @@ class CIRGenFunction : public CIRGenTypeCache {
 
   mlir::LogicalResult emitWhileStmt(const clang::WhileStmt &s);
 
-  mlir::Value emitX86BuiltinExpr(unsigned builtinID, const CallExpr *e);
+  mlir::Value emitX86BuiltinExpr(unsigned builtinID, const CallExpr *expr);
 
   /// Given an assignment `*lhs = rhs`, emit a test that checks if \p rhs is
   /// nonnull, if 1\p LHS is marked _Nonnull.
diff --git a/clang/lib/CIR/CodeGen/CIRGenModule.cpp b/clang/lib/CIR/CodeGen/CIRGenModule.cpp
index 03bbfbffce717..1d8e4a3b444ee 100644
--- a/clang/lib/CIR/CodeGen/CIRGenModule.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenModule.cpp
@@ -2227,6 +2227,15 @@ CIRGenModule::createCIRFunction(mlir::Location loc, StringRef name,
 
     if (!cgf)
       theModule.push_back(func);
+
+    if (this->getLangOpts().OpenACC) {
+      // We only have to handle this attribute, since OpenACCAnnotAttrs are
+      // handled via the end-of-TU work.
+      for (const auto *attr :
+           funcDecl->specific_attrs<OpenACCRoutineDeclAttr>())
+        emitOpenACCRoutineDecl(funcDecl, func, attr->getLocation(),
+                               attr->Clauses);
+    }
   }
   return func;
 }
diff --git a/clang/lib/CIR/CodeGen/CIRGenModule.h b/clang/lib/CIR/CodeGen/CIRGenModule.h
index 6600d086f8f61..d7aee8ebf4d7a 100644
--- a/clang/lib/CIR/CodeGen/CIRGenModule.h
+++ b/clang/lib/CIR/CodeGen/CIRGenModule.h
@@ -461,6 +461,12 @@ class CIRGenModule : public CIRGenTypeCache {
                                             OpenACCModifierKind modifiers,
                                             bool structured, bool implicit,
                                             bool requiresDtor);
+  // Each of the acc.routine operations must have a unique name, so we just use
+  // an integer counter.  This is how Flang does it, so it seems reasonable.
+  unsigned routineCounter = 0;
+  void emitOpenACCRoutineDecl(const clang::FunctionDecl *funcDecl,
+                              cir::FuncOp func, SourceLocation pragmaLoc,
+                              ArrayRef<const OpenACCClause *> clauses);
 
   // C++ related functions.
   void emitDeclContext(const DeclContext *dc);
diff --git a/clang/lib/CIR/CodeGen/CIRGenerator.cpp b/clang/lib/CIR/CodeGen/CIRGenerator.cpp
index aa4d9eba35c04..0208eeea7146a 100644
--- a/clang/lib/CIR/CodeGen/CIRGenerator.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenerator.cpp
@@ -166,6 +166,18 @@ void CIRGenerator::HandleCXXStaticMemberVarInstantiation(VarDecl *D) {
   cgm->handleCXXStaticMemberVarInstantiation(D);
 }
 
+void CIRGenerator::HandleOpenACCRoutineReference(const FunctionDecl *FD,
+                                                 const OpenACCRoutineDecl *RD) {
+  llvm::StringRef mangledName = cgm->getMangledName(FD);
+  cir::FuncOp entry =
+      mlir::dyn_cast_if_present<cir::FuncOp>(cgm->getGlobalValue(mangledName));
+
+  // if this wasn't generated, don't force it to be.
+  if (!entry)
+    return;
+  cgm->emitOpenACCRoutineDecl(FD, entry, RD->getBeginLoc(), RD->clauses());
+}
+
 void CIRGenerator::CompleteTentativeDefinition(VarDecl *d) {
   if (diags.hasErrorOccurred())
     return;
diff --git a/clang/lib/CIR/Dialect/Transforms/LoweringPrepare.cpp b/clang/lib/CIR/Dialect/Transforms/LoweringPrepare.cpp
index cedc2a73b9260..94e143e202736 100644
--- a/clang/lib/CIR/Dialect/Transforms/LoweringPrepare.cpp
+++ b/clang/lib/CIR/Dialect/Transforms/LoweringPrepare.cpp
@@ -74,6 +74,7 @@ struct LoweringPreparePass
   void lowerDynamicCastOp(cir::DynamicCastOp op);
   void lowerArrayDtor(cir::ArrayDtor op);
   void lowerArrayCtor(cir::ArrayCtor op);
+  void lowerTrivialCopyCall(cir::CallOp op);
 
   /// Build the function that initializes the specified global
   cir::FuncOp buildCXXGlobalVarDeclInitFunc(cir::GlobalOp op);
@@ -1086,6 +1087,25 @@ void LoweringPreparePass::lowerArrayCtor(cir::ArrayCtor op) {
                              true);
 }
 
+void LoweringPreparePass::lowerTrivialCopyCall(cir::CallOp op) {
+  cir::FuncOp funcOp = getCalledFunction(op);
+  if (!funcOp)
+    return;
+
+  std::optional<cir::CtorKind> ctorKind = funcOp.getCxxConstructorKind();
+  if (ctorKind && *ctorKind == cir::CtorKind::Copy &&
+      funcOp.isCxxTrivialMemberFunction()) {
+    // Replace the trivial copy constructor call with a `CopyOp`
+    CIRBaseBuilderTy builder(getContext());
+    mlir::ValueRange operands = op.getOperands();
+    mlir::Value dest = operands[0];
+    mlir::Value src = operands[1];
+    builder.setInsertionPoint(op);
+    builder.createCopy(dest, src);
+    op.erase();
+  }
+}
+
 void LoweringPreparePass::runOnOp(mlir::Operation *op) {
   if (auto arrayCtor = dyn_cast<cir::ArrayCtor>(op)) {
     lowerArrayCtor(arrayCtor);
@@ -1103,6 +1123,8 @@ void LoweringPreparePass::runOnOp(mlir::Operation *op) {
     lowerDynamicCastOp(dynamicCast);
   } else if (auto unary = mlir::dyn_cast<cir::UnaryOp>(op)) {
     lowerUnaryOp(unary);
+  } else if (auto callOp = dyn_cast<cir::CallOp>(op)) {
+    lowerTrivialCopyCall(callOp);
   } else if (auto fnOp = dyn_cast<cir::FuncOp>(op)) {
     if (auto globalCtor = fnOp.getGlobalCtorPriority())
       globalCtorList.emplace_back(fnOp.getName(), globalCtor.value());
@@ -1121,7 +1143,7 @@ void LoweringPreparePass::runOnOperation() {
   op->walk([&](mlir::Operation *op) {
     if (mlir::isa<cir::ArrayCtor, cir::ArrayDtor, cir::CastOp,
                   cir::ComplexMulOp, cir::ComplexDivOp, cir::DynamicCastOp,
-                  cir::FuncOp, cir::GlobalOp, cir::UnaryOp>(op))
+                  cir::FuncOp, cir::CallOp, cir::GlobalOp, cir::UnaryOp>(op))
       opsToTransform.push_back(op);
   });
 
diff --git a/clang/lib/CIR/FrontendAction/CIRGenAction.cpp b/clang/lib/CIR/FrontendAction/CIRGenAction.cpp
index 67bb5657d4001..daec8ae409e0f 100644
--- a/clang/lib/CIR/FrontendAction/CIRGenAction.cpp
+++ b/clang/lib/CIR/FrontendAction/CIRGenAction.cpp
@@ -88,6 +88,11 @@ class CIRGenConsumer : public clang::ASTConsumer {
     Gen->HandleCXXStaticMemberVarInstantiation(VD);
   }
 
+  void HandleOpenACCRoutineReference(const FunctionDecl *FD,
+                                     const OpenACCRoutineDecl *RD) override {
+    Gen->HandleOpenACCRoutineReference(FD, RD);
+  }
+
   void HandleInlineFunctionDefinition(FunctionDecl *D) override {
     Gen->HandleInlineFunctionDefinition(D);
   }
diff --git a/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp b/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp
index 8e9780754f68f..40e14474890dc 100644
--- a/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp
+++ b/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp
@@ -3979,6 +3979,13 @@ mlir::LogicalResult CIRToLLVMGetBitfieldOpLowering::matchAndRewrite(
   return mlir::success();
 }
 
+mlir::LogicalResult CIRToLLVMIsConstantOpLowering::matchAndRewrite(
+    cir::IsConstantOp op, OpAdaptor adaptor,
+    mlir::ConversionPatternRewriter &rewriter) const {
+  rewriter.replaceOpWithNewOp<mlir::LLVM::IsConstantOp>(op, adaptor.getVal());
+  return mlir::success();
+}
+
 mlir::LogicalResult CIRToLLVMInlineAsmOpLowering::matchAndRewrite(
     cir::InlineAsmOp op, OpAdaptor adaptor,
     mlir::ConversionPatternRewriter &rewriter) const {
diff --git a/clang/lib/CodeGen/CGExpr.cpp b/clang/lib/CodeGen/CGExpr.cpp
index 423be1826aa47..61eb7fee3b120 100644
--- a/clang/lib/CodeGen/CGExpr.cpp
+++ b/clang/lib/CodeGen/CGExpr.cpp
@@ -5798,6 +5798,7 @@ LValue CodeGenFunction::EmitCastLValue(const CastExpr *E) {
   case CK_IntegralToFixedPoint:
   case CK_MatrixCast:
   case CK_HLSLVectorTruncation:
+  case CK_HLSLMatrixTruncation:
   case CK_HLSLArrayRValue:
   case CK_HLSLElementwiseCast:
   case CK_HLSLAggregateSplatCast:
@@ -6356,8 +6357,15 @@ LValue CodeGenFunction::EmitBinaryOperatorLValue(const BinaryOperator *E) {
 LValue CodeGenFunction::EmitHLSLArrayAssignLValue(const BinaryOperator *E) {
   // Don't emit an LValue for the RHS because it might not be an LValue
   LValue LHS = EmitLValue(E->getLHS());
+
+  // If the RHS is a global resource array, copy all individual resources
+  // into LHS.
+  if (E->getRHS()->getType()->isHLSLResourceRecordArray())
+    if (CGM.getHLSLRuntime().emitResourceArrayCopy(LHS, E->getRHS(), *this))
+      return LHS;
+
   // In C the RHS of an assignment operator is an RValue.
-  // EmitAggregateAssign takes anan LValue for the RHS. Instead we can call
+  // EmitAggregateAssign takes an LValue for the RHS. Instead we can call
   // EmitInitializationToLValue to emit an RValue into an LValue.
   EmitInitializationToLValue(E->getRHS(), LHS);
   return LHS;
diff --git a/clang/lib/CodeGen/CGExprAgg.cpp b/clang/lib/CodeGen/CGExprAgg.cpp
index 67b5f919d1b2a..7cc4d6c8f06f6 100644
--- a/clang/lib/CodeGen/CGExprAgg.cpp
+++ b/clang/lib/CodeGen/CGExprAgg.cpp
@@ -1036,7 +1036,7 @@ void AggExprEmitter::VisitCastExpr(CastExpr *E) {
   case CK_ZeroToOCLOpaqueType:
   case CK_MatrixCast:
   case CK_HLSLVectorTruncation:
-
+  case CK_HLSLMatrixTruncation:
   case CK_IntToOCLSampler:
   case CK_FloatingToFixedPoint:
   case CK_FixedPointToFloating:
@@ -1550,6 +1550,7 @@ static bool castPreservesZero(const CastExpr *CE) {
   case CK_NonAtomicToAtomic:
   case CK_AtomicToNonAtomic:
   case CK_HLSLVectorTruncation:
+  case CK_HLSLMatrixTruncation:
   case CK_HLSLElementwiseCast:
   case CK_HLSLAggregateSplatCast:
     return true;
diff --git a/clang/lib/CodeGen/CGExprComplex.cpp b/clang/lib/CodeGen/CGExprComplex.cpp
index bca7c30557f03..e5815ef1130dc 100644
--- a/clang/lib/CodeGen/CGExprComplex.cpp
+++ b/clang/lib/CodeGen/CGExprComplex.cpp
@@ -621,6 +621,7 @@ ComplexPairTy ComplexExprEmitter::EmitCast(CastKind CK, Expr *Op,
   case CK_IntegralToFixedPoint:
   case CK_MatrixCast:
   case CK_HLSLVectorTruncation:
+  case CK_HLSLMatrixTruncation:
   case CK_HLSLArrayRValue:
   case CK_HLSLElementwiseCast:
   case CK_HLSLAggregateSplatCast:
diff --git a/clang/lib/CodeGen/CGExprConstant.cpp b/clang/lib/CodeGen/CGExprConstant.cpp
index 6407afc3d9447..0eec4dba4824a 100644
--- a/clang/lib/CodeGen/CGExprConstant.cpp
+++ b/clang/lib/CodeGen/CGExprConstant.cpp
@@ -1333,6 +1333,7 @@ class ConstExprEmitter
     case CK_ZeroToOCLOpaqueType:
     case CK_MatrixCast:
     case CK_HLSLVectorTruncation:
+    case CK_HLSLMatrixTruncation:
     case CK_HLSLArrayRValue:
     case CK_HLSLElementwiseCast:
     case CK_HLSLAggregateSplatCast:
diff --git a/clang/lib/CodeGen/CGExprScalar.cpp b/clang/lib/CodeGen/CGExprScalar.cpp
index 334951251c634..b1587500e6a11 100644
--- a/clang/lib/CodeGen/CGExprScalar.cpp
+++ b/clang/lib/CodeGen/CGExprScalar.cpp
@@ -2426,9 +2426,31 @@ static Value *EmitHLSLElementwiseCast(CodeGenFunction &CGF, LValue SrcVal,
     }
     return V;
   }
+  if (auto *MatTy = DestTy->getAs<ConstantMatrixType>()) {
+    assert(LoadList.size() >= MatTy->getNumElementsFlattened() &&
+           "Flattened type on RHS must have the same number or more elements "
+           "than vector on LHS.");
+
+    llvm::Value *V =
+        CGF.Builder.CreateLoad(CGF.CreateIRTemp(DestTy, "flatcast.tmp"));
+    // V is an allocated temporary to build the truncated matrix into.
+    for (unsigned I = 0, E = MatTy->getNumElementsFlattened(); I < E; I++) {
+      unsigned ColMajorIndex =
+          (I % MatTy->getNumRows()) * MatTy->getNumColumns() +
+          (I / MatTy->getNumRows());
+      RValue RVal = CGF.EmitLoadOfLValue(LoadList[ColMajorIndex], Loc);
+      assert(RVal.isScalar() &&
+             "All flattened source values should be scalars.");
+      llvm::Value *Cast = CGF.EmitScalarConversion(
+          RVal.getScalarVal(), LoadList[ColMajorIndex].getType(),
+          MatTy->getElementType(), Loc);
+      V = CGF.Builder.CreateInsertElement(V, Cast, I);
+    }
+    return V;
+  }
   // if its a builtin just do an extract element or load.
   assert(DestTy->isBuiltinType() &&
-         "Destination type must be a vector or builtin type.");
+         "Destination type must be a vector, matrix, or builtin type.");
   RValue RVal = CGF.EmitLoadOfLValue(LoadList[0], Loc);
   assert(RVal.isScalar() && "All flattened source values should be scalars.");
   return CGF.EmitScalarConversion(RVal.getScalarVal(), LoadList[0].getType(),
@@ -2958,15 +2980,47 @@ Value *ScalarExprEmitter::VisitCastExpr(CastExpr *CE) {
     llvm::Value *Zero = llvm::Constant::getNullValue(CGF.SizeTy);
     return Builder.CreateExtractElement(Vec, Zero, "cast.vtrunc");
   }
+  case CK_HLSLMatrixTruncation: {
+    assert((DestTy->isMatrixType() || DestTy->isBuiltinType()) &&
+           "Destination type must be a matrix or builtin type.");
+    Value *Mat = Visit(E);
+    if (auto *MatTy = DestTy->getAs<ConstantMatrixType>()) {
+      SmallVector<int> Mask;
+      unsigned NumCols = MatTy->getNumColumns();
+      unsigned NumRows = MatTy->getNumRows();
+      unsigned ColOffset = NumCols;
+      if (auto *SrcMatTy = E->getType()->getAs<ConstantMatrixType>())
+        ColOffset = SrcMatTy->getNumColumns();
+      for (unsigned R = 0; R < NumRows; R++) {
+        for (unsigned C = 0; C < NumCols; C++) {
+          unsigned I = R * ColOffset + C;
+          Mask.push_back(I);
+        }
+      }
+
+      return Builder.CreateShuffleVector(Mat, Mask, "trunc");
+    }
+    llvm::Value *Zero = llvm::Constant::getNullValue(CGF.SizeTy);
+    return Builder.CreateExtractElement(Mat, Zero, "cast.mtrunc");
+  }
   case CK_HLSLElementwiseCast: {
     RValue RV = CGF.EmitAnyExpr(E);
     SourceLocation Loc = CE->getExprLoc();
 
-    assert(RV.isAggregate() && "Not a valid HLSL Elementwise Cast.");
-    // RHS is an aggregate
-    LValue SrcVal = CGF.MakeAddrLValue(RV.getAggregateAddress(), E->getType());
+    Address SrcAddr = Address::invalid();
+
+    if (RV.isAggregate()) {
+      SrcAddr = RV.getAggregateAddress();
+    } else {
+      SrcAddr = CGF.CreateMemTemp(E->getType(), "hlsl.ewcast.src");
+      LValue TmpLV = CGF.MakeAddrLValue(SrcAddr, E->getType());
+      CGF.EmitStoreThroughLValue(RV, TmpLV);
+    }
+
+    LValue SrcVal = CGF.MakeAddrLValue(SrcAddr, E->getType());
     return EmitHLSLElementwiseCast(CGF, SrcVal, DestTy, Loc);
   }
+
   } // end of switch
 
   llvm_unreachable("unknown scalar cast");
diff --git a/clang/lib/CodeGen/CGHLSLRuntime.cpp b/clang/lib/CodeGen/CGHLSLRuntime.cpp
index 0c3701eb01679..f485fdd49e43f 100644
--- a/clang/lib/CodeGen/CGHLSLRuntime.cpp
+++ b/clang/lib/CodeGen/CGHLSLRuntime.cpp
@@ -22,6 +22,7 @@
 #include "clang/AST/ASTContext.h"
 #include "clang/AST/Attrs.inc"
 #include "clang/AST/Decl.h"
+#include "clang/AST/Expr.h"
 #include "clang/AST/HLSLResource.h"
 #include "clang/AST/RecursiveASTVisitor.h"
 #include "clang/AST/Type.h"
@@ -94,6 +95,14 @@ void addRootSignatureMD(llvm::dxbc::RootSignatureVersion RootSigVer,
   RootSignatureValMD->addOperand(MDVals);
 }
 
+// Find array variable declaration from DeclRef expression
+static const ValueDecl *getArrayDecl(const Expr *E) {
+  if (const DeclRefExpr *DRE =
+          dyn_cast_or_null<DeclRefExpr>(E->IgnoreImpCasts()))
+    return DRE->getDecl();
+  return nullptr;
+}
+
 // Find array variable declaration from nested array subscript AST nodes
 static const ValueDecl *getArrayDecl(const ArraySubscriptExpr *ASE) {
   const Expr *E = nullptr;
@@ -103,9 +112,7 @@ static const ValueDecl *getArrayDecl(const ArraySubscriptExpr *ASE) {
       return nullptr;
     ASE = dyn_cast<ArraySubscriptExpr>(E);
   }
-  if (const DeclRefExpr *DRE = dyn_cast_or_null<DeclRefExpr>(E))
-    return DRE->getDecl();
-  return nullptr;
+  return getArrayDecl(E);
 }
 
 // Get the total size of the array, or -1 if the array is unbounded.
@@ -1214,12 +1221,13 @@ std::optional<LValue> CGHLSLRuntime::emitResourceArraySubscriptExpr(
           ArraySubsExpr->getType()->isHLSLResourceRecordArray()) &&
          "expected resource array subscript expression");
 
-  // Let clang codegen handle local resource array subscripts,
+  // Let clang codegen handle local and static resource array subscripts,
   // or when the subscript references on opaque expression (as part of
   // ArrayInitLoopExpr AST node).
   const VarDecl *ArrayDecl =
       dyn_cast_or_null<VarDecl>(getArrayDecl(ArraySubsExpr));
-  if (!ArrayDecl || !ArrayDecl->hasGlobalStorage())
+  if (!ArrayDecl || !ArrayDecl->hasGlobalStorage() ||
+      ArrayDecl->getStorageClass() == SC_Static)
     return std::nullopt;
 
   // get the resource array type
@@ -1249,7 +1257,7 @@ std::optional<LValue> CGHLSLRuntime::emitResourceArraySubscriptExpr(
   // Find binding info for the resource array. For implicit binding
   // an HLSLResourceBindingAttr should have been added by SemaHLSL.
   ResourceBindingAttrs Binding(ArrayDecl);
-  assert((Binding.hasBinding()) &&
+  assert(Binding.hasBinding() &&
          "resource array must have a binding attribute");
 
   // Find the individual resource type.
@@ -1305,6 +1313,49 @@ std::optional<LValue> CGHLSLRuntime::emitResourceArraySubscriptExpr(
   return CGF.MakeAddrLValue(TmpVar, ResultTy, AlignmentSource::Decl);
 }
 
+// If RHSExpr is a global resource array, initialize all of its resources and
+// set them into LHS. Returns false if no copy has been performed and the
+// array copy should be handled by Clang codegen.
+bool CGHLSLRuntime::emitResourceArrayCopy(LValue &LHS, Expr *RHSExpr,
+                                          CodeGenFunction &CGF) {
+  QualType ResultTy = RHSExpr->getType();
+  assert(ResultTy->isHLSLResourceRecordArray() && "expected resource array");
+
+  // Let Clang codegen handle local and static resource array copies.
+  const VarDecl *ArrayDecl = dyn_cast_or_null<VarDecl>(getArrayDecl(RHSExpr));
+  if (!ArrayDecl || !ArrayDecl->hasGlobalStorage() ||
+      ArrayDecl->getStorageClass() == SC_Static)
+    return false;
+
+  // Find binding info for the resource array. For implicit binding
+  // the HLSLResourceBindingAttr should have been added by SemaHLSL.
+  ResourceBindingAttrs Binding(ArrayDecl);
+  assert(Binding.hasBinding() &&
+         "resource array must have a binding attribute");
+
+  // Find the individual resource type.
+  ASTContext &AST = ArrayDecl->getASTContext();
+  QualType ResTy = AST.getBaseElementType(ResultTy);
+  const auto *ResArrayTy = cast<ConstantArrayType>(ResultTy.getTypePtr());
+
+  // Use the provided LHS for the result.
+  AggValueSlot ValueSlot = AggValueSlot::forAddr(
+      LHS.getAddress(), Qualifiers(), AggValueSlot::IsDestructed_t(true),
+      AggValueSlot::DoesNotNeedGCBarriers, AggValueSlot::IsAliased_t(false),
+      AggValueSlot::DoesNotOverlap);
+
+  // Create Value for index and total array size (= range size).
+  int Size = getTotalArraySize(AST, ResArrayTy);
+  llvm::Value *Zero = llvm::ConstantInt::get(CGM.IntTy, 0);
+  llvm::Value *Range = llvm::ConstantInt::get(CGM.IntTy, Size);
+
+  // Initialize individual resources in the array into LHS.
+  std::optional<llvm::Value *> EndIndex = initializeLocalResourceArray(
+      CGF, ResTy->getAsCXXRecordDecl(), ResArrayTy, ValueSlot, Range, Zero,
+      ArrayDecl->getName(), Binding, {Zero}, RHSExpr->getExprLoc());
+  return EndIndex.has_value();
+}
+
 std::optional<LValue> CGHLSLRuntime::emitBufferArraySubscriptExpr(
     const ArraySubscriptExpr *E, CodeGenFunction &CGF,
     llvm::function_ref<llvm::Value *(bool Promote)> EmitIdxAfterBase) {
diff --git a/clang/lib/CodeGen/CGHLSLRuntime.h b/clang/lib/CodeGen/CGHLSLRuntime.h
index 77f43e8766745..c7cd668419d10 100644
--- a/clang/lib/CodeGen/CGHLSLRuntime.h
+++ b/clang/lib/CodeGen/CGHLSLRuntime.h
@@ -258,6 +258,7 @@ class CGHLSLRuntime {
   std::optional<LValue>
   emitResourceArraySubscriptExpr(const ArraySubscriptExpr *E,
                                  CodeGenFunction &CGF);
+  bool emitResourceArrayCopy(LValue &LHS, Expr *RHSExpr, CodeGenFunction &CGF);
 
   std::optional<LValue> emitBufferArraySubscriptExpr(
       const ArraySubscriptExpr *E, CodeGenFunction &CGF,
diff --git a/clang/lib/CodeGen/CodeGenModule.cpp b/clang/lib/CodeGen/CodeGenModule.cpp
index eda65739771cf..8d14ab7d6dccd 100644
--- a/clang/lib/CodeGen/CodeGenModule.cpp
+++ b/clang/lib/CodeGen/CodeGenModule.cpp
@@ -5983,7 +5983,8 @@ void CodeGenModule::EmitGlobalVarDefinition(const VarDecl *D,
              (D->getType()->isHLSLResourceRecord() ||
               D->getType()->isHLSLResourceRecordArray())) {
     Init = llvm::PoisonValue::get(getTypes().ConvertType(ASTTy));
-    NeedsGlobalCtor = D->getType()->isHLSLResourceRecord();
+    NeedsGlobalCtor = D->getType()->isHLSLResourceRecord() ||
+                      D->getStorageClass() == SC_Static;
   } else if (D->hasAttr<LoaderUninitializedAttr>()) {
     Init = llvm::UndefValue::get(getTypes().ConvertTypeForMem(ASTTy));
   } else if (!InitExpr) {
diff --git a/clang/lib/Edit/RewriteObjCFoundationAPI.cpp b/clang/lib/Edit/RewriteObjCFoundationAPI.cpp
index 40f8348241ecc..e8d4660fd36b2 100644
--- a/clang/lib/Edit/RewriteObjCFoundationAPI.cpp
+++ b/clang/lib/Edit/RewriteObjCFoundationAPI.cpp
@@ -1085,6 +1085,7 @@ static bool rewriteToNumericBoxedExpression(const ObjCMessageExpr *Msg,
       llvm_unreachable("OpenCL-specific cast in Objective-C?");
 
     case CK_HLSLVectorTruncation:
+    case CK_HLSLMatrixTruncation:
     case CK_HLSLElementwiseCast:
     case CK_HLSLAggregateSplatCast:
       llvm_unreachable("HLSL-specific cast in Objective-C?");
diff --git a/clang/lib/Sema/AnalysisBasedWarnings.cpp b/clang/lib/Sema/AnalysisBasedWarnings.cpp
index ae94b59760418..11691b2e9f9b7 100644
--- a/clang/lib/Sema/AnalysisBasedWarnings.cpp
+++ b/clang/lib/Sema/AnalysisBasedWarnings.cpp
@@ -36,7 +36,6 @@
 #include "clang/Analysis/Analyses/UnsafeBufferUsage.h"
 #include "clang/Analysis/AnalysisDeclContext.h"
 #include "clang/Analysis/CFG.h"
-#include "clang/Analysis/CFGStmtMap.h"
 #include "clang/Analysis/FlowSensitive/DataflowWorklist.h"
 #include "clang/Basic/Diagnostic.h"
 #include "clang/Basic/DiagnosticSema.h"
diff --git a/clang/lib/Sema/Sema.cpp b/clang/lib/Sema/Sema.cpp
index 325f62cf33444..71e74613973c5 100644
--- a/clang/lib/Sema/Sema.cpp
+++ b/clang/lib/Sema/Sema.cpp
@@ -1502,6 +1502,9 @@ void Sema::ActOnEndOfTranslationUnit() {
 
   if (LangOpts.HLSL)
     HLSL().ActOnEndOfTranslationUnit(getASTContext().getTranslationUnitDecl());
+  if (LangOpts.OpenACC)
+    OpenACC().ActOnEndOfTranslationUnit(
+        getASTContext().getTranslationUnitDecl());
 
   // If there were errors, disable 'unused' warnings since they will mostly be
   // noise. Don't warn for a use from a module: either we should warn on all
diff --git a/clang/lib/Sema/SemaChecking.cpp b/clang/lib/Sema/SemaChecking.cpp
index 70bd33d957a4c..42f3eb768dace 100644
--- a/clang/lib/Sema/SemaChecking.cpp
+++ b/clang/lib/Sema/SemaChecking.cpp
@@ -37,6 +37,7 @@
 #include "clang/AST/TemplateBase.h"
 #include "clang/AST/TemplateName.h"
 #include "clang/AST/Type.h"
+#include "clang/AST/TypeBase.h"
 #include "clang/AST/TypeLoc.h"
 #include "clang/AST/UnresolvedSet.h"
 #include "clang/Basic/AddressSpaces.h"
@@ -12572,9 +12573,10 @@ void Sema::CheckImplicitConversion(Expr *E, QualType T, SourceLocation CC,
       if (SourceMgr.isInSystemMacro(CC))
         return;
       return DiagnoseImpCast(*this, E, T, CC, diag::warn_impcast_vector_scalar);
-    } else if (getLangOpts().HLSL &&
-               Target->castAs<VectorType>()->getNumElements() <
-                   Source->castAs<VectorType>()->getNumElements()) {
+    }
+    if (getLangOpts().HLSL &&
+        Target->castAs<VectorType>()->getNumElements() <
+            Source->castAs<VectorType>()->getNumElements()) {
       // Diagnose vector truncation but don't return. We may also want to
       // diagnose an element conversion.
       DiagnoseImpCast(*this, E, T, CC,
@@ -12590,9 +12592,22 @@ void Sema::CheckImplicitConversion(Expr *E, QualType T, SourceLocation CC,
     Source = cast<VectorType>(Source)->getElementType().getTypePtr();
     Target = cast<VectorType>(Target)->getElementType().getTypePtr();
   }
-  if (auto VecTy = dyn_cast<VectorType>(Target))
+  if (const auto *VecTy = dyn_cast<VectorType>(Target))
     Target = VecTy->getElementType().getTypePtr();
 
+  if (isa<ConstantMatrixType>(Source)) {
+    if (Target->isScalarType())
+      return DiagnoseImpCast(*this, E, T, CC, diag::warn_impcast_matrix_scalar);
+
+    if (getLangOpts().HLSL &&
+        Target->castAs<ConstantMatrixType>()->getNumElementsFlattened() <
+            Source->castAs<ConstantMatrixType>()->getNumElementsFlattened()) {
+      // Diagnose Matrix truncation but don't return. We may also want to
+      // diagnose an element conversion.
+      DiagnoseImpCast(*this, E, T, CC,
+                      diag::warn_hlsl_impcast_matrix_truncation);
+    }
+  }
   // Strip complex types.
   if (isa<ComplexType>(Source)) {
     if (!isa<ComplexType>(Target)) {
diff --git a/clang/lib/Sema/SemaExprCXX.cpp b/clang/lib/Sema/SemaExprCXX.cpp
index d6f70e728be29..69719ebd1fc8c 100644
--- a/clang/lib/Sema/SemaExprCXX.cpp
+++ b/clang/lib/Sema/SemaExprCXX.cpp
@@ -5196,6 +5196,7 @@ Sema::PerformImplicitConversion(Expr *From, QualType ToType,
   case ICK_Incompatible_Pointer_Conversion:
   case ICK_HLSL_Array_RValue:
   case ICK_HLSL_Vector_Truncation:
+  case ICK_HLSL_Matrix_Truncation:
   case ICK_HLSL_Vector_Splat:
     llvm_unreachable("Improper second standard conversion");
   }
@@ -5203,12 +5204,10 @@ Sema::PerformImplicitConversion(Expr *From, QualType ToType,
   if (SCS.Dimension != ICK_Identity) {
     // If SCS.Element is not ICK_Identity the To and From types must be HLSL
     // vectors or matrices.
-
-    // TODO: Support HLSL matrices.
-    assert((!From->getType()->isMatrixType() && !ToType->isMatrixType()) &&
-           "Dimension conversion for matrix types is not implemented yet.");
-    assert((ToType->isVectorType() || ToType->isBuiltinType()) &&
-           "Dimension conversion output must be vector or scalar type.");
+    assert(
+        (ToType->isVectorType() || ToType->isConstantMatrixType() ||
+         ToType->isBuiltinType()) &&
+        "Dimension conversion output must be vector, matrix, or scalar type.");
     switch (SCS.Dimension) {
     case ICK_HLSL_Vector_Splat: {
       // Vector splat from any arithmetic type to a vector.
@@ -5234,6 +5233,17 @@ Sema::PerformImplicitConversion(Expr *From, QualType ToType,
 
       break;
     }
+    case ICK_HLSL_Matrix_Truncation: {
+      auto *FromMat = From->getType()->castAs<ConstantMatrixType>();
+      QualType TruncTy = FromMat->getElementType();
+      if (auto *ToMat = ToType->getAs<ConstantMatrixType>())
+        TruncTy = Context.getConstantMatrixType(TruncTy, ToMat->getNumRows(),
+                                                ToMat->getNumColumns());
+      From = ImpCastExprToType(From, TruncTy, CK_HLSLMatrixTruncation,
+                               From->getValueKind())
+                 .get();
+      break;
+    }
     case ICK_Identity:
     default:
       llvm_unreachable("Improper element standard conversion");
diff --git a/clang/lib/Sema/SemaHLSL.cpp b/clang/lib/Sema/SemaHLSL.cpp
index 89645e3b67db3..dadea331f930b 100644
--- a/clang/lib/Sema/SemaHLSL.cpp
+++ b/clang/lib/Sema/SemaHLSL.cpp
@@ -3823,7 +3823,6 @@ bool SemaHLSL::CanPerformAggregateSplatCast(Expr *Src, QualType DestTy) {
 }
 
 // Can we perform an HLSL Elementwise cast?
-// TODO: update this code when matrices are added; see issue #88060
 bool SemaHLSL::CanPerformElementwiseCast(Expr *Src, QualType DestTy) {
 
   // Don't handle casts where LHS and RHS are any combination of scalar/vector
@@ -3836,6 +3835,10 @@ bool SemaHLSL::CanPerformElementwiseCast(Expr *Src, QualType DestTy) {
       (DestTy->isScalarType() || DestTy->isVectorType()))
     return false;
 
+  if (SrcTy->isConstantMatrixType() &&
+      (DestTy->isScalarType() || DestTy->isConstantMatrixType()))
+    return false;
+
   llvm::SmallVector<QualType> DestTypes;
   BuildFlattenedTypeList(DestTy, DestTypes);
   llvm::SmallVector<QualType> SrcTypes;
@@ -4018,7 +4021,9 @@ void SemaHLSL::ActOnVariableDeclarator(VarDecl *VD) {
     // process explicit bindings
     processExplicitBindingsOnDecl(VD);
 
-    if (VD->getType()->isHLSLResourceRecordArray()) {
+    // Add implicit binding attribute to non-static resource arrays.
+    if (VD->getType()->isHLSLResourceRecordArray() &&
+        VD->getStorageClass() != SC_Static) {
       // If the resource array does not have an explicit binding attribute,
       // create an implicit one. It will be used to transfer implicit binding
       // order_ID to codegen.
@@ -4212,8 +4217,8 @@ bool SemaHLSL::ActOnUninitializedVarDecl(VarDecl *VD) {
   if (VD->getType().getAddressSpace() == LangAS::hlsl_constant)
     return true;
 
-  // Initialize resources at the global scope
-  if (VD->hasGlobalStorage()) {
+  // Initialize non-static resources at the global scope.
+  if (VD->hasGlobalStorage() && VD->getStorageClass() != SC_Static) {
     const Type *Ty = VD->getType().getTypePtr();
     if (Ty->isHLSLResourceRecord())
       return initGlobalResourceDecl(VD);
@@ -4237,10 +4242,10 @@ bool SemaHLSL::CheckResourceBinOp(BinaryOperatorKind Opc, Expr *LHSExpr,
   while (auto *ASE = dyn_cast<ArraySubscriptExpr>(E))
     E = ASE->getBase()->IgnoreParenImpCasts();
 
-  // Report error if LHS is a resource declared at a global scope.
+  // Report error if LHS is a non-static resource declared at a global scope.
   if (DeclRefExpr *DRE = dyn_cast<DeclRefExpr>(E->IgnoreParens())) {
     if (VarDecl *VD = dyn_cast<VarDecl>(DRE->getDecl())) {
-      if (VD->hasGlobalStorage()) {
+      if (VD->hasGlobalStorage() && VD->getStorageClass() != SC_Static) {
         // assignment to global resource is not allowed
         SemaRef.Diag(Loc, diag::err_hlsl_assign_to_global_resource) << VD;
         SemaRef.Diag(VD->getLocation(), diag::note_var_declared_here) << VD;
diff --git a/clang/lib/Sema/SemaOpenACC.cpp b/clang/lib/Sema/SemaOpenACC.cpp
index f0f3832e160cd..1115efbb8305c 100644
--- a/clang/lib/Sema/SemaOpenACC.cpp
+++ b/clang/lib/Sema/SemaOpenACC.cpp
@@ -12,6 +12,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "clang/Sema/SemaOpenACC.h"
+#include "clang/AST/ASTConsumer.h"
 #include "clang/AST/DeclOpenACC.h"
 #include "clang/AST/StmtOpenACC.h"
 #include "clang/Basic/DiagnosticSema.h"
@@ -2457,7 +2458,8 @@ OpenACCRoutineDecl *SemaOpenACC::CheckRoutineDecl(
     ArrayRef<const OpenACCClause *> Clauses, SourceLocation EndLoc) {
   assert(LParenLoc.isValid());
 
-  if (FunctionDecl *FD = getFunctionFromRoutineName(FuncRef)) {
+  FunctionDecl *FD = nullptr;
+  if ((FD = getFunctionFromRoutineName(FuncRef))) {
     // OpenACC 3.3 2.15:
     // In C and C++, function static variables are not supported in functions to
     // which a routine directive applies.
@@ -2509,11 +2511,9 @@ OpenACCRoutineDecl *SemaOpenACC::CheckRoutineDecl(
                                                         {DirLoc, BindLoc});
     FD->addAttr(RAA);
     // In case we are referencing not the 'latest' version, make sure we add
-    // the attribute to all declarations.
-    while (FD != FD->getMostRecentDecl()) {
-      FD = FD->getMostRecentDecl();
-      FD->addAttr(RAA);
-    }
+    // the attribute to all declarations after the 'found' one.
+    for (auto *CurFD : FD->redecls())
+      CurFD->addAttr(RAA->clone(getASTContext()));
   }
 
   LastRoutineDecl = OpenACCRoutineDecl::Create(
@@ -2522,9 +2522,20 @@ OpenACCRoutineDecl *SemaOpenACC::CheckRoutineDecl(
   LastRoutineDecl->setAccess(AS_public);
   getCurContext()->addDecl(LastRoutineDecl);
 
+  if (FD) {
+    // Add this attribute to the list of annotations so that codegen can visit
+    // it later. FD doesn't necessarily exist, but that case should be
+    // diagnosed.
+    RoutineRefList.emplace_back(FD, LastRoutineDecl);
+  }
   return LastRoutineDecl;
 }
 
+void SemaOpenACC::ActOnEndOfTranslationUnit(TranslationUnitDecl *TU) {
+  for (auto [FD, RoutineDecl] : RoutineRefList)
+    SemaRef.Consumer.HandleOpenACCRoutineReference(FD, RoutineDecl);
+}
+
 DeclGroupRef SemaOpenACC::ActOnEndRoutineDeclDirective(
     SourceLocation StartLoc, SourceLocation DirLoc, SourceLocation LParenLoc,
     Expr *ReferencedFunc, SourceLocation RParenLoc,
diff --git a/clang/lib/Sema/SemaOverload.cpp b/clang/lib/Sema/SemaOverload.cpp
index 897907e0f2b6a..9a1b19c7cff91 100644
--- a/clang/lib/Sema/SemaOverload.cpp
+++ b/clang/lib/Sema/SemaOverload.cpp
@@ -163,6 +163,7 @@ ImplicitConversionRank clang::GetConversionRank(ImplicitConversionKind Kind) {
       ICR_C_Conversion_Extension,
       ICR_Conversion,
       ICR_HLSL_Dimension_Reduction,
+      ICR_HLSL_Dimension_Reduction,
       ICR_Conversion,
       ICR_HLSL_Scalar_Widening,
   };
@@ -225,6 +226,7 @@ static const char *GetImplicitConversionName(ImplicitConversionKind Kind) {
       "Incompatible pointer conversion",
       "Fixed point conversion",
       "HLSL vector truncation",
+      "HLSL matrix truncation",
       "Non-decaying array conversion",
       "HLSL vector splat",
   };
@@ -2061,9 +2063,10 @@ static bool IsFloatingPointConversion(Sema &S, QualType FromType,
   return true;
 }
 
-static bool IsVectorElementConversion(Sema &S, QualType FromType,
-                                      QualType ToType,
-                                      ImplicitConversionKind &ICK, Expr *From) {
+static bool IsVectorOrMatrixElementConversion(Sema &S, QualType FromType,
+                                              QualType ToType,
+                                              ImplicitConversionKind &ICK,
+                                              Expr *From) {
   if (S.Context.hasSameUnqualifiedType(FromType, ToType))
     return true;
 
@@ -2103,6 +2106,57 @@ static bool IsVectorElementConversion(Sema &S, QualType FromType,
   return false;
 }
 
+/// Determine whether the conversion from FromType to ToType is a valid
+/// matrix conversion.
+///
+/// \param ICK Will be set to the matrix conversion kind, if this is a matrix
+/// conversion.
+static bool IsMatrixConversion(Sema &S, QualType FromType, QualType ToType,
+                               ImplicitConversionKind &ICK,
+                               ImplicitConversionKind &ElConv, Expr *From,
+                               bool InOverloadResolution, bool CStyle) {
+  // Implicit conversions for matrices are an HLSL feature not present in C/C++.
+  if (!S.getLangOpts().HLSL)
+    return false;
+
+  auto *ToMatrixType = ToType->getAs<ConstantMatrixType>();
+  auto *FromMatrixType = FromType->getAs<ConstantMatrixType>();
+
+  // If both arguments are matrix, handle possible matrix truncation and
+  // element conversion.
+  if (ToMatrixType && FromMatrixType) {
+    unsigned FromCols = FromMatrixType->getNumColumns();
+    unsigned ToCols = ToMatrixType->getNumColumns();
+    if (FromCols < ToCols)
+      return false;
+
+    unsigned FromRows = FromMatrixType->getNumRows();
+    unsigned ToRows = ToMatrixType->getNumRows();
+    if (FromRows < ToRows)
+      return false;
+
+    if (FromRows == ToRows && FromCols == ToCols)
+      ElConv = ICK_Identity;
+    else
+      ElConv = ICK_HLSL_Matrix_Truncation;
+
+    QualType FromElTy = FromMatrixType->getElementType();
+    QualType ToElTy = ToMatrixType->getElementType();
+    if (S.Context.hasSameUnqualifiedType(FromElTy, ToElTy))
+      return true;
+    return IsVectorOrMatrixElementConversion(S, FromElTy, ToElTy, ICK, From);
+  }
+  if (FromMatrixType && !ToMatrixType) {
+    ElConv = ICK_HLSL_Matrix_Truncation;
+    QualType FromElTy = FromMatrixType->getElementType();
+    if (S.Context.hasSameUnqualifiedType(FromElTy, ToType))
+      return true;
+    return IsVectorOrMatrixElementConversion(S, FromElTy, ToType, ICK, From);
+  }
+
+  return false;
+}
+
 /// Determine whether the conversion from FromType to ToType is a valid
 /// vector conversion.
 ///
@@ -2142,14 +2196,14 @@ static bool IsVectorConversion(Sema &S, QualType FromType, QualType ToType,
       QualType ToElTy = ToExtType->getElementType();
       if (S.Context.hasSameUnqualifiedType(FromElTy, ToElTy))
         return true;
-      return IsVectorElementConversion(S, FromElTy, ToElTy, ICK, From);
+      return IsVectorOrMatrixElementConversion(S, FromElTy, ToElTy, ICK, From);
     }
     if (FromExtType && !ToExtType) {
       ElConv = ICK_HLSL_Vector_Truncation;
       QualType FromElTy = FromExtType->getElementType();
       if (S.Context.hasSameUnqualifiedType(FromElTy, ToType))
         return true;
-      return IsVectorElementConversion(S, FromElTy, ToType, ICK, From);
+      return IsVectorOrMatrixElementConversion(S, FromElTy, ToType, ICK, From);
     }
     // Fallthrough for the case where ToType is a vector and FromType is not.
   }
@@ -2176,7 +2230,8 @@ static bool IsVectorConversion(Sema &S, QualType FromType, QualType ToType,
       if (S.getLangOpts().HLSL) {
         ElConv = ICK_HLSL_Vector_Splat;
         QualType ToElTy = ToExtType->getElementType();
-        return IsVectorElementConversion(S, FromType, ToElTy, ICK, From);
+        return IsVectorOrMatrixElementConversion(S, FromType, ToElTy, ICK,
+                                                 From);
       }
       ICK = ICK_Vector_Splat;
       return true;
@@ -2475,6 +2530,11 @@ static bool IsStandardConversion(Sema &S, Expr* From, QualType ToType,
     SCS.Second = SecondICK;
     SCS.Dimension = DimensionICK;
     FromType = ToType.getUnqualifiedType();
+  } else if (IsMatrixConversion(S, FromType, ToType, SecondICK, DimensionICK,
+                                From, InOverloadResolution, CStyle)) {
+    SCS.Second = SecondICK;
+    SCS.Dimension = DimensionICK;
+    FromType = ToType.getUnqualifiedType();
   } else if (!S.getLangOpts().CPlusPlus &&
              S.Context.typesAreCompatible(ToType, FromType)) {
     // Compatible conversions (Clang extension for C function overloading)
@@ -6253,6 +6313,7 @@ static bool CheckConvertedConstantConversions(Sema &S,
   case ICK_Incompatible_Pointer_Conversion:
   case ICK_Fixed_Point_Conversion:
   case ICK_HLSL_Vector_Truncation:
+  case ICK_HLSL_Matrix_Truncation:
     return false;
 
   case ICK_Lvalue_To_Rvalue:
diff --git a/clang/lib/Sema/SemaTemplate.cpp b/clang/lib/Sema/SemaTemplate.cpp
index ba91629783759..4a9e1bc93b918 100644
--- a/clang/lib/Sema/SemaTemplate.cpp
+++ b/clang/lib/Sema/SemaTemplate.cpp
@@ -3212,36 +3212,6 @@ void Sema::NoteAllFoundTemplates(TemplateName Name) {
   }
 }
 
-static QualType InstantiateTemplate(Sema &S, ElaboratedTypeKeyword Keyword,
-                                    TemplateName Template,
-                                    ArrayRef<TemplateArgument> Args,
-                                    SourceLocation Loc) {
-  TemplateArgumentListInfo ArgList;
-  for (auto Arg : Args) {
-    if (Arg.getKind() == TemplateArgument::Type) {
-      ArgList.addArgument(TemplateArgumentLoc(
-          Arg, S.Context.getTrivialTypeSourceInfo(Arg.getAsType())));
-    } else {
-      ArgList.addArgument(
-          S.getTrivialTemplateArgumentLoc(Arg, QualType(), Loc));
-    }
-  }
-
-  EnterExpressionEvaluationContext UnevaluatedContext(
-      S, Sema::ExpressionEvaluationContext::Unevaluated);
-  Sema::SFINAETrap SFINAE(S, /*AccessCheckingSFINAE=*/true);
-  Sema::ContextRAII TUContext(S, S.Context.getTranslationUnitDecl());
-
-  QualType Instantiation =
-      S.CheckTemplateIdType(Keyword, Template, Loc, ArgList, /*Scope=*/nullptr,
-                            /*ForNestedNameSpecifier=*/false);
-
-  if (SFINAE.hasErrorOccurred())
-    return QualType();
-
-  return Instantiation;
-}
-
 static QualType builtinCommonTypeImpl(Sema &S, ElaboratedTypeKeyword Keyword,
                                       TemplateName BaseTemplate,
                                       SourceLocation TemplateLoc,
@@ -3254,7 +3224,25 @@ static QualType builtinCommonTypeImpl(Sema &S, ElaboratedTypeKeyword Keyword,
       return builtinCommonTypeImpl(S, Keyword, BaseTemplate, TemplateLoc,
                                    {T1, T2});
 
-    return InstantiateTemplate(S, Keyword, BaseTemplate, {T1, T2}, TemplateLoc);
+    TemplateArgumentListInfo Args;
+    Args.addArgument(TemplateArgumentLoc(
+        T1, S.Context.getTrivialTypeSourceInfo(T1.getAsType())));
+    Args.addArgument(TemplateArgumentLoc(
+        T2, S.Context.getTrivialTypeSourceInfo(T2.getAsType())));
+
+    EnterExpressionEvaluationContext UnevaluatedContext(
+        S, Sema::ExpressionEvaluationContext::Unevaluated);
+    Sema::SFINAETrap SFINAE(S, /*ForValidityCheck=*/true);
+    Sema::ContextRAII TUContext(S, S.Context.getTranslationUnitDecl());
+
+    QualType BaseTemplateInst = S.CheckTemplateIdType(
+        Keyword, BaseTemplate, TemplateLoc, Args,
+        /*Scope=*/nullptr, /*ForNestedNameSpecifier=*/false);
+
+    if (SFINAE.hasErrorOccurred())
+      return QualType();
+
+    return BaseTemplateInst;
   };
 
   // Note A: For the common_type trait applied to a template parameter pack T of
@@ -3361,233 +3349,6 @@ static QualType builtinCommonTypeImpl(Sema &S, ElaboratedTypeKeyword Keyword,
   }
 }
 
-static QualType CopyCV(QualType From, QualType To) {
-  if (From.isConstQualified())
-    To.addConst();
-  if (From.isVolatileQualified())
-    To.addVolatile();
-  return To;
-}
-
-// Let COND-RES(X, Y) be
-//  decltype(false ? declval<X(&)()>()() : declval<Y(&)()>()())
-static QualType CondRes(Sema &S, QualType X, QualType Y, SourceLocation Loc) {
-  EnterExpressionEvaluationContext UnevaluatedContext(
-      S, Sema::ExpressionEvaluationContext::Unevaluated);
-  Sema::SFINAETrap SFINAE(S, /*AccessCheckingSFINAE=*/true);
-  Sema::ContextRAII TUContext(S, S.Context.getTranslationUnitDecl());
-
-  // false
-  OpaqueValueExpr CondExpr(SourceLocation(), S.Context.BoolTy, VK_PRValue);
-  ExprResult Cond = &CondExpr;
-
-  // declval<X(&)()>()()
-  OpaqueValueExpr LHSExpr(Loc, X.getNonLValueExprType(S.Context),
-                          Expr::getValueKindForType(X));
-  ExprResult LHS = &LHSExpr;
-
-  // declval<Y(&)()>()()
-  OpaqueValueExpr RHSExpr(Loc, Y.getNonLValueExprType(S.Context),
-                          Expr::getValueKindForType(Y));
-  ExprResult RHS = &RHSExpr;
-
-  ExprValueKind VK = VK_PRValue;
-  ExprObjectKind OK = OK_Ordinary;
-
-  // decltype(false ? declval<X(&)()>()() : declval<Y(&)()>()())
-  QualType Result = S.CheckConditionalOperands(Cond, LHS, RHS, VK, OK, Loc);
-
-  if (SFINAE.hasErrorOccurred())
-    return QualType();
-  if (VK == VK_LValue)
-    return S.BuiltinAddLValueReference(Result, Loc);
-  if (VK == VK_XValue)
-    return S.BuiltinAddRValueReference(Result, Loc);
-  return Result;
-}
-
-static QualType CommonRef(Sema &S, QualType A, QualType B, SourceLocation Loc) {
-  // Given types A and B, let X be remove_reference_t<A>, let Y be
-  // remove_reference_t<B>, and let COMMON-​REF(A, B) be:
-  assert(A->isReferenceType() && B->isReferenceType() &&
-         "A and B have to be ref qualified for a COMMON-REF");
-  auto X = A.getNonReferenceType();
-  auto Y = B.getNonReferenceType();
-
-  // If A and B are both lvalue reference types, COMMON-REF(A, B) is
-  // COND-RES(COPYCV(X, Y) &, COPYCV(​Y, X) &) if that type exists and is a
-  // reference type.
-  if (A->isLValueReferenceType() && B->isLValueReferenceType()) {
-    auto CR = CondRes(S, S.BuiltinAddLValueReference(CopyCV(X, Y), Loc),
-                      S.BuiltinAddLValueReference(CopyCV(Y, X), Loc), Loc);
-    if (CR.isNull() || !CR->isReferenceType())
-      return QualType();
-    return CR;
-  }
-
-  // Otherwise, let C be remove_reference_t<COMMON-REF(X&, Y&)>&&. If A and B
-  // are both rvalue reference types, C is well-formed, and
-  // is_convertible_v<A, C> && is_convertible_v<B, C> is true, then
-  // COMMON-REF(A, B) is C.
-  if (A->isRValueReferenceType() && B->isRValueReferenceType()) {
-    auto C = CommonRef(S, S.BuiltinAddLValueReference(X, Loc),
-                       S.BuiltinAddLValueReference(Y, Loc), Loc);
-    if (C.isNull())
-      return QualType();
-
-    C = C.getNonReferenceType();
-
-    if (S.BuiltinIsConvertible(A, C, Loc) && S.BuiltinIsConvertible(B, C, Loc))
-      return S.BuiltinAddRValueReference(C, Loc);
-    return QualType();
-  }
-
-  // Otherwise, if A is an lvalue reference and B is an rvalue reference, then
-  // COMMON-REF(A, B) is COMMON-REF(B, A).
-  if (A->isLValueReferenceType() && B->isRValueReferenceType())
-    std::swap(A, B);
-
-  // Otherwise, let D be COMMON-REF(const X&, Y&). If A is an rvalue reference
-  // and B is an lvalue reference and D is well-formed and
-  // is_convertible_v<A, D> is true, then COMMON-REF(A, B) is D.
-  if (A->isRValueReferenceType() && B->isLValueReferenceType()) {
-    auto X2 = X;
-    X2.addConst();
-    auto D = CommonRef(S, S.BuiltinAddLValueReference(X2, Loc),
-                       S.BuiltinAddLValueReference(Y, Loc), Loc);
-    if (!D.isNull() && S.BuiltinIsConvertible(A, D, Loc))
-      return D;
-    return QualType();
-  }
-
-  // Otherwise, COMMON-REF(A, B) is ill-formed.
-  // This is implemented by returning from the individual branches above.
-
-  llvm_unreachable("The above cases should be exhaustive");
-}
-
-static QualType builtinCommonReferenceImpl(Sema &S,
-                                           ElaboratedTypeKeyword Keyword,
-                                           TemplateName CommonReference,
-                                           TemplateName CommonType,
-                                           SourceLocation TemplateLoc,
-                                           ArrayRef<TemplateArgument> Ts) {
-  switch (Ts.size()) {
-  // If sizeof...(T) is zero, there shall be no member type.
-  case 0:
-    return QualType();
-
-  // Otherwise, if sizeof...(T) is one, let T0 denote the sole type in the
-  // pack T. The member typedef type shall denote the same type as T0.
-  case 1:
-    return Ts[0].getAsType();
-
-  // Otherwise, if sizeof...(T) is two, let T1 and T2 denote the two types in
-  // the pack T. Then
-  case 2: {
-    auto T1 = Ts[0].getAsType();
-    auto T2 = Ts[1].getAsType();
-
-    // Let R be COMMON-REF(T1, T2). If T1 and T2 are reference types, R is
-    // well-formed, and is_convertible_v<add_pointer_t<T1>, add_pointer_t<R>> &&
-    // is_convertible_v<add_pointer_t<T2>, add_pointer_t<R>> is true, then the
-    // member typedef type denotes R.
-    if (T1->isReferenceType() && T2->isReferenceType()) {
-      QualType R = CommonRef(S, T1, T2, TemplateLoc);
-      if (!R.isNull()) {
-        if (S.BuiltinIsConvertible(S.BuiltinAddPointer(T1, TemplateLoc),
-                                   S.BuiltinAddPointer(R, TemplateLoc),
-                                   TemplateLoc) &&
-            S.BuiltinIsConvertible(S.BuiltinAddPointer(T2, TemplateLoc),
-                                   S.BuiltinAddPointer(R, TemplateLoc),
-                                   TemplateLoc)) {
-          return R;
-        }
-      }
-    }
-
-    // Otherwise, if basic_common_reference<remove_cvref_t<T1>,
-    // remove_cvref_t<T2>, ​XREF(​T1), XREF(T2)>​::​type is well-formed,
-    // then the member typedef type denotes that type.
-    {
-      auto getXRef = [&](QualType T) {
-        BuiltinTemplateDecl *Quals[12] = {
-            S.Context.get__clang_internal_xref_Decl(),
-            S.Context.get__clang_internal_xref_constDecl(),
-            S.Context.get__clang_internal_xref_volatileDecl(),
-            S.Context.get__clang_internal_xref_constvolatileDecl(),
-            S.Context.get__clang_internal_xref_lvalueDecl(),
-            S.Context.get__clang_internal_xref_lvalueconstDecl(),
-            S.Context.get__clang_internal_xref_lvaluevolatileDecl(),
-            S.Context.get__clang_internal_xref_lvalueconstvolatileDecl(),
-            S.Context.get__clang_internal_xref_rvalueDecl(),
-            S.Context.get__clang_internal_xref_rvalueconstDecl(),
-            S.Context.get__clang_internal_xref_rvaluevolatileDecl(),
-            S.Context.get__clang_internal_xref_rvalueconstvolatileDecl(),
-        };
-        size_t Index = 0;
-        if (T->isLValueReferenceType()) {
-          T = T.getNonReferenceType();
-          Index += 4;
-        } else if (T->isRValueReferenceType()) {
-          T = T.getNonReferenceType();
-          Index += 8;
-        }
-        if (T.isConstQualified())
-          Index += 1;
-
-        if (T.isVolatileQualified())
-          Index += 2;
-
-        return Quals[Index];
-      };
-
-      auto BCR = InstantiateTemplate(S, Keyword, CommonReference,
-                                     {S.BuiltinRemoveCVRef(T1, TemplateLoc),
-                                      S.BuiltinRemoveCVRef(T2, TemplateLoc),
-                                      TemplateName{getXRef(T1)},
-                                      TemplateName{getXRef(T2)}},
-                                     TemplateLoc);
-      if (!BCR.isNull())
-        return BCR;
-    }
-
-    // Otherwise, if COND-RES(T1, T2) is well-formed, then the member typedef
-    // type denotes that type.
-    if (auto CR = CondRes(S, T1, T2, TemplateLoc); !CR.isNull())
-      return CR;
-
-    // Otherwise, if common_type_t<T1, T2> is well-formed, then the member
-    // typedef type denotes that type.
-    if (auto CT =
-            InstantiateTemplate(S, Keyword, CommonType, {T1, T2}, TemplateLoc);
-        !CT.isNull())
-      return CT;
-
-    // Otherwise, there shall be no member type.
-    return QualType();
-  }
-
-  // Otherwise, if sizeof...(T) is greater than two, let T1, T2, and Rest,
-  // respectively, denote the first, second, and (pack of) remaining types
-  // comprising T. Let C be the type common_reference_t<T1, T2>. Then:
-  default: {
-    auto T1 = Ts[0];
-    auto T2 = Ts[1];
-    auto Rest = Ts.drop_front(2);
-    auto C = builtinCommonReferenceImpl(S, Keyword, CommonReference, CommonType,
-                                        TemplateLoc, {T1, T2});
-    if (C.isNull())
-      return QualType();
-    llvm::SmallVector<TemplateArgument, 4> Args;
-    Args.emplace_back(C);
-    Args.append(Rest.begin(), Rest.end());
-    return builtinCommonReferenceImpl(S, Keyword, CommonReference, CommonType,
-                                      TemplateLoc, Args);
-  }
-  }
-}
-
 static bool isInVkNamespace(const RecordType *RT) {
   DeclContext *DC = RT->getDecl()->getDeclContext();
   if (!DC)
@@ -3746,89 +3507,6 @@ static QualType checkBuiltinTemplateIdType(
     return HasNoTypeMember;
   }
 
-  case BTK__builtin_common_reference: {
-    assert(Converted.size() == 5);
-    if (llvm::any_of(Converted, [](auto &C) { return C.isDependent(); }))
-      return QualType();
-
-    TemplateName BasicCommonReference = Converted[0].getAsTemplate();
-    TemplateName CommonType = Converted[1].getAsTemplate();
-    TemplateName HasTypeMember = Converted[2].getAsTemplate();
-    QualType HasNoTypeMember = Converted[3].getAsType();
-    ArrayRef<TemplateArgument> Ts = Converted[4].getPackAsArray();
-    if (auto CR =
-            builtinCommonReferenceImpl(SemaRef, Keyword, BasicCommonReference,
-                                       CommonType, TemplateLoc, Ts);
-        !CR.isNull()) {
-      TemplateArgumentListInfo TAs;
-      TAs.addArgument(TemplateArgumentLoc(
-          TemplateArgument(CR), SemaRef.Context.getTrivialTypeSourceInfo(
-                                    CR, TemplateArgs[1].getLocation())));
-      return SemaRef.CheckTemplateIdType(Keyword, HasTypeMember, TemplateLoc,
-                                         TAs, /*Scope=*/nullptr,
-                                         /*ForNestedNameSpecifier=*/false);
-    }
-    return HasNoTypeMember;
-  }
-
-  case BTK__clang_internal_xref_:
-  case BTK__clang_internal_xref_const:
-  case BTK__clang_internal_xref_volatile:
-  case BTK__clang_internal_xref_constvolatile:
-  case BTK__clang_internal_xref_lvalue:
-  case BTK__clang_internal_xref_lvalueconst:
-  case BTK__clang_internal_xref_lvaluevolatile:
-  case BTK__clang_internal_xref_lvalueconstvolatile:
-  case BTK__clang_internal_xref_rvalue:
-  case BTK__clang_internal_xref_rvalueconst:
-  case BTK__clang_internal_xref_rvaluevolatile:
-  case BTK__clang_internal_xref_rvalueconstvolatile: {
-    if (llvm::any_of(Converted, [](auto &C) { return C.isDependent(); }))
-      return QualType();
-
-    auto BTK = BTD->getBuiltinTemplateKind();
-    auto anyOf = [&](auto... Vals) { return ((BTK == Vals) || ...); };
-
-    bool AddCV = anyOf(BTK__clang_internal_xref_constvolatile,
-                       BTK__clang_internal_xref_lvalueconstvolatile,
-                       BTK__clang_internal_xref_rvalueconstvolatile);
-
-    bool AddConst = AddCV || anyOf(BTK__clang_internal_xref_const,
-                                   BTK__clang_internal_xref_lvalueconst,
-                                   BTK__clang_internal_xref_rvalueconst);
-
-    bool AddVolatile = AddCV || anyOf(BTK__clang_internal_xref_volatile,
-                                      BTK__clang_internal_xref_lvaluevolatile,
-                                      BTK__clang_internal_xref_rvaluevolatile);
-
-    bool AddLValue = anyOf(BTK__clang_internal_xref_lvalue,
-                           BTK__clang_internal_xref_lvalueconst,
-                           BTK__clang_internal_xref_lvaluevolatile,
-                           BTK__clang_internal_xref_lvalueconstvolatile);
-
-    bool AddRValue = anyOf(BTK__clang_internal_xref_rvalue,
-                           BTK__clang_internal_xref_rvalueconst,
-                           BTK__clang_internal_xref_rvaluevolatile,
-                           BTK__clang_internal_xref_rvalueconstvolatile);
-
-    assert(Converted.size() == 1);
-
-    QualType T = Converted[0].getAsType();
-
-    if (AddConst)
-      T.addConst();
-
-    if (AddVolatile)
-      T.addVolatile();
-
-    if (AddLValue)
-      T = SemaRef.BuiltinAddLValueReference(T, TemplateLoc);
-    else if (AddRValue)
-      T = SemaRef.BuiltinAddRValueReference(T, TemplateLoc);
-
-    return T;
-  }
-
   case BTK__hlsl_spirv_type: {
     assert(Converted.size() == 4);
 
diff --git a/clang/lib/Sema/SemaType.cpp b/clang/lib/Sema/SemaType.cpp
index 9488a853ffab1..9f5aa153d1cbe 100644
--- a/clang/lib/Sema/SemaType.cpp
+++ b/clang/lib/Sema/SemaType.cpp
@@ -32,8 +32,6 @@
 #include "clang/Lex/Preprocessor.h"
 #include "clang/Sema/DeclSpec.h"
 #include "clang/Sema/DelayedDiagnostic.h"
-#include "clang/Sema/EnterExpressionEvaluationContext.h"
-#include "clang/Sema/Initialization.h"
 #include "clang/Sema/Lookup.h"
 #include "clang/Sema/ParsedAttr.h"
 #include "clang/Sema/ParsedTemplate.h"
@@ -10074,81 +10072,6 @@ QualType Sema::BuiltinChangeSignedness(QualType BaseType, UTTKind UKind,
   return Context.getQualifiedType(Underlying, BaseType.getQualifiers());
 }
 
-bool Sema::BuiltinIsConvertible(QualType From, QualType To, SourceLocation Loc,
-                                bool CheckNothrow) {
-  if (To->isVoidType())
-    return From->isVoidType();
-
-  // [meta.rel]
-  // From and To shall be complete types, cv void, or arrays of unknown bound.
-  if ((!From->isIncompleteArrayType() && !From->isVoidType() &&
-       RequireCompleteType(
-           Loc, From, diag::err_incomplete_type_used_in_type_trait_expr)) ||
-      (!To->isIncompleteArrayType() && !To->isVoidType() &&
-       RequireCompleteType(Loc, To,
-                           diag::err_incomplete_type_used_in_type_trait_expr)))
-    return false;
-
-  // C++11 [meta.rel]p4:
-  //   Given the following function prototype:
-  //
-  //     template <class T>
-  //       typename add_rvalue_reference<T>::type create();
-  //
-  //   the predicate condition for a template specialization
-  //   is_convertible<From, To> shall be satisfied if and only if
-  //   the return expression in the following code would be
-  //   well-formed, including any implicit conversions to the return
-  //   type of the function:
-  //
-  //     To test() {
-  //       return create<From>();
-  //     }
-  //
-  //   Access checking is performed as if in a context unrelated to To and
-  //   From. Only the validity of the immediate context of the expression
-  //   of the return-statement (including conversions to the return type)
-  //   is considered.
-  //
-  // We model the initialization as a copy-initialization of a temporary
-  // of the appropriate type, which for this expression is identical to the
-  // return statement (since NRVO doesn't apply).
-
-  // Functions aren't allowed to return function or array types.
-  if (To->isFunctionType() || To->isArrayType())
-    return false;
-
-  // A function definition requires a non-abstract return type.
-  if (isAbstractType(Loc, To))
-    return false;
-
-  From = BuiltinAddRValueReference(From, Loc);
-
-  // Build a fake source and destination for initialization.
-  InitializedEntity ToEntity(InitializedEntity::InitializeTemporary(To));
-  OpaqueValueExpr FromExpr(Loc, From.getNonLValueExprType(Context),
-                           Expr::getValueKindForType(From));
-  InitializationKind Kind =
-      InitializationKind::CreateCopy(Loc, SourceLocation());
-
-  // Perform the initialization in an unevaluated context within a SFINAE
-  // trap at translation unit scope.
-  EnterExpressionEvaluationContext Unevaluated(
-      *this, Sema::ExpressionEvaluationContext::Unevaluated);
-  Sema::SFINAETrap SFINAE(*this, /*AccessCheckingSFINAE=*/true);
-  Sema::ContextRAII TUContext(*this, Context.getTranslationUnitDecl());
-  Expr *FromExprPtr = &FromExpr;
-  InitializationSequence Init(*this, ToEntity, Kind, FromExprPtr);
-  if (Init.Failed())
-    return false;
-
-  ExprResult Result = Init.Perform(*this, ToEntity, Kind, FromExprPtr);
-  if (Result.isInvalid() || SFINAE.hasErrorOccurred())
-    return false;
-
-  return !CheckNothrow || canThrow(Result.get()) == CT_Cannot;
-}
-
 QualType Sema::BuildUnaryTransformType(QualType BaseType, UTTKind UKind,
                                        SourceLocation Loc) {
   if (BaseType->isDependentType())
diff --git a/clang/lib/Sema/SemaTypeTraits.cpp b/clang/lib/Sema/SemaTypeTraits.cpp
index 3802da8bc2d45..38877967af05e 100644
--- a/clang/lib/Sema/SemaTypeTraits.cpp
+++ b/clang/lib/Sema/SemaTypeTraits.cpp
@@ -1212,6 +1212,76 @@ static bool EvaluateBinaryTypeTrait(Sema &Self, TypeTrait BTT,
                                     const TypeSourceInfo *Rhs,
                                     SourceLocation KeyLoc);
 
+static ExprResult CheckConvertibilityForTypeTraits(
+    Sema &Self, const TypeSourceInfo *Lhs, const TypeSourceInfo *Rhs,
+    SourceLocation KeyLoc, llvm::BumpPtrAllocator &OpaqueExprAllocator) {
+
+  QualType LhsT = Lhs->getType();
+  QualType RhsT = Rhs->getType();
+
+  // C++0x [meta.rel]p4:
+  //   Given the following function prototype:
+  //
+  //     template <class T>
+  //       typename add_rvalue_reference<T>::type create();
+  //
+  //   the predicate condition for a template specialization
+  //   is_convertible<From, To> shall be satisfied if and only if
+  //   the return expression in the following code would be
+  //   well-formed, including any implicit conversions to the return
+  //   type of the function:
+  //
+  //     To test() {
+  //       return create<From>();
+  //     }
+  //
+  //   Access checking is performed as if in a context unrelated to To and
+  //   From. Only the validity of the immediate context of the expression
+  //   of the return-statement (including conversions to the return type)
+  //   is considered.
+  //
+  // We model the initialization as a copy-initialization of a temporary
+  // of the appropriate type, which for this expression is identical to the
+  // return statement (since NRVO doesn't apply).
+
+  // Functions aren't allowed to return function or array types.
+  if (RhsT->isFunctionType() || RhsT->isArrayType())
+    return ExprError();
+
+  // A function definition requires a complete, non-abstract return type.
+  if (!Self.isCompleteType(Rhs->getTypeLoc().getBeginLoc(), RhsT) ||
+      Self.isAbstractType(Rhs->getTypeLoc().getBeginLoc(), RhsT))
+    return ExprError();
+
+  // Compute the result of add_rvalue_reference.
+  if (LhsT->isObjectType() || LhsT->isFunctionType())
+    LhsT = Self.Context.getRValueReferenceType(LhsT);
+
+  // Build a fake source and destination for initialization.
+  InitializedEntity To(InitializedEntity::InitializeTemporary(RhsT));
+  Expr *From = new (OpaqueExprAllocator.Allocate<OpaqueValueExpr>())
+      OpaqueValueExpr(KeyLoc, LhsT.getNonLValueExprType(Self.Context),
+                      Expr::getValueKindForType(LhsT));
+  InitializationKind Kind =
+      InitializationKind::CreateCopy(KeyLoc, SourceLocation());
+
+  // Perform the initialization in an unevaluated context within a SFINAE
+  // trap at translation unit scope.
+  EnterExpressionEvaluationContext Unevaluated(
+      Self, Sema::ExpressionEvaluationContext::Unevaluated);
+  Sema::SFINAETrap SFINAE(Self, /*ForValidityCheck=*/true);
+  Sema::ContextRAII TUContext(Self, Self.Context.getTranslationUnitDecl());
+  InitializationSequence Init(Self, To, Kind, From);
+  if (Init.Failed())
+    return ExprError();
+
+  ExprResult Result = Init.Perform(Self, To, Kind, From);
+  if (Result.isInvalid() || SFINAE.hasErrorOccurred())
+    return ExprError();
+
+  return Result;
+}
+
 static APValue EvaluateSizeTTypeTrait(Sema &S, TypeTrait Kind,
                                       SourceLocation KWLoc,
                                       ArrayRef<TypeSourceInfo *> Args,
@@ -1372,8 +1442,9 @@ static bool EvaluateBooleanTypeTrait(Sema &S, TypeTrait Kind,
           S.Context.getPointerType(T.getNonReferenceType()));
       TypeSourceInfo *UPtr = S.Context.CreateTypeSourceInfo(
           S.Context.getPointerType(U.getNonReferenceType()));
-      return S.BuiltinIsConvertible(UPtr->getType(), TPtr->getType(),
-                                    RParenLoc);
+      return !CheckConvertibilityForTypeTraits(S, UPtr, TPtr, RParenLoc,
+                                               OpaqueExprAllocator)
+                  .isInvalid();
     }
 
     if (Kind == clang::TT_IsNothrowConstructible)
@@ -1624,9 +1695,20 @@ static bool EvaluateBinaryTypeTrait(Sema &Self, TypeTrait BTT,
   }
   case BTT_IsConvertible:
   case BTT_IsConvertibleTo:
-  case BTT_IsNothrowConvertible:
-    return Self.BuiltinIsConvertible(LhsT, RhsT, KeyLoc,
-                                     BTT == BTT_IsNothrowConvertible);
+  case BTT_IsNothrowConvertible: {
+    if (RhsT->isVoidType())
+      return LhsT->isVoidType();
+    llvm::BumpPtrAllocator OpaqueExprAllocator;
+    ExprResult Result = CheckConvertibilityForTypeTraits(Self, Lhs, Rhs, KeyLoc,
+                                                         OpaqueExprAllocator);
+    if (Result.isInvalid())
+      return false;
+
+    if (BTT != BTT_IsNothrowConvertible)
+      return true;
+
+    return Self.canThrow(Result.get()) == CT_Cannot;
+  }
 
   case BTT_IsAssignable:
   case BTT_IsNothrowAssignable:
diff --git a/clang/lib/StaticAnalyzer/Core/BugReporter.cpp b/clang/lib/StaticAnalyzer/Core/BugReporter.cpp
index 5fe64dc5e9270..4c066520b668f 100644
--- a/clang/lib/StaticAnalyzer/Core/BugReporter.cpp
+++ b/clang/lib/StaticAnalyzer/Core/BugReporter.cpp
@@ -25,7 +25,6 @@
 #include "clang/AST/StmtObjC.h"
 #include "clang/Analysis/AnalysisDeclContext.h"
 #include "clang/Analysis/CFG.h"
-#include "clang/Analysis/CFGStmtMap.h"
 #include "clang/Analysis/PathDiagnostic.h"
 #include "clang/Analysis/ProgramPoint.h"
 #include "clang/Basic/LLVM.h"
diff --git a/clang/lib/StaticAnalyzer/Core/ExprEngineC.cpp b/clang/lib/StaticAnalyzer/Core/ExprEngineC.cpp
index 4ddf8fd5b4b0f..db27c06cd18a3 100644
--- a/clang/lib/StaticAnalyzer/Core/ExprEngineC.cpp
+++ b/clang/lib/StaticAnalyzer/Core/ExprEngineC.cpp
@@ -560,6 +560,7 @@ void ExprEngine::VisitCast(const CastExpr *CastE, const Expr *Ex,
       case CK_VectorSplat:
       case CK_HLSLElementwiseCast:
       case CK_HLSLAggregateSplatCast:
+      case CK_HLSLMatrixTruncation:
       case CK_HLSLVectorTruncation: {
         QualType resultType = CastE->getType();
         if (CastE->isGLValue())
diff --git a/clang/test/CIR/CodeGen/cxx-special-member-attr.cpp b/clang/test/CIR/CodeGen/cxx-special-member-attr.cpp
index 815ef2c2aaa25..f2c2c1f683395 100644
--- a/clang/test/CIR/CodeGen/cxx-special-member-attr.cpp
+++ b/clang/test/CIR/CodeGen/cxx-special-member-attr.cpp
@@ -3,25 +3,15 @@
 
 struct Flub {
   int a = 123;
-  // CIR: @_ZN4FlubC1ERKS_(%arg0: !cir.ptr<!rec_Flub> loc({{.*}}), %arg1: !cir.ptr<!rec_Flub> loc({{.*}})) special_member<#cir.cxx_ctor<!rec_Flub, copy, trivial true>>
-  // CIR: @_ZN4FlubC2EOS_(%arg0: !cir.ptr<!rec_Flub> loc({{.*}}), %arg1: !cir.ptr<!rec_Flub> loc({{.*}})) special_member<#cir.cxx_ctor<!rec_Flub, move, trivial true>
-  // CIR: @_ZN4FlubaSERKS_(%arg0: !cir.ptr<!rec_Flub> loc({{.*}}), %arg1: !cir.ptr<!rec_Flub> loc({{.*}})) -> !cir.ptr<!rec_Flub> special_member<#cir.cxx_assign<!rec_Flub, copy, trivial true>>
-  // CIR: @_ZN4FlubaSEOS_(%arg0: !cir.ptr<!rec_Flub> loc({{.*}}), %arg1: !cir.ptr<!rec_Flub> loc({{.*}})) -> !cir.ptr<!rec_Flub> special_member<#cir.cxx_assign<!rec_Flub, move, trivial true>>
 };
 
 struct Foo {
   int a;
 
-  // CIR: @_ZN3FooC2Ev(%arg0: !cir.ptr<!rec_Foo> loc({{.*}})) special_member<#cir.cxx_ctor<!rec_Foo, default>>
   Foo() : a(123) {}
-
-  // CIR: @_ZN3FooC2ERKS_(%arg0: !cir.ptr<!rec_Foo> loc({{.*}}), %arg1: !cir.ptr<!rec_Foo> loc({{.*}})) special_member<#cir.cxx_ctor<!rec_Foo, copy>>
   Foo(const Foo &other) : a(other.a) {}
-
-  // CIR: @_ZN3FooC2EOS_(%arg0: !cir.ptr<!rec_Foo> loc({{.*}}), %arg1: !cir.ptr<!rec_Foo> loc({{.*}})) special_member<#cir.cxx_ctor<!rec_Foo, move>>
   Foo(Foo &&other) noexcept : a(other.a) { other.a = 0; }
 
-  // CIR: @_ZN3FooaSERKS_(%arg0: !cir.ptr<!rec_Foo> loc({{.*}}), %arg1: !cir.ptr<!rec_Foo> loc({{.*}})) -> !cir.ptr<!rec_Foo> special_member<#cir.cxx_assign<!rec_Foo, copy>>
   Foo &operator=(const Foo &other) {
     if (this != &other) {
       a = other.a;
@@ -29,7 +19,6 @@ struct Foo {
     return *this;
   }
 
-  // CIR: @_ZN3FooaSEOS_(%arg0: !cir.ptr<!rec_Foo> loc({{.*}}), %arg1: !cir.ptr<!rec_Foo> loc({{.*}})) -> !cir.ptr<!rec_Foo> special_member<#cir.cxx_assign<!rec_Foo, move>>
   Foo &operator=(Foo &&other) noexcept {
     if (this != &other) {
       a = other.a;
@@ -38,22 +27,40 @@ struct Foo {
     return *this;
   }
 
-  // CIR: @_ZN3FooD1Ev(!cir.ptr<!rec_Foo>) special_member<#cir.cxx_dtor<!rec_Foo>>
   ~Foo();
 };
 
-void trivial() {
+void trivial_func() {
   Flub f1{};
+
   Flub f2 = f1;
+  // Trivial copy constructors/assignments are replaced with cir.copy
+  // CIR: cir.copy {{.*}} : !cir.ptr<!rec_Flub>
+
   Flub f3 = static_cast<Flub&&>(f1);
+  // CIR: @_ZN4FlubC1EOS_(%arg0: !cir.ptr<!rec_Flub> loc({{.*}}), %arg1: !cir.ptr<!rec_Flub> loc({{.*}})) special_member<#cir.cxx_ctor<!rec_Flub, move, trivial true>
+
   f2 = f1;
+  // CIR: @_ZN4FlubaSERKS_(%arg0: !cir.ptr<!rec_Flub> loc({{.*}}), %arg1: !cir.ptr<!rec_Flub> loc({{.*}})) -> !cir.ptr<!rec_Flub> special_member<#cir.cxx_assign<!rec_Flub, copy, trivial true>>
+
   f1 = static_cast<Flub&&>(f3);
+  // CIR: @_ZN4FlubaSEOS_(%arg0: !cir.ptr<!rec_Flub> loc({{.*}}), %arg1: !cir.ptr<!rec_Flub> loc({{.*}})) -> !cir.ptr<!rec_Flub> special_member<#cir.cxx_assign<!rec_Flub, move, trivial true>>
 }
 
-void non_trivial() {
+void non_trivial_func() {
   Foo f1{};
+  // CIR: @_ZN3FooC2Ev(%arg0: !cir.ptr<!rec_Foo> loc({{.*}})) special_member<#cir.cxx_ctor<!rec_Foo, default>>
+
   Foo f2 = f1;
+  // CIR: @_ZN3FooC2ERKS_(%arg0: !cir.ptr<!rec_Foo> loc({{.*}}), %arg1: !cir.ptr<!rec_Foo> loc({{.*}})) special_member<#cir.cxx_ctor<!rec_Foo, copy>>
+
   Foo f3 = static_cast<Foo&&>(f1);
+  // CIR: @_ZN3FooC2EOS_(%arg0: !cir.ptr<!rec_Foo> loc({{.*}}), %arg1: !cir.ptr<!rec_Foo> loc({{.*}})) special_member<#cir.cxx_ctor<!rec_Foo, move>>
+
   f2 = f1;
+  // CIR: @_ZN3FooaSERKS_(%arg0: !cir.ptr<!rec_Foo> loc({{.*}}), %arg1: !cir.ptr<!rec_Foo> loc({{.*}})) -> !cir.ptr<!rec_Foo> special_member<#cir.cxx_assign<!rec_Foo, copy>>
+
   f1 = static_cast<Foo&&>(f3);
+  // CIR: @_ZN3FooaSEOS_(%arg0: !cir.ptr<!rec_Foo> loc({{.*}}), %arg1: !cir.ptr<!rec_Foo> loc({{.*}})) -> !cir.ptr<!rec_Foo> special_member<#cir.cxx_assign<!rec_Foo, move>>
+  // CIR: @_ZN3FooD1Ev(!cir.ptr<!rec_Foo>) special_member<#cir.cxx_dtor<!rec_Foo>>
 }
diff --git a/clang/test/CIR/CodeGen/lambda.cpp b/clang/test/CIR/CodeGen/lambda.cpp
index 10cab66f1d306..3c2b0969fa7a2 100644
--- a/clang/test/CIR/CodeGen/lambda.cpp
+++ b/clang/test/CIR/CodeGen/lambda.cpp
@@ -367,7 +367,7 @@ struct A {
 // CIR:   cir.scope {
 // CIR:     %[[LAM_ADDR:.*]] = cir.alloca ![[REC_LAM_A]], !cir.ptr<![[REC_LAM_A]]>, ["ref.tmp0"]
 // CIR:     %[[STRUCT_A:.*]] = cir.get_member %[[LAM_ADDR]][0] {name = "this"} : !cir.ptr<![[REC_LAM_A]]> -> !cir.ptr<!rec_A>
-// CIR:     cir.call @_ZN1AC1ERKS_(%[[STRUCT_A]], %[[THIS]]){{.*}} : (!cir.ptr<!rec_A>, !cir.ptr<!rec_A>){{.*}} -> ()
+// CIR:     cir.copy %[[THIS]] to %[[STRUCT_A]] : !cir.ptr<!rec_A>
 // CIR:     %[[LAM_RET:.*]] = cir.call @_ZZN1A3fooEvENKUlvE_clEv(%[[LAM_ADDR]])
 // CIR:     cir.store{{.*}} %[[LAM_RET]], %[[RETVAL]]
 // CIR:   }
@@ -383,7 +383,7 @@ struct A {
 // LLVM:   br label %[[SCOPE_BB:.*]]
 // LLVM: [[SCOPE_BB]]:
 // LLVM:   %[[STRUCT_A:.*]] = getelementptr %[[REC_LAM_A]], ptr %[[LAM_ALLOCA]], i32 0, i32 0
-// LLVM:   call void @_ZN1AC1ERKS_(ptr %[[STRUCT_A]], ptr %[[THIS]])
+// LLVM:   call void @llvm.memcpy.p0.p0.i32(ptr %[[STRUCT_A]], ptr %[[THIS]], i32 4, i1 false)
 // LLVM:   %[[LAM_RET:.*]] = call i32 @_ZZN1A3fooEvENKUlvE_clEv(ptr %[[LAM_ALLOCA]])
 // LLVM:   store i32 %[[LAM_RET]], ptr %[[RETVAL]]
 // LLVM:   br label %[[RET_BB:.*]]
diff --git a/clang/test/CIR/CodeGen/struct.cpp b/clang/test/CIR/CodeGen/struct.cpp
index c15e7e7c57b9f..dc3e24113d8d8 100644
--- a/clang/test/CIR/CodeGen/struct.cpp
+++ b/clang/test/CIR/CodeGen/struct.cpp
@@ -109,13 +109,13 @@ void paren_expr() {
 // CIR:   %[[B_ADDR:.*]] = cir.alloca !rec_Point, !cir.ptr<!rec_Point>, ["b", init]
 // CIR:   %[[CONST:.*]] = cir.const #cir.zero : !rec_Point
 // CIR:   cir.store{{.*}} %[[CONST]], %[[A_ADDR]] : !rec_Point, !cir.ptr<!rec_Point>
-// CIR:   cir.call @_ZZ10paren_exprvEN5PointC1ERKS_(%[[B_ADDR]], %[[A_ADDR]]) nothrow : (!cir.ptr<!rec_Point>, !cir.ptr<!rec_Point>) -> ()
+// CIR:   cir.copy %[[A_ADDR]] to %[[B_ADDR]] : !cir.ptr<!rec_Point>
 
 // LLVM: define{{.*}} void @_Z10paren_exprv()
 // LLVM:   %[[A_ADDR:.*]] = alloca %struct.Point, i64 1, align 4
 // LLVM:   %[[B_ADDR:.*]] = alloca %struct.Point, i64 1, align 4
 // LLVM:   store %struct.Point zeroinitializer, ptr %[[A_ADDR]], align 4
-// LLVM:   call void @_ZZ10paren_exprvEN5PointC1ERKS_(ptr %[[B_ADDR]], ptr %[[A_ADDR]])
+// LLVM:   call void @llvm.memcpy.p0.p0.i32(ptr %[[B_ADDR]], ptr %[[A_ADDR]], i32 8, i1 false)
 
 // OGCG: define{{.*}} void @_Z10paren_exprv()
 // OGCG:   %[[A_ADDR:.*]] = alloca %struct.Point, align 4
@@ -133,14 +133,13 @@ void choose_expr() {
 // CIR:   %[[A_ADDR:.*]] = cir.alloca !rec_CompleteS, !cir.ptr<!rec_CompleteS>, ["a"]
 // CIR:   %[[B_ADDR:.*]] = cir.alloca !rec_CompleteS, !cir.ptr<!rec_CompleteS>, ["b"]
 // CIR:   %[[C_ADDR:.*]] = cir.alloca !rec_CompleteS, !cir.ptr<!rec_CompleteS>, ["c", init]
-// TODO(cir): Call to default copy constructor should be replaced by `cir.copy` op
-// CIR:   cir.call @_ZN9CompleteSC1ERKS_(%[[C_ADDR]], %[[A_ADDR]]) nothrow : (!cir.ptr<!rec_CompleteS>, !cir.ptr<!rec_CompleteS>) -> ()
+// CIR:   cir.copy %[[A_ADDR]] to %[[C_ADDR]] : !cir.ptr<!rec_CompleteS>
 
 // LLVM: define{{.*}} void @_Z11choose_exprv()
 // LLVM:   %[[A_ADDR:.*]] = alloca %struct.CompleteS, i64 1, align 4
 // LLVM:   %[[B_ADDR:.*]] = alloca %struct.CompleteS, i64 1, align 4
 // LLVM:   %[[C_ADDR:.*]] = alloca %struct.CompleteS, i64 1, align 4
-// LLVM:   call void @_ZN9CompleteSC1ERKS_(ptr %[[C_ADDR]], ptr %[[A_ADDR]])
+// LLVM:   call void @llvm.memcpy.p0.p0.i32(ptr %[[C_ADDR]], ptr %[[A_ADDR]], i32 8, i1 false)
 
 // OGCG: define{{.*}} void @_Z11choose_exprv()
 // OGCG:   %[[A_ADDR:.*]] = alloca %struct.CompleteS, align 4
@@ -160,15 +159,14 @@ void generic_selection() {
 // CIR:   %[[B_ADDR:.*]] = cir.alloca !rec_CompleteS, !cir.ptr<!rec_CompleteS>, ["b"]
 // CIR:   %[[C_ADDR:.*]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["c"]
 // CIR:   %[[D_ADDR:.*]] = cir.alloca !rec_CompleteS, !cir.ptr<!rec_CompleteS>, ["d", init]
-// TODO(cir): Call to default copy constructor should be replaced by `cir.copy` op
-// CIR:   cir.call @_ZN9CompleteSC1ERKS_(%[[D_ADDR]], %[[A_ADDR]]) nothrow : (!cir.ptr<!rec_CompleteS>, !cir.ptr<!rec_CompleteS>) -> ()
+// CIR:   cir.copy %[[A_ADDR]] to %[[D_ADDR]] : !cir.ptr<!rec_CompleteS>
 
 // LLVM: define{{.*}} void @_Z17generic_selectionv()
 // LLVM:   %1 = alloca %struct.CompleteS, i64 1, align 4
 // LLVM:   %2 = alloca %struct.CompleteS, i64 1, align 4
 // LLVM:   %3 = alloca i32, i64 1, align 4
 // LLVM:   %4 = alloca %struct.CompleteS, i64 1, align 4
-// LLVM:   call void @_ZN9CompleteSC1ERKS_(ptr %4, ptr %1)
+// LLVM:   call void @llvm.memcpy.p0.p0.i32(ptr %4, ptr %1, i32 8, i1 false)
 
 // OGCG: define{{.*}} void @_Z17generic_selectionv()
 // OGCG:   %[[A_ADDR:.*]] = alloca %struct.CompleteS, align 4
@@ -188,7 +186,7 @@ void designated_init_update_expr() {
 // CIR: %[[A_ADDR:.*]] = cir.alloca !rec_CompleteS, !cir.ptr<!rec_CompleteS>, ["a"]
 // CIR: %[[B_ADDR:.*]] = cir.alloca !rec_Container, !cir.ptr<!rec_Container>, ["b", init]
 // CIR: %[[C_ADDR:.*]] = cir.get_member %[[B_ADDR]][0] {name = "c"} : !cir.ptr<!rec_Container> -> !cir.ptr<!rec_CompleteS>
-// CIR: cir.call @_ZN9CompleteSC1ERKS_(%2, %[[A_ADDR]]) nothrow : (!cir.ptr<!rec_CompleteS>, !cir.ptr<!rec_CompleteS>) -> ()
+// CIR: cir.copy %[[A_ADDR]] to %[[C_ADDR]] : !cir.ptr<!rec_CompleteS>
 // CIR: %[[ELEM_0_PTR:.*]] = cir.get_member %[[C_ADDR]][0] {name = "a"} : !cir.ptr<!rec_CompleteS> -> !cir.ptr<!s32i>
 // CIR: %[[CONST_1:.*]] = cir.const #cir.int<1> : !s32i
 // CIR: cir.store{{.*}} %[[CONST_1]], %[[ELEM_0_PTR]] : !s32i, !cir.ptr<!s32i>
@@ -197,7 +195,7 @@ void designated_init_update_expr() {
 // LLVM: %[[A_ADDR:.*]] = alloca %struct.CompleteS, i64 1, align 4
 // LLVM: %[[B_ADDR:.*]] = alloca %struct.Container, i64 1, align 4
 // LLVM: %[[C_ADDR:.*]] = getelementptr %struct.Container, ptr %[[B_ADDR]], i32 0, i32 0
-// LLVM: call void @_ZN9CompleteSC1ERKS_(ptr %[[C_ADDR]], ptr %[[A_ADDR]])
+// LLVM: call void @llvm.memcpy.p0.p0.i32(ptr %[[C_ADDR]], ptr %[[A_ADDR]], i32 8, i1 false)
 // LLVM: %[[ELEM_0_PTR:.*]] = getelementptr %struct.CompleteS, ptr %[[C_ADDR]], i32 0, i32 0
 // LLVM: store i32 1, ptr %[[ELEM_0_PTR]], align 4
 // LLVM: %[[ELEM_1_PTR:.*]] = getelementptr %struct.CompleteS, ptr %[[C_ADDR]], i32 0, i32 1
diff --git a/clang/test/CIR/CodeGenBuiltins/X86/avx-builtins.c b/clang/test/CIR/CodeGenBuiltins/X86/avx-builtins.c
index 82fa4358dc400..d9a8771023fa7 100644
--- a/clang/test/CIR/CodeGenBuiltins/X86/avx-builtins.c
+++ b/clang/test/CIR/CodeGenBuiltins/X86/avx-builtins.c
@@ -73,4 +73,76 @@ __m256i test_mm256_undefined_si256(void) {
   // OGCG-LABEL: test_mm256_undefined_si256
   // OGCG: ret <4 x i64> zeroinitializer
   return _mm256_undefined_si256();
-}
\ No newline at end of file
+}
+
+__m256d test_mm256_shuffle_pd(__m256d A, __m256d B) {
+  // CIR-LABEL: test_mm256_shuffle_pd
+  // CIR: %{{.*}} = cir.vec.shuffle(%{{.*}}, %{{.*}} : !cir.vector<4 x !cir.double>) [#cir.int<0> : !s32i, #cir.int<4> : !s32i, #cir.int<2> : !s32i, #cir.int<6> : !s32i] : !cir.vector<4 x !cir.double>
+
+  // LLVM-LABEL: test_mm256_shuffle_pd
+  // LLVM: shufflevector <4 x double> %{{.*}}, <4 x double> %{{.*}}, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
+
+  // OGCG-LABEL: test_mm256_shuffle_pd
+  // OGCG: shufflevector <4 x double> %{{.*}}, <4 x double> %{{.*}}, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
+  return _mm256_shuffle_pd(A, B, 0);
+}
+
+__m256 test_mm256_shuffle_ps(__m256 A, __m256 B) {
+  // CIR-LABEL: test_mm256_shuffle_ps
+  // CIR: %{{.*}} = cir.vec.shuffle(%{{.*}}, %{{.*}} : !cir.vector<8 x !cir.float>) [#cir.int<0> : !s32i, #cir.int<0> : !s32i, #cir.int<8> : !s32i, #cir.int<8> : !s32i, #cir.int<4> : !s32i, #cir.int<4> : !s32i, #cir.int<12> : !s32i, #cir.int<12> : !s32i] : !cir.vector<8 x !cir.float>
+
+  // LLVM-LABEL: test_mm256_shuffle_ps
+  // LLVM: shufflevector <8 x float> %{{.*}}, <8 x float> %{{.*}}, <8 x i32> <i32 0, i32 0, i32 8, i32 8, i32 4, i32 4, i32 12, i32 12>
+
+  // OGCG-LABEL: test_mm256_shuffle_ps
+  // OGCG: shufflevector <8 x float> %{{.*}}, <8 x float> %{{.*}}, <8 x i32> <i32 0, i32 0, i32 8, i32 8, i32 4, i32 4, i32 12, i32 12>
+  return _mm256_shuffle_ps(A, B, 0);
+}
+
+__m128 test_mm_permute_ps(__m128 A) {
+    // CIR-LABEL: test_mm_permute_ps
+    // CIR: cir.vec.shuffle(%{{.*}}, %{{.*}} :  !cir.vector<4 x !cir.float>) [#cir.int<2> : !s32i, #cir.int<3> : !s32i, #cir.int<0> : !s32i, #cir.int<1> : !s32i] : !cir.vector<4 x !cir.float>
+
+	// LLVM-LABEL: test_mm_permute_ps
+    // LLVM: shufflevector <4 x float> %{{.*}}, <4 x float> poison, <4 x i32> <i32 2, i32 3, i32 0, i32 1>
+
+    // OGCG-LABEL: test_mm_permute_ps
+	// OGCG: shufflevector <4 x float> %{{.*}}, <4 x float> poison, <4 x i32> <i32 2, i32 3, i32 0, i32 1>
+    return _mm_permute_ps(A, 0x4E);
+}
+
+__m256 test_mm256_permute_ps(__m256 A) {
+    // CIR-LABEL: test_mm256_permute_ps
+    // CIR: cir.vec.shuffle(%{{.*}}, %{{.*}} :  !cir.vector<8 x !cir.float>) [#cir.int<2> : !s32i, #cir.int<3> : !s32i, #cir.int<0> : !s32i, #cir.int<1> : !s32i, #cir.int<6> : !s32i, #cir.int<7> : !s32i, #cir.int<4> : !s32i, #cir.int<5> : !s32i] : !cir.vector<8 x !cir.float>
+
+    // LLVM-LABEL: test_mm256_permute_ps
+    // LLVM: shufflevector <8 x float> %{{.*}}, <8 x float> poison, <8 x i32> <i32 2, i32 3, i32 0, i32 1, i32 6, i32 7, i32 4, i32 5>
+
+    // OGCG-LABEL: test_mm256_permute_ps
+    // OGCG: shufflevector <8 x float> %{{.*}}, <8 x float> poison, <8 x i32> <i32 2, i32 3, i32 0, i32 1, i32 6, i32 7, i32 4, i32 5>
+    return _mm256_permute_ps(A, 0x4E);
+}
+
+__m128d test_mm_permute_pd(__m128d A) {
+    // CIR-LABEL: test_mm_permute_pd
+    // CIR: cir.vec.shuffle(%{{.*}}, %{{.*}} :  !cir.vector<2 x !cir.double>) [#cir.int<1> : !s32i, #cir.int<0> : !s32i] : !cir.vector<2 x !cir.double>
+
+    // LLVM-LABEL: test_mm_permute_pd
+    // LLVM: shufflevector <2 x double> %{{.*}}, <2 x double> poison, <2 x i32> <i32 1, i32 0>
+
+    // OGCG-LABEL: test_mm_permute_pd
+    // OGCG: shufflevector <2 x double> %{{.*}}, <2 x double> poison, <2 x i32> <i32 1, i32 0>
+    return _mm_permute_pd(A, 0x1);
+}
+
+__m256d test_mm256_permute_pd(__m256d A) {
+    // CIR-LABEL: test_mm256_permute_pd
+    // CIR: cir.vec.shuffle(%{{.*}}, %{{.*}} :  !cir.vector<4 x !cir.double>) [#cir.int<1> : !s32i, #cir.int<0> : !s32i, #cir.int<3> : !s32i, #cir.int<2> : !s32i] : !cir.vector<4 x !cir.double>
+
+    // LLVM-LABEL: test_mm256_permute_pd
+    // LLVM: shufflevector <4 x double> %{{.*}}, <4 x double> poison, <4 x i32> <i32 1, i32 0, i32 3, i32 2>
+
+    // OGCG-LABEL: test_mm256_permute_pd
+    // OGCG: shufflevector <4 x double> %{{.*}}, <4 x double> poison, <4 x i32> <i32 1, i32 0, i32 3, i32 2>
+    return _mm256_permute_pd(A, 0x5);
+}
diff --git a/clang/test/CIR/CodeGenBuiltins/X86/avx2-builtins.c b/clang/test/CIR/CodeGenBuiltins/X86/avx2-builtins.c
new file mode 100644
index 0000000000000..b7497c2053b2d
--- /dev/null
+++ b/clang/test/CIR/CodeGenBuiltins/X86/avx2-builtins.c
@@ -0,0 +1,53 @@
+// RUN: %clang_cc1 -x c -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-unknown-linux -target-feature +avx2 -fclangir -emit-cir -o %t.cir -Wall -Werror
+// RUN: FileCheck --check-prefixes=CIR --input-file=%t.cir %s
+// RUN: %clang_cc1 -x c -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-unknown-linux -target-feature +avx2 -fno-signed-char -fclangir -emit-cir -o %t.cir -Wall -Werror
+// RUN: FileCheck --check-prefixes=CIR --input-file=%t.cir %s
+
+// RUN: %clang_cc1 -x c -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-unknown-linux -target-feature +avx2 -fclangir -emit-llvm -o %t.ll -Wall -Werror
+// RUN: FileCheck --check-prefixes=LLVM --input-file=%t.ll %s
+// RUN: %clang_cc1 -x c -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-unknown-linux -target-feature +avx2 -fno-signed-char -fclangir -emit-llvm -o %t.ll -Wall -Werror
+// RUN: FileCheck --check-prefixes=LLVM --input-file=%t.ll %s
+
+// RUN: %clang_cc1 -x c++ -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-unknown-linux -target-feature +avx2 -fclangir -emit-cir -o %t.cir -Wall -Werror
+// RUN: FileCheck --check-prefixes=CIR --input-file=%t.cir %s
+// RUN: %clang_cc1 -x c++ -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-unknown-linux -target-feature +avx2 -fno-signed-char -fclangir -emit-cir -o %t.cir -Wall -Werror
+// RUN: FileCheck --check-prefixes=CIR --input-file=%t.cir %s
+
+// RUN: %clang_cc1 -x c++ -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-unknown-linux -target-feature +avx2 -fclangir -emit-llvm -o %t.ll -Wall -Werror
+// RUN: FileCheck --check-prefixes=LLVM --input-file=%t.ll %s
+// RUN: %clang_cc1 -x c++ -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-unknown-linux -target-feature +avx2 -fno-signed-char -fclangir -emit-llvm -o %t.ll -Wall -Werror
+// RUN: FileCheck --check-prefixes=LLVM --input-file=%t.ll %s
+
+// RUN: %clang_cc1 -x c -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +avx2 -emit-llvm -o - -Wall -Werror | FileCheck %s --check-prefixes=OGCG
+// RUN: %clang_cc1 -x c -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +avx2 -fno-signed-char -emit-llvm -o - -Wall -Werror | FileCheck %s --check-prefixes=OGCG
+// RUN: %clang_cc1 -x c++ -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +avx2 -emit-llvm -o - -Wall -Werror | FileCheck %s --check-prefixes=OGCG
+// RUN: %clang_cc1 -x c++ -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +avx2 -fno-signed-char -emit-llvm -o - -Wall -Werror | FileCheck %s --check-prefixes=OGCG
+
+// This test mimics clang/test/CodeGen/X86/avx2-builtins.c, which eventually
+// CIR shall be able to support fully.
+
+#include <immintrin.h>
+
+__m256i test_mm256_shufflelo_epi16(__m256i a) {
+  // CIR-LABEL: _mm256_shufflelo_epi16
+  // CIR: %{{.*}} = cir.vec.shuffle(%{{.*}}, %{{.*}} : !cir.vector<16 x !s16i>) [#cir.int<3> : !s32i, #cir.int<0> : !s32i, #cir.int<1> : !s32i, #cir.int<1> : !s32i, #cir.int<4> : !s32i, #cir.int<5> : !s32i, #cir.int<6> : !s32i, #cir.int<7> : !s32i, #cir.int<11> : !s32i, #cir.int<8> : !s32i, #cir.int<9> : !s32i, #cir.int<9> : !s32i, #cir.int<12> : !s32i, #cir.int<13> : !s32i, #cir.int<14> : !s32i, #cir.int<15> : !s32i] : !cir.vector<16 x !s16i>
+
+  // LLVM-LABEL: test_mm256_shufflelo_epi16
+  // LLVM: shufflevector <16 x i16> %{{.*}}, <16 x i16> poison, <16 x i32> <i32 3, i32 0, i32 1, i32 1, i32 4, i32 5, i32 6, i32 7, i32 11, i32 8, i32 9, i32 9, i32 12, i32 13, i32 14, i32 15>
+
+  // OGCG-LABEL: test_mm256_shufflelo_epi16
+  // OGCG: shufflevector <16 x i16> %{{.*}}, <16 x i16> poison, <16 x i32> <i32 3, i32 0, i32 1, i32 1, i32 4, i32 5, i32 6, i32 7, i32 11, i32 8, i32 9, i32 9, i32 12, i32 13, i32 14, i32 15>
+  return _mm256_shufflelo_epi16(a, 83);
+}
+
+__m256i test_mm256_shufflehi_epi16(__m256i a) {
+  // CIR-LABEL: _mm256_shufflehi_epi16
+  // CIR: %{{.*}} = cir.vec.shuffle(%{{.*}}, %{{.*}} : !cir.vector<16 x !s16i>) [#cir.int<0> : !s32i, #cir.int<1> : !s32i, #cir.int<2> : !s32i, #cir.int<3> : !s32i, #cir.int<7> : !s32i, #cir.int<6> : !s32i, #cir.int<6> : !s32i, #cir.int<5> : !s32i, #cir.int<8> : !s32i, #cir.int<9> : !s32i, #cir.int<10> : !s32i, #cir.int<11> : !s32i, #cir.int<15> : !s32i, #cir.int<14> : !s32i, #cir.int<14> : !s32i, #cir.int<13> : !s32i] : !cir.vector<16 x !s16i>
+
+  // LLVM-LABEL: test_mm256_shufflehi_epi16
+  // LLVM: shufflevector <16 x i16> %{{.*}}, <16 x i16> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 7, i32 6, i32 6, i32 5, i32 8, i32 9, i32 10, i32 11, i32 15, i32 14, i32 14, i32 13>
+
+  // OGCG-LABEL: test_mm256_shufflehi_epi16
+  // OGCG: shufflevector <16 x i16> %{{.*}}, <16 x i16> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 7, i32 6, i32 6, i32 5, i32 8, i32 9, i32 10, i32 11, i32 15, i32 14, i32 14, i32 13>
+  return _mm256_shufflehi_epi16(a, 107);
+}
diff --git a/clang/test/CIR/CodeGenBuiltins/X86/avx512bw-builtins.c b/clang/test/CIR/CodeGenBuiltins/X86/avx512bw-builtins.c
index 4863ba0bd8848..48a89769ea10f 100644
--- a/clang/test/CIR/CodeGenBuiltins/X86/avx512bw-builtins.c
+++ b/clang/test/CIR/CodeGenBuiltins/X86/avx512bw-builtins.c
@@ -1,15 +1,32 @@
-// RUN: %clang_cc1 -x c -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-unknown-linux -target-feature +avx512bw -fclangir -emit-cir -o %t.cir -Wall -Werror
+// RUN: %clang_cc1 -x c -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-unknown-linux -target-feature +avx512bw -fclangir -emit-cir -o %t.cir -Wall -Werror -Wsign-conversion
+// RUN: FileCheck --check-prefix=CIR --input-file=%t.cir %s
+// RUN: %clang_cc1 -x c -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-unknown-linux -target-feature +avx512bw -fno-signed-char  -fclangir -emit-cir -o %t.cir -Wall -Werror -Wsign-conversion
 // RUN: FileCheck --check-prefix=CIR --input-file=%t.cir %s
-// RUN: %clang_cc1 -x c -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-unknown-linux -target-feature +avx512bw -fclangir -emit-llvm -o %t.ll -Wall -Werror
-// RUN: FileCheck --check-prefix=LLVM --input-file=%t.ll %s
 
-// RUN: %clang_cc1 -x c++ -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-unknown-linux -target-feature +avx512bw -fno-signed-char -fclangir -emit-cir -o %t.cir -Wall -Werror
+// RUN: %clang_cc1 -x c++ -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-unknown-linux -target-feature +avx512bw -fclangir -emit-cir -o %t.cir -Wall -Werror -Wsign-conversion
+// RUN: FileCheck --check-prefix=CIR --input-file=%t.cir %s
+// RUN: %clang_cc1 -x c++ -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-unknown-linux -target-feature +avx512bw -fno-signed-char  -fclangir -emit-cir -o %t.cir -Wall -Werror -Wsign-conversion
 // RUN: FileCheck --check-prefix=CIR --input-file=%t.cir %s
-// RUN: %clang_cc1 -x c++ -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-unknown-linux -target-feature +avx512bw -fno-signed-char -fclangir -emit-llvm -o %t.ll -Wall -Werror
-// RUN: FileCheck --check-prefix=LLVM --input-file=%t.ll %s
 
-// RUN: %clang_cc1 -x c -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-unknown-linux -target-feature +avx512bw -emit-llvm -o - -Wall -Werror | FileCheck %s -check-prefix=OGCG
-// RUN: %clang_cc1 -x c++ -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-unknown-linux -target-feature +avx512bw -emit-llvm -o - -Wall -Werror | FileCheck %s -check-prefix=OGCG
+// RUN: %clang_cc1 -x c -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-unknown-linux -target-feature +avx512bw  -fclangir -emit-llvm -o %t.ll -Wall -Werror -Wsign-conversion
+// RUN: FileCheck --check-prefixes=LLVM --input-file=%t.ll %s
+// RUN: %clang_cc1 -x c -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-unknown-linux  -target-feature +avx512bw -fno-signed-char  -fclangir -emit-llvm -o %t.ll -Wall -Werror -Wsign-conversion
+// RUN: FileCheck --check-prefixes=LLVM --input-file=%t.ll %s
+
+// RUN: %clang_cc1 -x c++ -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-unknown-linux -target-feature +avx512bw  -fclangir -emit-llvm -o %t.ll -Wall -Werror -Wsign-conversion
+// RUN: FileCheck --check-prefixes=LLVM --input-file=%t.ll %s
+// RUN: %clang_cc1 -x c++ -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-unknown-linux  -target-feature +avx512bw -fno-signed-char  -fclangir -emit-llvm -o %t.ll -Wall -Werror -Wsign-conversion
+// RUN: FileCheck --check-prefixes=LLVM --input-file=%t.ll %s
+
+// RUN: %clang_cc1 -x c -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-unknown-linux -target-feature +avx512bw -emit-llvm -o - -Wall -Werror -Wsign-conversion | FileCheck %s --check-prefix=OGCG
+// RUN: %clang_cc1 -x c -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-unknown-linux -target-feature +avx512bw -fno-signed-char -emit-llvm -o - -Wall -Werror -Wsign-conversion | FileCheck %s --check-prefix=OGCG
+// RUN: %clang_cc1 -x c -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +avx512bw -emit-llvm -o - -Wall -Werror -Wsign-conversion | FileCheck %s --check-prefixes=OGCG
+// RUN: %clang_cc1 -x c -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +avx512bw -fno-signed-char -emit-llvm -o - -Wall -Werror -Wsign-conversion | FileCheck %s --check-prefixes=OGCG
+
+// RUN: %clang_cc1 -x c++ -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-unknown-linux -target-feature +avx512bw -emit-llvm -o - -Wall -Werror -Wsign-conversion | FileCheck %s --check-prefix=OGCG
+// RUN: %clang_cc1 -x c++ -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-unknown-linux -target-feature +avx512bw -fno-signed-char -emit-llvm -o - -Wall -Werror -Wsign-conversion | FileCheck %s --check-prefix=OGCG
+// RUN: %clang_cc1 -x c++ -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +avx512bw -emit-llvm -o - -Wall -Werror -Wsign-conversion | FileCheck %s --check-prefixes=OGCG
+// RUN: %clang_cc1 -x c++ -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +avx512bw -fno-signed-char -emit-llvm -o - -Wall -Werror -Wsign-conversion | FileCheck %s --check-prefixes=OGCG
 
 // This test mimics clang/test/CodeGen/X86/avx512bw-builtins.c, which eventually
 // CIR shall be able to support fully.
@@ -116,7 +133,6 @@ __mmask32 test_kshiftri_mask32_out_of_range(__mmask32 A) {
   return _kshiftri_mask32(A, 33);
 }
 
-
 __mmask32 test_kadd_mask32(__mmask32 A, __mmask32 B) {
   // CIR-LABEL: _kadd_mask32
   // CIR: cir.cast bitcast {{.*}} : !u32i -> !cir.vector<32 x !cir.int<u, 1>>
@@ -465,3 +481,81 @@ __mmask64 test_kmov_q(__mmask64 A) {
 
   return __builtin_ia32_kmovq(A);
 }
+
+__mmask32 test_mm512_kunpackw(__mmask32 A, __mmask32 B) {
+  // CIR-LABEL: _mm512_kunpackw
+  // CIR: cir.cast bitcast {{.*}} : !u32i -> !cir.vector<32 x !cir.int<u, 1>>
+  // CIR: cir.cast bitcast {{.*}} : !u32i -> !cir.vector<32 x !cir.int<u, 1>>
+  // CIR: cir.vec.shuffle
+  // CIR: cir.vec.shuffle
+  // CIR: cir.vec.shuffle
+  // CIR: cir.cast bitcast {{.*}} : !cir.vector<32 x !cir.int<u, 1>> -> !u32i
+
+  // LLVM-LABEL: _mm512_kunpackw
+  // LLVM: [[A_VEC:%.*]] = bitcast i32 %{{.*}} to <32 x i1>
+  // LLVM: [[B_VEC:%.*]] = bitcast i32 %{{.*}} to <32 x i1>
+  // LLVM: [[A_HALF:%.*]] = shufflevector <32 x i1> [[A_VEC]], <32 x i1> [[A_VEC]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  // LLVM: [[B_HALF:%.*]] = shufflevector <32 x i1> [[B_VEC]], <32 x i1> [[B_VEC]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  // LLVM: [[RES:%.*]] = shufflevector <16 x i1> [[B_HALF]], <16 x i1> [[A_HALF]], <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+  // LLVM: bitcast <32 x i1> [[RES]] to i32
+
+  // OGCG-LABEL: _mm512_kunpackw
+  // OGCG: [[A_VEC:%.*]] = bitcast i32 %{{.*}} to <32 x i1>
+  // OGCG: [[B_VEC:%.*]] = bitcast i32 %{{.*}} to <32 x i1>
+  // OGCG: [[A_HALF:%.*]] = shufflevector <32 x i1> [[A_VEC]], <32 x i1> [[A_VEC]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  // OGCG: [[B_HALF:%.*]] = shufflevector <32 x i1> [[B_VEC]], <32 x i1> [[B_VEC]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  // OGCG: [[RES:%.*]] = shufflevector <16 x i1> [[B_HALF]], <16 x i1> [[A_HALF]], <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+  // OGCG: bitcast <32 x i1> [[RES]] to i32
+  return _mm512_kunpackw(A, B);
+}
+
+__mmask64 test_mm512_kunpackd(__mmask64 A, __mmask64 B) {
+  // CIR-LABEL: _mm512_kunpackd
+  // CIR: cir.cast bitcast {{.*}} : !u64i -> !cir.vector<64 x !cir.int<u, 1>>
+  // CIR: cir.cast bitcast {{.*}} : !u64i -> !cir.vector<64 x !cir.int<u, 1>>
+  // CIR: cir.vec.shuffle
+  // CIR: cir.vec.shuffle
+  // CIR: cir.vec.shuffle
+  // CIR: cir.cast bitcast {{.*}} : !cir.vector<64 x !cir.int<u, 1>> -> !u64i
+
+  // LLVM-LABEL: _mm512_kunpackd
+  // LLVM: [[A_VEC:%.*]] = bitcast i64 %{{.*}} to <64 x i1>
+  // LLVM: [[B_VEC:%.*]] = bitcast i64 %{{.*}} to <64 x i1>
+  // LLVM: [[A_HALF:%.*]] = shufflevector <64 x i1> [[A_VEC]], <64 x i1> [[A_VEC]], <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+  // LLVM: [[B_HALF:%.*]] = shufflevector <64 x i1> [[B_VEC]], <64 x i1> [[B_VEC]], <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+  // LLVM: [[RES:%.*]] = shufflevector <32 x i1> [[B_HALF]], <32 x i1> [[A_HALF]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
+  // LLVM: bitcast <64 x i1> [[RES]] to i64
+
+  // OGCG-LABEL: _mm512_kunpackd
+  // OGCG: [[A_VEC:%.*]] = bitcast i64 %{{.*}} to <64 x i1>
+  // OGCG: [[B_VEC:%.*]] = bitcast i64 %{{.*}} to <64 x i1>
+  // OGCG: [[A_HALF:%.*]] = shufflevector <64 x i1> [[A_VEC]], <64 x i1> [[A_VEC]], <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+  // OGCG: [[B_HALF:%.*]] = shufflevector <64 x i1> [[B_VEC]], <64 x i1> [[B_VEC]], <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+  // OGCG: [[RES:%.*]] = shufflevector <32 x i1> [[B_HALF]], <32 x i1> [[A_HALF]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
+  // OGCG: bitcast <64 x i1> [[RES]] to i64
+  return _mm512_kunpackd(A, B);
+}
+
+__m512i test_mm512_shufflelo_epi16(__m512i __A) {
+  // CIR-LABEL: _mm512_shufflelo_epi16
+  // CIR: %{{.*}} = cir.vec.shuffle(%{{.*}}, %{{.*}} : !cir.vector<32 x !s16i>) [#cir.int<1> : !s32i, #cir.int<1> : !s32i, #cir.int<0> : !s32i, #cir.int<0> : !s32i, #cir.int<4> : !s32i, #cir.int<5> : !s32i, #cir.int<6> : !s32i, #cir.int<7> : !s32i, #cir.int<9> : !s32i, #cir.int<9> : !s32i, #cir.int<8> : !s32i, #cir.int<8> : !s32i, #cir.int<12> : !s32i, #cir.int<13> : !s32i, #cir.int<14> : !s32i, #cir.int<15> : !s32i, #cir.int<17> : !s32i, #cir.int<17> : !s32i, #cir.int<16> : !s32i, #cir.int<16> : !s32i, #cir.int<20> : !s32i, #cir.int<21> : !s32i, #cir.int<22> : !s32i, #cir.int<23> : !s32i, #cir.int<25> : !s32i, #cir.int<25> : !s32i, #cir.int<24> : !s32i, #cir.int<24> : !s32i, #cir.int<28> : !s32i, #cir.int<29> : !s32i, #cir.int<30> : !s32i, #cir.int<31> : !s32i] : !cir.vector<32 x !s16i>
+
+  // LLVM-LABEL: test_mm512_shufflelo_epi16
+  // LLVM: shufflevector <32 x i16> %{{.*}}, <32 x i16> poison, <32 x i32> <i32 1, i32 1, i32 0, i32 0, i32 4, i32 5, i32 6, i32 7, i32 9, i32 9, i32 8, i32 8, i32 12, i32 13, i32 14, i32 15, i32 17, i32 17, i32 16, i32 16, i32 20, i32 21, i32 22, i32 23, i32 25, i32 25, i32 24, i32 24, i32 28, i32 29, i32 30, i32 31>
+
+  // OGCG-LABEL: test_mm512_shufflelo_epi16
+  // OGCG: shufflevector <32 x i16> %{{.*}}, <32 x i16> poison, <32 x i32> <i32 1, i32 1, i32 0, i32 0, i32 4, i32 5, i32 6, i32 7, i32 9, i32 9, i32 8, i32 8, i32 12, i32 13, i32 14, i32 15, i32 17, i32 17, i32 16, i32 16, i32 20, i32 21, i32 22, i32 23, i32 25, i32 25, i32 24, i32 24, i32 28, i32 29, i32 30, i32 31>
+  return _mm512_shufflelo_epi16(__A, 5);
+}
+
+__m512i test_mm512_shufflehi_epi16(__m512i __A) {
+  // CIR-LABEL: _mm512_shufflehi_epi16
+  // CIR: %{{.*}} = cir.vec.shuffle(%{{.*}}, %{{.*}} : !cir.vector<32 x !s16i>) [#cir.int<0> : !s32i, #cir.int<1> : !s32i, #cir.int<2> : !s32i, #cir.int<3> : !s32i, #cir.int<5> : !s32i, #cir.int<5> : !s32i, #cir.int<4> : !s32i, #cir.int<4> : !s32i, #cir.int<8> : !s32i, #cir.int<9> : !s32i, #cir.int<10> : !s32i, #cir.int<11> : !s32i, #cir.int<13> : !s32i, #cir.int<13> : !s32i, #cir.int<12> : !s32i, #cir.int<12> : !s32i, #cir.int<16> : !s32i, #cir.int<17> : !s32i, #cir.int<18> : !s32i, #cir.int<19> : !s32i, #cir.int<21> : !s32i, #cir.int<21> : !s32i, #cir.int<20> : !s32i, #cir.int<20> : !s32i, #cir.int<24> : !s32i, #cir.int<25> : !s32i, #cir.int<26> : !s32i, #cir.int<27> : !s32i, #cir.int<29> : !s32i, #cir.int<29> : !s32i, #cir.int<28> : !s32i, #cir.int<28> : !s32i] : !cir.vector<32 x !s16i>
+
+  // LLVM-LABEL: test_mm512_shufflehi_epi16
+  // LLVM: shufflevector <32 x i16> %{{.*}}, <32 x i16> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 5, i32 5, i32 4, i32 4, i32 8, i32 9, i32 10, i32 11, i32 13, i32 13, i32 12, i32 12, i32 16, i32 17, i32 18, i32 19, i32 21, i32 21, i32 20, i32 20, i32 24, i32 25, i32 26, i32 27, i32 29, i32 29, i32 28, i32 28>
+
+  // OGCG-LABEL: test_mm512_shufflehi_epi16
+  // OGCG: shufflevector <32 x i16> %{{.*}}, <32 x i16> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 5, i32 5, i32 4, i32 4, i32 8, i32 9, i32 10, i32 11, i32 13, i32 13, i32 12, i32 12, i32 16, i32 17, i32 18, i32 19, i32 21, i32 21, i32 20, i32 20, i32 24, i32 25, i32 26, i32 27, i32 29, i32 29, i32 28, i32 28>
+  return _mm512_shufflehi_epi16(__A, 5);
+}
diff --git a/clang/test/CIR/CodeGenBuiltins/X86/avx512f-builtins.c b/clang/test/CIR/CodeGenBuiltins/X86/avx512f-builtins.c
index 9d957f5de554d..a0e50ff9b4017 100644
--- a/clang/test/CIR/CodeGenBuiltins/X86/avx512f-builtins.c
+++ b/clang/test/CIR/CodeGenBuiltins/X86/avx512f-builtins.c
@@ -78,6 +78,54 @@ __m512i test_mm512_undefined_epi32(void) {
   return _mm512_undefined_epi32();
 }
 
+__m512d test_mm512_shuffle_pd(__m512d __M, __m512d __V) {
+  // CIR-LABEL: test_mm512_shuffle_pd
+  // CIR: %{{.*}} = cir.vec.shuffle(%{{.*}}, %{{.*}} : !cir.vector<8 x !cir.double>) [#cir.int<0> : !s32i, #cir.int<8> : !s32i, #cir.int<3> : !s32i, #cir.int<10> : !s32i, #cir.int<4> : !s32i, #cir.int<12> : !s32i, #cir.int<6> : !s32i, #cir.int<14> : !s32i] : !cir.vector<8 x !cir.double>
+
+  // LLVM-LABEL: test_mm512_shuffle_pd
+  // LLVM: shufflevector <8 x double> %{{.*}}, <8 x double> %{{.*}}, <8 x i32> <i32 0, i32 8, i32 3, i32 10, i32 4, i32 12, i32 6, i32 14>
+
+  // OGCG-LABEL: test_mm512_shuffle_pd
+  // OGCG: shufflevector <8 x double> %{{.*}}, <8 x double> %{{.*}}, <8 x i32> <i32 0, i32 8, i32 3, i32 10, i32 4, i32 12, i32 6, i32 14>
+  return _mm512_shuffle_pd(__M, __V, 4);
+}
+
+__m512 test_mm512_shuffle_ps(__m512 __M, __m512 __V) {
+  // CIR-LABEL: test_mm512_shuffle_ps
+  // CIR: %{{.*}} = cir.vec.shuffle(%{{.*}}, %{{.*}} : !cir.vector<16 x !cir.float>) [#cir.int<0> : !s32i, #cir.int<1> : !s32i, #cir.int<16> : !s32i, #cir.int<16> : !s32i, #cir.int<4> : !s32i, #cir.int<5> : !s32i, #cir.int<20> : !s32i, #cir.int<20> : !s32i, #cir.int<8> : !s32i, #cir.int<9> : !s32i, #cir.int<24> : !s32i, #cir.int<24> : !s32i, #cir.int<12> : !s32i, #cir.int<13> : !s32i, #cir.int<28> : !s32i, #cir.int<28> : !s32i] : !cir.vector<16 x !cir.float>
+
+  // LLVM-LABEL: test_mm512_shuffle_ps
+  // LLVM: shufflevector <16 x float> %{{.*}}, <16 x float> %{{.*}}, <16 x i32> <i32 0, i32 1, i32 16, i32 16, i32 4, i32 5, i32 20, i32 20, i32 8, i32 9, i32 24, i32 24, i32 12, i32 13, i32 28, i32 28>
+
+  // OGCG-LABEL: test_mm512_shuffle_ps
+  // OGCG: shufflevector <16 x float> %{{.*}}, <16 x float> %{{.*}}, <16 x i32> <i32 0, i32 1, i32 16, i32 16, i32 4, i32 5, i32 20, i32 20, i32 8, i32 9, i32 24, i32 24, i32 12, i32 13, i32 28, i32 28>
+  return _mm512_shuffle_ps(__M, __V, 4);
+}
+
+__m512 test_mm512_permute_ps(__m512 A) {
+    // CIR-LABEL: test_mm512_permute_ps
+    // CIR: cir.vec.shuffle(%{{.*}}, %{{.*}} : !cir.vector<16 x !cir.float>) [#cir.int<2> : !s32i, #cir.int<3> : !s32i, #cir.int<0> : !s32i, #cir.int<1> : !s32i, #cir.int<6> : !s32i, #cir.int<7> : !s32i, #cir.int<4> : !s32i, #cir.int<5> : !s32i, #cir.int<10> : !s32i, #cir.int<11> : !s32i, #cir.int<8> : !s32i, #cir.int<9> : !s32i, #cir.int<14> : !s32i, #cir.int<15> : !s32i, #cir.int<12> : !s32i, #cir.int<13> : !s32i] : !cir.vector<16 x !cir.float>
+
+    // LLVM-LABEL: test_mm512_permute_ps
+    // LLVM: shufflevector <16 x float> %{{.*}}, <16 x float> poison, <16 x i32> <i32 2, i32 3, i32 0, i32 1, i32 6, i32 7, i32 4, i32 5, i32 10, i32 11, i32 8, i32 9, i32 14, i32 15, i32 12, i32 13>
+
+    // OGCG-LABEL: test_mm512_permute_ps
+    // OGCG: shufflevector <16 x float> %{{.*}}, <16 x float> poison, <16 x i32> <i32 2, i32 3, i32 0, i32 1, i32 6, i32 7, i32 4, i32 5, i32 10, i32 11, i32 8, i32 9, i32 14, i32 15, i32 12, i32 13>
+    return _mm512_permute_ps(A, 0x4E);
+}
+
+__m512d test_mm512_permute_pd(__m512d A) {
+    // CIR-LABEL: test_mm512_permute_pd
+    // CIR: cir.vec.shuffle(%{{.*}}, %{{.*}} : !cir.vector<8 x !cir.double>) [#cir.int<1> : !s32i, #cir.int<0> : !s32i, #cir.int<3> : !s32i, #cir.int<2> : !s32i, #cir.int<5> : !s32i, #cir.int<4> : !s32i, #cir.int<7> : !s32i, #cir.int<6> : !s32i] : !cir.vector<8 x !cir.double>
+
+    // LLVM-LABEL: test_mm512_permute_pd
+    // LLVM: shufflevector <8 x double> %{{.*}}, <8 x double> poison, <8 x i32> <i32 1, i32 0, i32 3, i32 2, i32 5, i32 4, i32 7, i32 6>
+
+    // OGCG-LABEL: test_mm512_permute_pd
+    // OGCG: shufflevector <8 x double> %{{.*}}, <8 x double> poison, <8 x i32> <i32 1, i32 0, i32 3, i32 2, i32 5, i32 4, i32 7, i32 6>
+    return _mm512_permute_pd(A, 0x55);
+}
+
 __mmask16 test_mm512_kand(__mmask16 A, __mmask16 B) {
   // CIR-LABEL: _mm512_kand
   // CIR: cir.cast bitcast {{.*}} : !u16i -> !cir.vector<16 x !cir.int<u, 1>>
@@ -228,6 +276,33 @@ __mmask16 test_kmov_w(__mmask16 A) {
   // OGCG: bitcast <16 x i1> {{.*}} to i16
   return __builtin_ia32_kmovw(A);
 }
+
+__mmask16 test_mm512_kunpackb(__mmask16 A, __mmask16 B) {
+  // CIR-LABEL: _mm512_kunpackb
+  // CIR: cir.cast bitcast {{.*}} : !u16i -> !cir.vector<16 x !cir.int<u, 1>>
+  // CIR: cir.cast bitcast {{.*}} : !u16i -> !cir.vector<16 x !cir.int<u, 1>>
+  // CIR: cir.vec.shuffle
+  // CIR: cir.vec.shuffle
+  // CIR: cir.vec.shuffle
+  // CIR: cir.cast bitcast {{.*}} : !cir.vector<16 x !cir.int<u, 1>> -> !u16i
+
+  // LLVM-LABEL: _mm512_kunpackb
+  // LLVM: [[A_VEC:%.*]] = bitcast i16 %{{.*}} to <16 x i1>
+  // LLVM: [[B_VEC:%.*]] = bitcast i16 %{{.*}} to <16 x i1>
+  // LLVM: [[A_HALF:%.*]] = shufflevector <16 x i1> [[A_VEC]], <16 x i1> [[A_VEC]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  // LLVM: [[B_HALF:%.*]] = shufflevector <16 x i1> [[B_VEC]], <16 x i1> [[B_VEC]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  // LLVM: [[RES:%.*]] = shufflevector <8 x i1> [[B_HALF]], <8 x i1> [[A_HALF]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  // LLVM: bitcast <16 x i1> [[RES]] to i16
+
+  // OGCG-LABEL: _mm512_kunpackb
+  // OGCG: [[A_VEC:%.*]] = bitcast i16 %{{.*}} to <16 x i1>
+  // OGCG: [[B_VEC:%.*]] = bitcast i16 %{{.*}} to <16 x i1>
+  // OGCG: [[A_HALF:%.*]] = shufflevector <16 x i1> [[A_VEC]], <16 x i1> [[A_VEC]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  // OGCG: [[B_HALF:%.*]] = shufflevector <16 x i1> [[B_VEC]], <16 x i1> [[B_VEC]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  // OGCG: [[RES:%.*]] = shufflevector <8 x i1> [[B_HALF]], <8 x i1> [[A_HALF]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  // OGCG: bitcast <16 x i1> [[RES]] to i16
+  return _mm512_kunpackb(A, B);
+}
 __m256 test_mm512_i64gather_ps(__m512i __index, void const *__addr) {
   // CIR-LABEL: test_mm512_i64gather_ps
   // CIR: cir.call_llvm_intrinsic "x86.avx512.mask.gather.qps.512"
@@ -419,3 +494,36 @@ __m512i test_mm512_mask_i32gather_epi64(__m512i __v1_old, __mmask8 __mask, __m25
   // OGCG: call <8 x i64> @llvm.x86.avx512.mask.gather.dpq.512
   return _mm512_mask_i32gather_epi64(__v1_old, __mask, __index, __addr, 2);
 }
+
+__m512i test_mm512_ror_epi32(__m512i __A) {
+  // CIR-LABEL: test_mm512_ror_epi32
+  // CIR: cir.cast integral %{{.*}} : !s32i -> !u32i
+  // CIR: cir.vec.splat %{{.*}} : !u32i, !cir.vector<16 x !u32i>
+  // CIR: cir.call_llvm_intrinsic "fshr" %{{.*}}: (!cir.vector<16 x !s32i>, !cir.vector<16 x !s32i>, !cir.vector<16 x !u32i>) -> !cir.vector<16 x !s32i> 
+
+  // LLVM-LABEL: test_mm512_ror_epi32
+  // LLVM: %[[CASTED_VAR:.*]] = bitcast <8 x i64> %{{.*}} to <16 x i32>
+  // LLVM: call <16 x i32> @llvm.fshr.v16i32(<16 x i32> %[[CASTED_VAR]], <16 x i32> %[[CASTED_VAR]], <16 x i32> splat (i32 5))
+
+  // OGCG-LABEL: test_mm512_ror_epi32
+  // OGCG: %[[CASTED_VAR:.*]] = bitcast <8 x i64> %{{.*}} to <16 x i32>
+  // OGCG: call <16 x i32> @llvm.fshr.v16i32(<16 x i32> %[[CASTED_VAR]], <16 x i32> %[[CASTED_VAR]], <16 x i32> splat (i32 5))
+  return _mm512_ror_epi32(__A, 5); 
+}
+
+__m512i test_mm512_ror_epi64(__m512i __A) {
+  // CIR-LABEL: test_mm512_ror_epi64
+  // CIR: cir.cast integral %{{.*}} : !s32i -> !u32i
+  // CIR: cir.cast integral %{{.*}} : !u32i -> !u64i
+  // CIR: cir.vec.splat %{{.*}} : !u64i, !cir.vector<8 x !u64i>
+  // CIR: cir.call_llvm_intrinsic "fshr" %{{.*}}: (!cir.vector<8 x !s64i>, !cir.vector<8 x !s64i>, !cir.vector<8 x !u64i>) -> !cir.vector<8 x !s64i> 
+
+  // LLVM-LABEL: test_mm512_ror_epi64
+  // LLVM: %[[VAR:.*]] = load <8 x i64>, ptr %{{.*}}, align 64
+  // LLVM: call <8 x i64> @llvm.fshr.v8i64(<8 x i64> %[[VAR]], <8 x i64> %[[VAR]], <8 x i64> splat (i64 5))
+
+  // OGCG-LABEL: test_mm512_ror_epi64
+  // OGCG: %[[VAR:.*]] = load <8 x i64>, ptr %{{.*}}, align 64
+  // OGCG: call <8 x i64> @llvm.fshr.v8i64(<8 x i64> %[[VAR]], <8 x i64> %[[VAR]], <8 x i64> splat (i64 5))
+  return _mm512_ror_epi64(__A, 5); 
+}
diff --git a/clang/test/CIR/CodeGenBuiltins/X86/sse-builtins.c b/clang/test/CIR/CodeGenBuiltins/X86/sse-builtins.c
index c893859b297cc..db52021d1aa9f 100644
--- a/clang/test/CIR/CodeGenBuiltins/X86/sse-builtins.c
+++ b/clang/test/CIR/CodeGenBuiltins/X86/sse-builtins.c
@@ -71,3 +71,15 @@ __m128 test_mm_undefined_ps(void) {
   // OGCG: ret <4 x float> zeroinitializer
   return _mm_undefined_ps();
 }
+
+__m128 test_mm_shuffle_ps(__m128 A, __m128 B) {
+  // CIR-LABEL: _mm_shuffle_ps
+  // CIR: %{{.*}} = cir.vec.shuffle(%{{.*}}, %{{.*}} : !cir.vector<4 x !cir.float>) [#cir.int<0> : !s32i, #cir.int<0> : !s32i, #cir.int<4> : !s32i, #cir.int<4> : !s32i] : !cir.vector<4 x !cir.float>
+
+  // LLVM-LABEL: test_mm_shuffle_ps
+  // LLVM: shufflevector <4 x float> {{.*}}, <4 x float> {{.*}}, <4 x i32> <i32 0, i32 0, i32 4, i32 4>
+
+  // OGCG-LABEL: test_mm_shuffle_ps
+  // OGCG: shufflevector <4 x float> {{.*}}, <4 x float> {{.*}}, <4 x i32> <i32 0, i32 0, i32 4, i32 4>
+  return _mm_shuffle_ps(A, B, 0);
+}
diff --git a/clang/test/CIR/CodeGenBuiltins/X86/sse2-builtins.c b/clang/test/CIR/CodeGenBuiltins/X86/sse2-builtins.c
index f5e07cdc28ccd..4bb17e9d20bc6 100644
--- a/clang/test/CIR/CodeGenBuiltins/X86/sse2-builtins.c
+++ b/clang/test/CIR/CodeGenBuiltins/X86/sse2-builtins.c
@@ -8,8 +8,11 @@
 // RUN: %clang_cc1 -x c++ -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-unknown-linux -target-feature +sse2 -fno-signed-char -fclangir -emit-llvm -o %t.ll -Wall -Werror
 // RUN: FileCheck --check-prefixes=LLVM --input-file=%t.ll %s
 
-// RUN: %clang_cc1 -x c -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-unknown-linux -target-feature +sse -emit-llvm -o - -Wall -Werror | FileCheck %s -check-prefix=OGCG
-// RUN: %clang_cc1 -x c++ -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-unknown-linux -target-feature +sse -emit-llvm -o - -Wall -Werror | FileCheck %s -check-prefix=OGCG
+// RUN: %clang_cc1 -x c -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +sse2 -emit-llvm -o - -Wall -Werror | FileCheck %s --check-prefixes=OGCG
+// RUN: %clang_cc1 -x c -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +sse2 -fno-signed-char -emit-llvm -o - -Wall -Werror | FileCheck %s --check-prefixes=OGCG
+
+// RUN: %clang_cc1 -x c++ -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +sse2 -emit-llvm -o - -Wall -Werror | FileCheck %s --check-prefixes=OGCG
+// RUN: %clang_cc1 -x c++ -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +sse2 -fno-signed-char -emit-llvm -o - -Wall -Werror | FileCheck %s --check-prefixes=OGCG
 
 // This test mimics clang/test/CodeGen/X86/sse2-builtins.c, which eventually
 // CIR shall be able to support fully.
@@ -108,3 +111,51 @@ void test_mm_pause(void) {
   // LLVM: call void @llvm.x86.sse2.pause()
   // OGCG: call void @llvm.x86.sse2.pause()
 }
+
+__m128i test_mm_shufflelo_epi16(__m128i A) {
+  // CIR-LABEL: _mm_shufflelo_epi16
+  // CIR: %{{.*}} = cir.vec.shuffle(%{{.*}}, %{{.*}} : !cir.vector<8 x !s16i>) [#cir.int<0> : !s32i, #cir.int<0> : !s32i, #cir.int<0> : !s32i, #cir.int<0> : !s32i, #cir.int<4> : !s32i, #cir.int<5> : !s32i, #cir.int<6> : !s32i, #cir.int<7> : !s32i] : !cir.vector<8 x !s16i>
+
+  // LLVM-LABEL: test_mm_shufflelo_epi16
+  // LLVM: shufflevector <8 x i16> %{{.*}}, <8 x i16> poison, <8 x i32> <i32 0, i32 0, i32 0, i32 0, i32 4, i32 5, i32 6, i32 7>
+
+  // OGCG-LABEL: test_mm_shufflelo_epi16
+  // OGCG: shufflevector <8 x i16> %{{.*}}, <8 x i16> poison, <8 x i32> <i32 0, i32 0, i32 0, i32 0, i32 4, i32 5, i32 6, i32 7>
+  return _mm_shufflelo_epi16(A, 0);
+}
+
+__m128i test_mm_shufflehi_epi16(__m128i A) {
+  // CIR-LABEL: _mm_shufflehi_epi16
+  // CIR: %{{.*}} = cir.vec.shuffle(%{{.*}}, %{{.*}} : !cir.vector<8 x !s16i>) [#cir.int<0> : !s32i, #cir.int<1> : !s32i, #cir.int<2> : !s32i, #cir.int<3> : !s32i, #cir.int<4> : !s32i, #cir.int<4> : !s32i, #cir.int<4> : !s32i, #cir.int<4> : !s32i] : !cir.vector<8 x !s16i>
+
+  // LLVM-LABEL: test_mm_shufflehi_epi16
+  // LLVM: shufflevector <8 x i16> %{{.*}}, <8 x i16> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 4, i32 4, i32 4>
+
+  // OGCG-LABEL: test_mm_shufflehi_epi16
+  // OGCG: shufflevector <8 x i16> %{{.*}}, <8 x i16> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 4, i32 4, i32 4>
+  return _mm_shufflehi_epi16(A, 0);
+}
+
+__m128d test_mm_shuffle_pd(__m128d A, __m128d B) {
+  // CIR-LABEL: test_mm_shuffle_pd
+  // CIR: %{{.*}} = cir.vec.shuffle(%{{.*}}, %{{.*}} : !cir.vector<2 x !cir.double>) [#cir.int<1> : !s32i, #cir.int<2> : !s32i] : !cir.vector<2 x !cir.double>
+
+  // LLVM-LABEL: test_mm_shuffle_pd
+  // LLVM: shufflevector <2 x double> %{{.*}}, <2 x double> %{{.*}}, <2 x i32> <i32 1, i32 2>
+
+  // OGCG-LABEL: test_mm_shuffle_pd
+  // OGCG: shufflevector <2 x double> %{{.*}}, <2 x double> %{{.*}}, <2 x i32> <i32 1, i32 2>
+  return _mm_shuffle_pd(A, B, 1);
+}
+
+__m128i test_mm_shuffle_epi32(__m128i A) {
+	// CIR-LABEL: test_mm_shuffle_epi32
+	// CIR: %{{.*}} = cir.vec.shuffle(%{{.*}}, %{{.*}}: !cir.vector<4 x !s32i>) [#cir.int<2> : !s32i, #cir.int<3> : !s32i, #cir.int<0> : !s32i, #cir.int<1> : !s32i] : !cir.vector<4 x !s32i>
+
+    // LLVM-LABEL: test_mm_shuffle_epi32
+	// LLVM: shufflevector <4 x i32> %{{.*}}, <4 x i32> poison, <4 x i32> <i32 2, i32 3, i32 0, i32 1>
+
+	// OGCG-LABEL: test_mm_shuffle_epi32
+    // OGCG: shufflevector <4 x i32> %{{.*}}, <4 x i32> poison, <4 x i32> <i32 2, i32 3, i32 0, i32 1>
+    return _mm_shuffle_epi32(A, 0x4E);
+}
diff --git a/clang/test/CIR/CodeGenBuiltins/X86/xop-builtins.c b/clang/test/CIR/CodeGenBuiltins/X86/xop-builtins.c
new file mode 100644
index 0000000000000..0aaba7b46327d
--- /dev/null
+++ b/clang/test/CIR/CodeGenBuiltins/X86/xop-builtins.c
@@ -0,0 +1,92 @@
+// RUN: %clang_cc1 -x c -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +xop -emit-cir -o %t.cir
+// RUN: FileCheck --check-prefix=CIR --input-file=%t.cir %s
+// RUN: %clang_cc1 -x c -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +xop -fno-signed-char -emit-cir -o %t.cir
+// RUN: FileCheck --check-prefix=CIR --input-file=%t.cir %s
+
+// RUN: %clang_cc1 -x c -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +xop -fclangir -emit-llvm -o %t.ll
+// RUN: FileCheck --check-prefix=LLVM --input-file=%t.ll %s
+// RUN: %clang_cc1 -x c -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +xop -fno-signed-char -fclangir -emit-llvm -o %t.ll
+// RUN: FileCheck --check-prefix=LLVM --input-file=%t.ll %s
+
+// RUN: %clang_cc1 -x c++ -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +xop -emit-cir -o %t.cir
+// RUN: FileCheck --check-prefix=CIR --input-file=%t.cir %s
+// RUN: %clang_cc1 -x c++ -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +xop -fno-signed-char -emit-cir -o %t.cir
+// RUN: FileCheck --check-prefix=CIR --input-file=%t.cir %s
+
+// RUN: %clang_cc1 -x c++ -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +xop -fclangir -emit-llvm -o %t.ll
+// RUN: FileCheck --check-prefix=LLVM --input-file=%t.ll %s
+// RUN: %clang_cc1 -x c++ -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +xop -fno-signed-char -fclangir -emit-llvm -o %t.ll
+// RUN: FileCheck --check-prefix=LLVM --input-file=%t.ll %s
+
+// RUN: %clang_cc1 -x c -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +xop -emit-llvm -o - -Wall -Werror | FileCheck %s -check-prefix=OGCG
+// RUN: %clang_cc1 -x c -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +xop -fno-signed-char -emit-llvm -o - -Wall -Werror | FileCheck %s -check-prefix=OGCG
+// RUN: %clang_cc1 -x c++ -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +xop -emit-llvm -o - -Wall -Werror | FileCheck %s -check-prefix=OGCG
+// RUN: %clang_cc1 -x c++ -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +xop -fno-signed-char -emit-llvm -o - -Wall -Werror | FileCheck %s -check-prefix=OGCG
+
+#include <x86intrin.h>
+
+// This test mimics clang/test/CodeGen/X86/xop-builtins.c, which eventually
+// CIR shall be able to support fully.
+
+__m128i test_mm_roti_epi8(__m128i a) {
+  // CIR-LABEL: test_mm_roti_epi8
+  // CIR: cir.vec.splat %{{.*}} : !{{[us]}}8i, !cir.vector<16 x !{{[us]}}8i> 
+  // CIR: cir.call_llvm_intrinsic "fshl" %{{.*}} : (!cir.vector<16 x !{{[su]}}8i>, !cir.vector<16 x !{{[su]}}8i>, !cir.vector<16 x !{{[su]}}8i>) -> !cir.vector<16 x !{{[su]}}8i> 
+  
+  // LLVM-LABEL: test_mm_roti_epi8
+  // LLVM: %[[CASTED_VAR:.*]] = bitcast <2 x i64> %{{.*}} to <16 x i8>
+  // LLVM: call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %[[CASTED_VAR]], <16 x i8> %[[CASTED_VAR]], <16 x i8> splat (i8 1))
+  
+  // OGCG-LABEL: test_mm_roti_epi8
+  // OGCG: %[[CASTED_VAR:.*]] = bitcast <2 x i64> %{{.*}} to <16 x i8>
+  // OGCG: call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %[[CASTED_VAR]], <16 x i8> %[[CASTED_VAR]], <16 x i8> splat (i8 1))
+  return _mm_roti_epi8(a, 1);
+}
+
+__m128i test_mm_roti_epi16(__m128i a) {
+  // CIR-LABEL: test_mm_roti_epi16
+  // CIR: cir.cast integral %{{.*}} : !u8i -> !u16i
+  // CIR: cir.vec.splat %{{.*}} : !{{[us]}}16i, !cir.vector<8 x !u16i> 
+  // CIR: cir.call_llvm_intrinsic "fshl" %{{.*}} : (!cir.vector<8 x !{{[su]}}16i>, !cir.vector<8 x !{{[su]}}16i>, !cir.vector<8 x !u16i>) -> !cir.vector<8 x !{{[su]}}16i> 
+  
+  // LLVM-LABEL: test_mm_roti_epi16
+  // LLVM: %[[CASTED_VAR:.*]] = bitcast <2 x i64> %{{.*}} to <8 x i16>
+  // LLVM: call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %[[CASTED_VAR]], <8 x i16> %[[CASTED_VAR]], <8 x i16> splat (i16 50))
+  
+  // OGCG-LABEL: test_mm_roti_epi16
+  // OGCG: %[[CASTED_VAR:.*]] = bitcast <2 x i64> %{{.*}} to <8 x i16>
+  // OGCG: call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %[[CASTED_VAR]], <8 x i16> %[[CASTED_VAR]], <8 x i16> splat (i16 50))
+  return _mm_roti_epi16(a, 50);
+ }
+
+__m128i test_mm_roti_epi32(__m128i a) {
+  // CIR-LABEL: test_mm_roti_epi32
+  // CIR: cir.cast integral %{{.*}} : !u8i -> !u32i
+  // CIR: cir.vec.splat %{{.*}} : !{{[us]}}32i, !cir.vector<4 x !u32i> 
+  // CIR: cir.call_llvm_intrinsic "fshl" %{{.*}} : (!cir.vector<4 x !{{[su]}}32i>, !cir.vector<4 x !{{[su]}}32i>, !cir.vector<4 x !u32i>) -> !cir.vector<4 x !{{[su]}}32i> 
+  
+  // LLVM-LABEL: test_mm_roti_epi32
+  // LLVM: %[[CASTED_VAR:.*]] = bitcast <2 x i64> %{{.*}} to <4 x i32>
+  // LLVM: call <4 x i32> @llvm.fshl.v4i32(<4 x i32> %[[CASTED_VAR]], <4 x i32> %[[CASTED_VAR]], <4 x i32> splat (i32 226))
+  
+  // OGCG-LABEL: test_mm_roti_epi32
+  // OGCG: %[[CASTED_VAR:.*]] = bitcast <2 x i64> %{{.*}} to <4 x i32>
+  // OGCG: call <4 x i32> @llvm.fshl.v4i32(<4 x i32> %[[CASTED_VAR]], <4 x i32> %[[CASTED_VAR]], <4 x i32> splat (i32 226))
+  return _mm_roti_epi32(a, -30);
+ }
+
+__m128i test_mm_roti_epi64(__m128i a) {
+  // CIR-LABEL: test_mm_roti_epi64
+  // CIR: cir.cast integral %{{.*}} : !u8i -> !u64i
+  // CIR: cir.vec.splat %{{.*}} : !u64i, !cir.vector<2 x !u64i> 
+  // CIR: cir.call_llvm_intrinsic "fshl" %{{.*}} : (!cir.vector<2 x !{{[su]}}64i>, !cir.vector<2 x !{{[su]}}64i>, !cir.vector<2 x !u64i>) -> !cir.vector<2 x !s64i> 
+  
+  // LLVM-LABEL: test_mm_roti_epi64
+  // LLVM: %[[VAR:.*]] = load <2 x i64>, ptr %{{.*}}, align 16
+  // LLVM: call <2 x i64> @llvm.fshl.v2i64(<2 x i64> %[[VAR]], <2 x i64> %[[VAR]], <2 x i64> splat (i64 100))
+  
+  // OGCG-LABEL: test_mm_roti_epi64
+  // OGCG: %[[VAR:.*]] = load <2 x i64>, ptr %{{.*}}, align 16
+  // OGCG: call <2 x i64> @llvm.fshl.v2i64(<2 x i64> %[[VAR]], <2 x i64> %[[VAR]], <2 x i64> splat (i64 100))
+  return _mm_roti_epi64(a, 100);
+ }
diff --git a/clang/test/CIR/CodeGenBuiltins/builtin-constant-p.c b/clang/test/CIR/CodeGenBuiltins/builtin-constant-p.c
new file mode 100644
index 0000000000000..d684659216cba
--- /dev/null
+++ b/clang/test/CIR/CodeGenBuiltins/builtin-constant-p.c
@@ -0,0 +1,281 @@
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -emit-cir %s -o %t.cir
+// RUN: FileCheck --input-file=%t.cir %s -check-prefix=CIR
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -fclangir -emit-llvm %s -o %t-cir.ll
+// RUN: FileCheck --input-file=%t-cir.ll %s -check-prefix=LLVM
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -emit-llvm %s -o %t.ll
+// RUN: FileCheck --input-file=%t.ll %s -check-prefix=OGCG
+
+int a = 42;
+
+/* --- Compound literals */
+
+struct foo { int x, y; };
+
+int y;
+struct foo f = (struct foo){ __builtin_constant_p(y), 42 };
+
+// CIR: cir.global external @f = #cir.const_record<{#cir.int<0> : !s32i, #cir.int<42> : !s32i}> : !rec_foo
+// LLVM: @f = global %struct.foo { i32 0, i32 42 }
+// OGCG: @f = global %struct.foo { i32 0, i32 42 }
+
+struct foo test0(int expr) {
+  struct foo f = (struct foo){ __builtin_constant_p(expr), 42 };
+  return f;
+}
+
+// CIR: cir.func {{.*}} @test0(%[[ARG0:.*]]: !s32i {{.*}}) -> !rec_foo
+// CIR:   %[[EXPR_ADDR:.*]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["expr", init]
+// CIR:   cir.store %[[ARG0]], %[[EXPR_ADDR]]
+// CIR:   %[[EXPR:.*]] = cir.load{{.*}} %[[EXPR_ADDR]]
+// CIR:   %[[IS_CONSTANT:.*]] = cir.is_constant %[[EXPR]] : !s32i -> !cir.bool
+
+// LLVM: define{{.*}} %struct.foo @test0(i32 %[[ARG0:.*]])
+// LLVM:   %[[EXPR_ADDR:.*]] = alloca i32
+// LLVM:   store i32 %[[ARG0]], ptr %[[EXPR_ADDR]]
+// LLVM:   %[[EXPR:.*]] = load i32, ptr %[[EXPR_ADDR]]
+// LLVM:   %[[IS_CONSTANT:.*]] = call i1 @llvm.is.constant.i32(i32 %[[EXPR]])
+
+// OGCG: define{{.*}} i64 @test0(i32 {{.*}} %[[ARG0:.*]])
+// OGCG:   %[[EXPR_ADDR:.*]] = alloca i32
+// OGCG:   store i32 %[[ARG0]], ptr %[[EXPR_ADDR]]
+// OGCG:   %[[EXPR:.*]] = load i32, ptr %[[EXPR_ADDR]]
+// OGCG:   %[[IS_CONSTANT:.*]] = call i1 @llvm.is.constant.i32(i32 %[[EXPR]])
+
+/* --- Pointer types */
+
+int test1(void) {
+  return __builtin_constant_p(&a - 13);
+}
+
+// CIR: cir.func {{.*}} @test1() -> !s32i
+// CIR:   %[[TMP1:.*]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["__retval"]
+// CIR:   %[[ZERO:.*]] = cir.const #cir.int<0> : !s32i
+// CIR:   cir.store %[[ZERO]], %[[TMP1]] : !s32i, !cir.ptr<!s32i>
+// CIR:   %[[TMP2:.*]] = cir.load %[[TMP1]] : !cir.ptr<!s32i>, !s32i
+// CIR:   cir.return %[[TMP2]] : !s32i
+
+// LLVM: define{{.*}} i32 @test1()
+// LLVM:   %[[TMP1:.*]] = alloca i32
+// LLVM:   store i32 0, ptr %[[TMP1]]
+// LLVM:   %[[TMP2:.*]] = load i32, ptr %[[TMP1]]
+// LLVM:   ret i32 %[[TMP2]]
+
+// OGCG: define{{.*}} i32 @test1()
+// OGCG:   ret i32 0
+
+/* --- Aggregate types */
+
+int b[] = {1, 2, 3};
+
+int test2(void) {
+  return __builtin_constant_p(b);
+}
+
+// CIR: cir.func {{.*}} @test2() -> !s32i
+// CIR:   %[[TMP1:.*]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["__retval"]
+// CIR:   %[[ZERO:.*]] = cir.const #cir.int<0> : !s32i
+// CIR:   cir.store %[[ZERO]], %[[TMP1]] : !s32i, !cir.ptr<!s32i>
+// CIR:   %[[TMP2:.*]] = cir.load %[[TMP1]] : !cir.ptr<!s32i>, !s32i
+// CIR:   cir.return %[[TMP2]] : !s32i
+
+// LLVM: define{{.*}} i32 @test2()
+// LLVM:   %[[TMP1:.*]] = alloca i32
+// LLVM:   store i32 0, ptr %[[TMP1]]
+// LLVM:   %[[TMP2:.*]] = load i32, ptr %[[TMP1]]
+// LLVM:   ret i32 %[[TMP2]]
+
+// OGCG: define{{.*}} i32 @test2()
+// OGCG:   ret i32 0
+
+const char test3_c[] = {1, 2, 3, 0};
+
+int test3(void) {
+  return __builtin_constant_p(test3_c);
+}
+
+// CIR: cir.func {{.*}} @test3() -> !s32i
+// CIR:   %[[TMP1:.*]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["__retval"]
+// CIR:   %[[ZERO:.*]] = cir.const #cir.int<0> : !s32i
+// CIR:   cir.store %[[ZERO]], %[[TMP1]] : !s32i, !cir.ptr<!s32i>
+// CIR:   %[[TMP2:.*]] = cir.load %[[TMP1]] : !cir.ptr<!s32i>, !s32i
+// CIR:   cir.return %[[TMP2]] : !s32i
+
+// LLVM: define{{.*}} i32 @test3()
+// LLVM:   %[[TMP1:.*]] = alloca i32
+// LLVM:   store i32 0, ptr %[[TMP1]]
+// LLVM:   %[[TMP2:.*]] = load i32, ptr %[[TMP1]]
+// LLVM:   ret i32 %[[TMP2]]
+
+// OGCG: define{{.*}} i32 @test3()
+// OGCG:   ret i32 0
+
+inline char test4_i(const char *x) {
+  return x[1];
+}
+
+int test4(void) {
+  return __builtin_constant_p(test4_i(test3_c));
+}
+
+// CIR: cir.func {{.*}} @test4() -> !s32i
+// CIR:   %[[TMP1:.*]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["__retval"]
+// CIR:   %[[ZERO:.*]] = cir.const #cir.int<0> : !s32i
+// CIR:   cir.store %[[ZERO]], %[[TMP1]] : !s32i, !cir.ptr<!s32i>
+// CIR:   %[[TMP2:.*]] = cir.load %[[TMP1]] : !cir.ptr<!s32i>, !s32i
+// CIR:   cir.return %[[TMP2]] : !s32i
+
+// LLVM: define{{.*}} i32 @test4()
+// LLVM:   %[[TMP1:.*]] = alloca i32
+// LLVM:   store i32 0, ptr %[[TMP1]]
+// LLVM:   %[[TMP2:.*]] = load i32, ptr %[[TMP1]]
+// LLVM:   ret i32 %[[TMP2]]
+
+// OGCG: define{{.*}} i32 @test4()
+// OGCG:   ret i32 0
+
+/* --- Constant global variables */
+
+const int c = 42;
+
+int test5(void) {
+  return __builtin_constant_p(c);
+}
+
+// CIR: cir.func {{.*}} @test5() -> !s32i
+// CIR:   %[[TMP1:.*]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["__retval"]
+// CIR:   %[[ONE:.*]] = cir.const #cir.int<1> : !s32i
+// CIR:   cir.store %[[ONE]], %[[TMP1]] : !s32i, !cir.ptr<!s32i>
+// CIR:   %[[TMP2:.*]] = cir.load %[[TMP1]] : !cir.ptr<!s32i>, !s32i
+// CIR:   cir.return %[[TMP2]] : !s32i
+
+// LLVM: define{{.*}} i32 @test5()
+// LLVM:   %[[TMP1:.*]] = alloca i32
+// LLVM:   store i32 1, ptr %[[TMP1]]
+// LLVM:   %[[TMP2:.*]] = load i32, ptr %[[TMP1]]
+// LLVM:   ret i32 %[[TMP2]]
+
+// OGCG: define{{.*}} i32 @test5()
+// OGCG:   ret i32 1
+
+/* --- Array types */
+
+int arr[] = { 1, 2, 3 };
+
+int test6(void) {
+  return __builtin_constant_p(arr[2]);
+}
+
+// CIR: cir.func {{.*}} @test6() -> !s32i
+// CIR:   %[[TWO:.*]] = cir.const #cir.int<2> : !s32i
+// CIR:   %[[ARR:.*]] = cir.get_global @arr : !cir.ptr<!cir.array<!s32i x 3>>
+// CIR:   %[[ARR_PTR:.*]] = cir.cast array_to_ptrdecay %[[ARR]] : !cir.ptr<!cir.array<!s32i x 3>> -> !cir.ptr<!s32i>
+// CIR:   %[[ELE_PTR:.*]] = cir.ptr_stride %[[ARR_PTR]], %[[TWO]] : (!cir.ptr<!s32i>, !s32i) -> !cir.ptr<!s32i>
+// CIR:   %[[ELE:.*]] = cir.load{{.*}} %[[ELE_PTR]] : !cir.ptr<!s32i>, !s32i
+// CIR:   %[[IS_CONSTANT:.*]] = cir.is_constant %[[ELE]] : !s32i -> !cir.bool
+
+// LLVM: define {{.*}} i32 @test6()
+// LLVM:   %[[TMP1:.*]] = load i32, ptr getelementptr inbounds nuw (i8, ptr @arr, i64 8)
+// LLVM:   %[[TMP2:.*]] = call i1 @llvm.is.constant.i32(i32 %[[TMP1]])
+
+// OGCG: define {{.*}} i32 @test6()
+// OGCG:   %[[TMP1:.*]] = load i32, ptr getelementptr inbounds ([3 x i32], ptr @arr, i64 0, i64 2)
+// OGCG:   %[[TMP2:.*]] = call i1 @llvm.is.constant.i32(i32 %[[TMP1]])
+
+const int c_arr[] = { 1, 2, 3 };
+
+int test7(void) {
+  return __builtin_constant_p(c_arr[2]);
+}
+
+// CIR: cir.func {{.*}} @test7() -> !s32i
+// CIR:   %[[TMP1:.*]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["__retval"]
+// CIR:   %[[ONE:.*]] = cir.const #cir.int<1> : !s32i
+// CIR:   cir.store %[[ONE]], %[[TMP1]] : !s32i, !cir.ptr<!s32i>
+// CIR:   %[[TMP2:.*]] = cir.load %[[TMP1]] : !cir.ptr<!s32i>, !s32i
+// CIR:   cir.return %[[TMP2]] : !s32i
+
+// LLVM: define{{.*}} i32 @test7()
+// LLVM:   %[[TMP1:.*]] = alloca i32
+// LLVM:   store i32 1, ptr %[[TMP1]]
+// LLVM:   %[[TMP2:.*]] = load i32, ptr %[[TMP1]]
+// LLVM:   ret i32 %[[TMP2]]
+
+// OGCG: define{{.*}} i32 @test7()
+// OGCG:   ret i32 1
+
+int test8(void) {
+  return __builtin_constant_p(c_arr);
+}
+
+// CIR: cir.func {{.*}} @test8() -> !s32i
+// CIR:   %[[TMP1:.*]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["__retval"]
+// CIR:   %[[ZERO:.*]] = cir.const #cir.int<0> : !s32i
+// CIR:   cir.store %[[ZERO]], %[[TMP1]] : !s32i, !cir.ptr<!s32i>
+// CIR:   %[[TMP2:.*]] = cir.load %[[TMP1]] : !cir.ptr<!s32i>, !s32i
+// CIR:   cir.return %[[TMP2]] : !s32i
+
+// LLVM: define{{.*}} i32 @test8()
+// LLVM:   %[[TMP1:.*]] = alloca i32
+// LLVM:   store i32 0, ptr %[[TMP1]]
+// LLVM:   %[[TMP2:.*]] = load i32, ptr %[[TMP1]]
+// LLVM:   ret i32 %[[TMP2]]
+
+// OGCG: define{{.*}} i32 @test8()
+// OGCG:   ret i32 0
+
+/* --- Function pointers */
+
+int test9(void) {
+  return __builtin_constant_p(&test9);
+}
+
+// CIR: cir.func {{.*}} @test9() -> !s32i
+// CIR:   %[[TMP1:.*]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["__retval"]
+// CIR:   %[[ZERO:.*]] = cir.const #cir.int<0> : !s32i
+// CIR:   cir.store %[[ZERO]], %[[TMP1]] : !s32i, !cir.ptr<!s32i>
+// CIR:   %[[TMP2:.*]] = cir.load %[[TMP1]] : !cir.ptr<!s32i>, !s32i
+// CIR:   cir.return %[[TMP2]] : !s32i
+
+// LLVM: define{{.*}} i32 @test9()
+// LLVM:   %[[TMP1:.*]] = alloca i32
+// LLVM:   store i32 0, ptr %[[TMP1]]
+// LLVM:   %[[TMP2:.*]] = load i32, ptr %[[TMP1]]
+// LLVM:   ret i32 %[[TMP2]]
+
+// OGCG: define{{.*}} i32 @test9()
+// OGCG:   ret i32 0
+
+int test10(void) {
+  return __builtin_constant_p(&test10 != 0);
+}
+
+// CIR: cir.func {{.*}} @test10() -> !s32i
+// CIR:   %[[TMP1:.*]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["__retval"]
+// CIR:   %[[ONE:.*]] = cir.const #cir.int<1> : !s32i
+// CIR:   cir.store %[[ONE]], %[[TMP1]] : !s32i, !cir.ptr<!s32i>
+// CIR:   %[[TMP2:.*]] = cir.load %[[TMP1]] : !cir.ptr<!s32i>, !s32i
+// CIR:   cir.return %[[TMP2]] : !s32i
+
+// LLVM: define{{.*}} i32 @test10()
+// LLVM:   %[[TMP1:.*]] = alloca i32
+// LLVM:   store i32 1, ptr %[[TMP1]]
+// LLVM:   %[[TMP2:.*]] = load i32, ptr %[[TMP1]]
+// LLVM:   ret i32 %[[TMP2]]
+
+// OGCG: define{{.*}} i32 @test10()
+// OGCG:   ret i32 1
+
+int test11_f(void);
+void test11(void) {
+  int a, b;
+  (void)__builtin_constant_p((a = b, test11_f()));
+}
+
+// CIR: cir.func {{.*}} @test11()
+// CIR-NOT: call {{.*}}test11_f
+
+// LLVM: define{{.*}} void @test11()
+// LLVM-NOT: call {{.*}}test11_f
+
+// OGCG: define{{.*}} void @test11()
+// OGCG-NOT: call {{.*}}test11_f
diff --git a/clang/test/CIR/CodeGenOpenACC/combined-firstprivate-clause.cpp b/clang/test/CIR/CodeGenOpenACC/combined-firstprivate-clause.cpp
index 5ee51aaa2446e..ba3c53b6bb03e 100644
--- a/clang/test/CIR/CodeGenOpenACC/combined-firstprivate-clause.cpp
+++ b/clang/test/CIR/CodeGenOpenACC/combined-firstprivate-clause.cpp
@@ -43,7 +43,7 @@ struct HasDtor {
 // CHECK-NEXT: acc.yield
 // CHECK-NEXT: } copy {
 // CHECK-NEXT: ^bb0(%[[ARG_FROM:.*]]: !cir.ptr<!rec_NoCopyConstruct> {{.*}}, %[[ARG_TO:.*]]: !cir.ptr<!rec_NoCopyConstruct> {{.*}}):
-// CHECK-NEXT: cir.call @_ZN15NoCopyConstructC1ERKS_(%[[ARG_TO]], %[[ARG_FROM]]) nothrow : (!cir.ptr<!rec_NoCopyConstruct>, !cir.ptr<!rec_NoCopyConstruct>) -> ()
+// CHECK-NEXT: cir.copy %[[ARG_FROM]] to %[[ARG_TO]] : !cir.ptr<!rec_NoCopyConstruct>
 // CHECK-NEXT: acc.yield
 // CHECK-NEXT: }
 //
@@ -63,7 +63,7 @@ struct HasDtor {
 // CHECK-NEXT: acc.yield
 // CHECK-NEXT: } copy {
 // CHECK-NEXT: ^bb0(%[[ARG_FROM:.*]]: !cir.ptr<!rec_NonDefaultCtor> {{.*}}, %[[ARG_TO:.*]]: !cir.ptr<!rec_NonDefaultCtor> {{.*}}):
-// CHECK-NEXT: cir.call @_ZN14NonDefaultCtorC1ERKS_(%[[ARG_TO]], %[[ARG_FROM]]) nothrow : (!cir.ptr<!rec_NonDefaultCtor>, !cir.ptr<!rec_NonDefaultCtor>) -> ()
+// CHECK-NEXT: cir.copy %[[ARG_FROM]] to %[[ARG_TO]] : !cir.ptr<!rec_NonDefaultCtor>
 // CHECK-NEXT: acc.yield
 // CHECK-NEXT: }
 //
@@ -73,7 +73,7 @@ struct HasDtor {
 // CHECK-NEXT: acc.yield
 // CHECK-NEXT: } copy {
 // CHECK-NEXT: ^bb0(%[[ARG_FROM:.*]]: !cir.ptr<!rec_HasDtor> {{.*}}, %[[ARG_TO:.*]]: !cir.ptr<!rec_HasDtor> {{.*}}):
-// CHECK-NEXT: cir.call @_ZN7HasDtorC1ERKS_(%[[ARG_TO]], %[[ARG_FROM]]) nothrow : (!cir.ptr<!rec_HasDtor>, !cir.ptr<!rec_HasDtor>) -> ()
+// CHECK-NEXT: cir.copy %[[ARG_FROM]] to %[[ARG_TO]] : !cir.ptr<!rec_HasDtor>
 // CHECK-NEXT: acc.yield
 // CHECK-NEXT: } destroy {
 // CHECK-NEXT: ^bb0(%[[ORIG:.*]]: !cir.ptr<!rec_HasDtor> {{.*}}, %[[ARG:.*]]: !cir.ptr<!rec_HasDtor> {{.*}}):
@@ -176,7 +176,7 @@ struct HasDtor {
 // CHECK-NEXT: %[[STRIDE_FROM:.*]] = cir.ptr_stride %[[DECAY_FROM]], %[[ITR_LOAD]] : (!cir.ptr<!rec_NoCopyConstruct>, !u64i) -> !cir.ptr<!rec_NoCopyConstruct>
 // CHECK-NEXT: %[[DECAY_TO:.*]] = cir.cast array_to_ptrdecay %[[ARG_TO]] : !cir.ptr<!cir.array<!rec_NoCopyConstruct x 5>> -> !cir.ptr<!rec_NoCopyConstruct>
 // CHECK-NEXT: %[[STRIDE_TO:.*]] = cir.ptr_stride %[[DECAY_TO]], %[[ITR_LOAD]] : (!cir.ptr<!rec_NoCopyConstruct>, !u64i) -> !cir.ptr<!rec_NoCopyConstruct>
-// CHECK-NEXT: cir.call @_ZN15NoCopyConstructC1ERKS_(%[[STRIDE_TO]], %[[STRIDE_FROM]]) nothrow : (!cir.ptr<!rec_NoCopyConstruct>, !cir.ptr<!rec_NoCopyConstruct>) -> ()
+// CHECK-NEXT: cir.copy %[[STRIDE_FROM]] to %[[STRIDE_TO]] : !cir.ptr<!rec_NoCopyConstruct>
 // CHECK-NEXT: cir.yield
 // CHECK-NEXT: } step {
 // CHECK-NEXT: %[[ITR_LOAD]] = cir.load %[[ITR]] : !cir.ptr<!u64i>, !u64i
@@ -246,7 +246,7 @@ struct HasDtor {
 // CHECK-NEXT: %[[STRIDE_FROM:.*]] = cir.ptr_stride %[[DECAY_FROM]], %[[ITR_LOAD]] : (!cir.ptr<!rec_NonDefaultCtor>, !u64i) -> !cir.ptr<!rec_NonDefaultCtor>
 // CHECK-NEXT: %[[DECAY_TO:.*]] = cir.cast array_to_ptrdecay %[[ARG_TO]] : !cir.ptr<!cir.array<!rec_NonDefaultCtor x 5>> -> !cir.ptr<!rec_NonDefaultCtor>
 // CHECK-NEXT: %[[STRIDE_TO:.*]] = cir.ptr_stride %[[DECAY_TO]], %[[ITR_LOAD]] : (!cir.ptr<!rec_NonDefaultCtor>, !u64i) -> !cir.ptr<!rec_NonDefaultCtor>
-// CHECK-NEXT: cir.call @_ZN14NonDefaultCtorC1ERKS_(%[[STRIDE_TO]], %[[STRIDE_FROM]]) nothrow : (!cir.ptr<!rec_NonDefaultCtor>, !cir.ptr<!rec_NonDefaultCtor>) -> ()
+// CHECK-NEXT: cir.copy %[[STRIDE_FROM]] to %[[STRIDE_TO]] : !cir.ptr<!rec_NonDefaultCtor>
 // CHECK-NEXT: cir.yield
 // CHECK-NEXT: } step {
 // CHECK-NEXT: %[[ITR_LOAD]] = cir.load %[[ITR]] : !cir.ptr<!u64i>, !u64i
@@ -281,7 +281,7 @@ struct HasDtor {
 // CHECK-NEXT: %[[STRIDE_FROM:.*]] = cir.ptr_stride %[[DECAY_FROM]], %[[ITR_LOAD]] : (!cir.ptr<!rec_HasDtor>, !u64i) -> !cir.ptr<!rec_HasDtor>
 // CHECK-NEXT: %[[DECAY_TO:.*]] = cir.cast array_to_ptrdecay %[[ARG_TO]] : !cir.ptr<!cir.array<!rec_HasDtor x 5>> -> !cir.ptr<!rec_HasDtor>
 // CHECK-NEXT: %[[STRIDE_TO:.*]] = cir.ptr_stride %[[DECAY_TO]], %[[ITR_LOAD]] : (!cir.ptr<!rec_HasDtor>, !u64i) -> !cir.ptr<!rec_HasDtor>
-// CHECK-NEXT: cir.call @_ZN7HasDtorC1ERKS_(%[[STRIDE_TO]], %[[STRIDE_FROM]]) nothrow : (!cir.ptr<!rec_HasDtor>, !cir.ptr<!rec_HasDtor>) -> ()
+// CHECK-NEXT: cir.copy %[[STRIDE_FROM]] to %[[STRIDE_TO]] : !cir.ptr<!rec_HasDtor>
 // CHECK-NEXT: cir.yield
 // CHECK-NEXT: } step {
 // CHECK-NEXT: %[[ITR_LOAD]] = cir.load %[[ITR]] : !cir.ptr<!u64i>, !u64i
diff --git a/clang/test/CIR/CodeGenOpenACC/compute-firstprivate-clause-templates.cpp b/clang/test/CIR/CodeGenOpenACC/compute-firstprivate-clause-templates.cpp
index 741b7dc4cb315..52340f2c3efbf 100644
--- a/clang/test/CIR/CodeGenOpenACC/compute-firstprivate-clause-templates.cpp
+++ b/clang/test/CIR/CodeGenOpenACC/compute-firstprivate-clause-templates.cpp
@@ -30,7 +30,7 @@ struct HasDtor {
 // CHECK-NEXT: acc.yield
 // CHECK-NEXT: } copy {
 // CHECK-NEXT: ^bb0(%[[ARG_FROM:.*]]: !cir.ptr<!rec_NonDefaultCtor> {{.*}}, %[[ARG_TO:.*]]: !cir.ptr<!rec_NonDefaultCtor> {{.*}}):
-// CHECK-NEXT: cir.call @_ZN14NonDefaultCtorC1ERKS_(%[[ARG_TO]], %[[ARG_FROM]]) nothrow : (!cir.ptr<!rec_NonDefaultCtor>, !cir.ptr<!rec_NonDefaultCtor>) -> ()
+// CHECK-NEXT: cir.copy %[[ARG_FROM]] to %[[ARG_TO]] : !cir.ptr<!rec_NonDefaultCtor>
 // CHECK-NEXT: acc.yield
 // CHECK-NEXT: }
 //
@@ -40,7 +40,7 @@ struct HasDtor {
 // CHECK-NEXT: acc.yield
 // CHECK-NEXT: } copy {
 // CHECK-NEXT: ^bb0(%[[ARG_FROM:.*]]: !cir.ptr<!rec_HasDtor> {{.*}}, %[[ARG_TO:.*]]: !cir.ptr<!rec_HasDtor> {{.*}}):
-// CHECK-NEXT: cir.call @_ZN7HasDtorC1ERKS_(%[[ARG_TO]], %[[ARG_FROM]]) nothrow : (!cir.ptr<!rec_HasDtor>, !cir.ptr<!rec_HasDtor>) -> ()
+// CHECK-NEXT: cir.copy %[[ARG_FROM]] to %[[ARG_TO]] : !cir.ptr<!rec_HasDtor>
 // CHECK-NEXT: acc.yield
 // CHECK-NEXT: } destroy {
 // CHECK-NEXT: ^bb0(%[[ORIG:.*]]: !cir.ptr<!rec_HasDtor> {{.*}}, %[[ARG:.*]]: !cir.ptr<!rec_HasDtor> {{.*}}):
diff --git a/clang/test/CIR/CodeGenOpenACC/compute-firstprivate-clause.cpp b/clang/test/CIR/CodeGenOpenACC/compute-firstprivate-clause.cpp
index b98eb0edba7c6..8f723c16175d3 100644
--- a/clang/test/CIR/CodeGenOpenACC/compute-firstprivate-clause.cpp
+++ b/clang/test/CIR/CodeGenOpenACC/compute-firstprivate-clause.cpp
@@ -1,4 +1,5 @@
-// RUN: %clang_cc1 -fopenacc -triple x86_64-linux-gnu -Wno-openacc-self-if-potential-conflict -emit-cir -fclangir -triple x86_64-linux-pc %s -o - | FileCheck %s
+// RUN: %clang_cc1 -fopenacc -triple x86_64-linux-gnu -Wno-openacc-self-if-potential-conflict -emit-cir -fclangir -triple x86_64-linux-pc %s -o %t.ll
+// RUN: FileCheck --input-file=%t.ll %s
 
 struct NoCopyConstruct {};
 
@@ -43,7 +44,7 @@ struct HasDtor {
 // CHECK-NEXT: acc.yield
 // CHECK-NEXT: } copy {
 // CHECK-NEXT: ^bb0(%[[ARG_FROM:.*]]: !cir.ptr<!rec_NoCopyConstruct> {{.*}}, %[[ARG_TO:.*]]: !cir.ptr<!rec_NoCopyConstruct> {{.*}}):
-// CHECK-NEXT: cir.call @_ZN15NoCopyConstructC1ERKS_(%[[ARG_TO]], %[[ARG_FROM]]) nothrow : (!cir.ptr<!rec_NoCopyConstruct>, !cir.ptr<!rec_NoCopyConstruct>) -> ()
+// CHECK-NEXT: cir.copy %[[ARG_FROM]] to %[[ARG_TO]] : !cir.ptr<!rec_NoCopyConstruct>
 // CHECK-NEXT: acc.yield
 // CHECK-NEXT: }
 //
@@ -63,7 +64,7 @@ struct HasDtor {
 // CHECK-NEXT: acc.yield
 // CHECK-NEXT: } copy {
 // CHECK-NEXT: ^bb0(%[[ARG_FROM:.*]]: !cir.ptr<!rec_NonDefaultCtor> {{.*}}, %[[ARG_TO:.*]]: !cir.ptr<!rec_NonDefaultCtor> {{.*}}):
-// CHECK-NEXT: cir.call @_ZN14NonDefaultCtorC1ERKS_(%[[ARG_TO]], %[[ARG_FROM]]) nothrow : (!cir.ptr<!rec_NonDefaultCtor>, !cir.ptr<!rec_NonDefaultCtor>) -> ()
+// CHECK-NEXT: cir.copy %[[ARG_FROM]] to %[[ARG_TO]] : !cir.ptr<!rec_NonDefaultCtor>
 // CHECK-NEXT: acc.yield
 // CHECK-NEXT: }
 //
@@ -73,7 +74,7 @@ struct HasDtor {
 // CHECK-NEXT: acc.yield
 // CHECK-NEXT: } copy {
 // CHECK-NEXT: ^bb0(%[[ARG_FROM:.*]]: !cir.ptr<!rec_HasDtor> {{.*}}, %[[ARG_TO:.*]]: !cir.ptr<!rec_HasDtor> {{.*}}):
-// CHECK-NEXT: cir.call @_ZN7HasDtorC1ERKS_(%[[ARG_TO]], %[[ARG_FROM]]) nothrow : (!cir.ptr<!rec_HasDtor>, !cir.ptr<!rec_HasDtor>) -> ()
+// CHECK-NEXT: cir.copy %[[ARG_FROM]] to %[[ARG_TO]] : !cir.ptr<!rec_HasDtor>
 // CHECK-NEXT: acc.yield
 // CHECK-NEXT: } destroy {
 // CHECK-NEXT: ^bb0(%[[ORIG:.*]]: !cir.ptr<!rec_HasDtor> {{.*}}, %[[ARG:.*]]: !cir.ptr<!rec_HasDtor> {{.*}}):
@@ -176,7 +177,7 @@ struct HasDtor {
 // CHECK-NEXT: %[[STRIDE_FROM:.*]] = cir.ptr_stride %[[DECAY_FROM]], %[[ITR_LOAD]] : (!cir.ptr<!rec_NoCopyConstruct>, !u64i) -> !cir.ptr<!rec_NoCopyConstruct>
 // CHECK-NEXT: %[[DECAY_TO:.*]] = cir.cast array_to_ptrdecay %[[ARG_TO]] : !cir.ptr<!cir.array<!rec_NoCopyConstruct x 5>> -> !cir.ptr<!rec_NoCopyConstruct>
 // CHECK-NEXT: %[[STRIDE_TO:.*]] = cir.ptr_stride %[[DECAY_TO]], %[[ITR_LOAD]] : (!cir.ptr<!rec_NoCopyConstruct>, !u64i) -> !cir.ptr<!rec_NoCopyConstruct>
-// CHECK-NEXT: cir.call @_ZN15NoCopyConstructC1ERKS_(%[[STRIDE_TO]], %[[STRIDE_FROM]]) nothrow : (!cir.ptr<!rec_NoCopyConstruct>, !cir.ptr<!rec_NoCopyConstruct>) -> ()
+// CHECK-NEXT: cir.copy %[[STRIDE_FROM]] to %[[STRIDE_TO]] : !cir.ptr<!rec_NoCopyConstruct>
 // CHECK-NEXT: cir.yield
 // CHECK-NEXT: } step {
 // CHECK-NEXT: %[[ITR_LOAD]] = cir.load %[[ITR]] : !cir.ptr<!u64i>, !u64i
@@ -246,7 +247,7 @@ struct HasDtor {
 // CHECK-NEXT: %[[STRIDE_FROM:.*]] = cir.ptr_stride %[[DECAY_FROM]], %[[ITR_LOAD]] : (!cir.ptr<!rec_NonDefaultCtor>, !u64i) -> !cir.ptr<!rec_NonDefaultCtor>
 // CHECK-NEXT: %[[DECAY_TO:.*]] = cir.cast array_to_ptrdecay %[[ARG_TO]] : !cir.ptr<!cir.array<!rec_NonDefaultCtor x 5>> -> !cir.ptr<!rec_NonDefaultCtor>
 // CHECK-NEXT: %[[STRIDE_TO:.*]] = cir.ptr_stride %[[DECAY_TO]], %[[ITR_LOAD]] : (!cir.ptr<!rec_NonDefaultCtor>, !u64i) -> !cir.ptr<!rec_NonDefaultCtor>
-// CHECK-NEXT: cir.call @_ZN14NonDefaultCtorC1ERKS_(%[[STRIDE_TO]], %[[STRIDE_FROM]]) nothrow : (!cir.ptr<!rec_NonDefaultCtor>, !cir.ptr<!rec_NonDefaultCtor>) -> ()
+// CHECK-NEXT: cir.copy %[[STRIDE_FROM]] to %[[STRIDE_TO]] : !cir.ptr<!rec_NonDefaultCtor>
 // CHECK-NEXT: cir.yield
 // CHECK-NEXT: } step {
 // CHECK-NEXT: %[[ITR_LOAD]] = cir.load %[[ITR]] : !cir.ptr<!u64i>, !u64i
@@ -281,7 +282,7 @@ struct HasDtor {
 // CHECK-NEXT: %[[STRIDE_FROM:.*]] = cir.ptr_stride %[[DECAY_FROM]], %[[ITR_LOAD]] : (!cir.ptr<!rec_HasDtor>, !u64i) -> !cir.ptr<!rec_HasDtor>
 // CHECK-NEXT: %[[DECAY_TO:.*]] = cir.cast array_to_ptrdecay %[[ARG_TO]] : !cir.ptr<!cir.array<!rec_HasDtor x 5>> -> !cir.ptr<!rec_HasDtor>
 // CHECK-NEXT: %[[STRIDE_TO:.*]] = cir.ptr_stride %[[DECAY_TO]], %[[ITR_LOAD]] : (!cir.ptr<!rec_HasDtor>, !u64i) -> !cir.ptr<!rec_HasDtor>
-// CHECK-NEXT: cir.call @_ZN7HasDtorC1ERKS_(%[[STRIDE_TO]], %[[STRIDE_FROM]]) nothrow : (!cir.ptr<!rec_HasDtor>, !cir.ptr<!rec_HasDtor>) -> ()
+// CHECK-NEXT: cir.copy %[[STRIDE_FROM]] to %[[STRIDE_TO]] : !cir.ptr<!rec_HasDtor>
 // CHECK-NEXT: cir.yield
 // CHECK-NEXT: } step {
 // CHECK-NEXT: %[[ITR_LOAD]] = cir.load %[[ITR]] : !cir.ptr<!u64i>, !u64i
diff --git a/clang/test/CIR/CodeGenOpenACC/firstprivate-clause-recipes.cpp b/clang/test/CIR/CodeGenOpenACC/firstprivate-clause-recipes.cpp
index 95168812316ea..3a13a81b48828 100644
--- a/clang/test/CIR/CodeGenOpenACC/firstprivate-clause-recipes.cpp
+++ b/clang/test/CIR/CodeGenOpenACC/firstprivate-clause-recipes.cpp
@@ -75,7 +75,7 @@ void do_things(unsigned A, unsigned B) {
 // CHECK-NEXT: %[[BOUND1_STRIDE_FROM:.*]] = cir.ptr_stride %[[BOUND2_STRIDE_DECAY_FROM]], %[[ITR1_LOAD]] : (!cir.ptr<!rec_NoOps>, !u64i) -> !cir.ptr<!rec_NoOps>
 // CHECK-NEXT: %[[BOUND2_STRIDE_DECAY_TO:.*]] = cir.cast array_to_ptrdecay %[[BOUND2_STRIDE_TO]] : !cir.ptr<!cir.array<!rec_NoOps x 5>> -> !cir.ptr<!rec_NoOps>
 // CHECK-NEXT: %[[BOUND1_STRIDE_TO:.*]] = cir.ptr_stride %[[BOUND2_STRIDE_DECAY_TO]], %[[ITR1_LOAD]] : (!cir.ptr<!rec_NoOps>, !u64i) -> !cir.ptr<!rec_NoOps>
-// CHECK-NEXT: cir.call @_ZN5NoOpsC1ERKS_(%[[BOUND1_STRIDE_TO]], %[[BOUND1_STRIDE_FROM]]) nothrow : (!cir.ptr<!rec_NoOps>, !cir.ptr<!rec_NoOps>) -> ()
+// CHECK-NEXT: cir.copy %[[BOUND1_STRIDE_FROM]] to %[[BOUND1_STRIDE_TO]] : !cir.ptr<!rec_NoOps>
 // CHECK-NEXT: cir.yield
 // CHECK-NEXT: } step {
 // CHECK-NEXT: %[[ITR1_LOAD]] = cir.load %[[ITR1]] : !cir.ptr<!u64i>, !u64i
@@ -342,7 +342,7 @@ void do_things(unsigned A, unsigned B) {
 // CHECK-NEXT: %[[STRIDE_FROM:.*]] = cir.ptr_stride %[[BOUND2_STRIDE_LOAD_FROM]], %[[ITR1_LOAD]] : (!cir.ptr<!rec_NoOps>, !u64i) -> !cir.ptr<!rec_NoOps>
 // CHECK-NEXT: %[[BOUND2_STRIDE_LOAD_TO:.*]] = cir.load %[[BOUND2_STRIDE_TO]] : !cir.ptr<!cir.ptr<!rec_NoOps>>, !cir.ptr<!rec_NoOps>
 // CHECK-NEXT: %[[STRIDE_TO:.*]] = cir.ptr_stride %[[BOUND2_STRIDE_LOAD_TO]], %[[ITR1_LOAD]] : (!cir.ptr<!rec_NoOps>, !u64i) -> !cir.ptr<!rec_NoOps>
-// CHECK-NEXT: cir.call @_ZN5NoOpsC1ERKS_(%[[BOUND1_STRIDE_TO]], %[[BOUND1_STRIDE_FROM]]) nothrow : (!cir.ptr<!rec_NoOps>, !cir.ptr<!rec_NoOps>) -> ()
+// CHECK-NEXT: cir.copy %[[STRIDE_FROM]] to %[[STRIDE_TO]] : !cir.ptr<!rec_NoOps>
 // CHECK-NEXT: cir.yield
 // CHECK-NEXT: } step {
 // CHECK-NEXT: %[[ITR1_LOAD]] = cir.load %[[ITR1]] : !cir.ptr<!u64i>, !u64i
@@ -580,7 +580,7 @@ void do_things(unsigned A, unsigned B) {
 // CHECK-NEXT: %[[STRIDE_FROM:.*]] = cir.ptr_stride %[[BOUND2_STRIDE_LOAD_FROM]], %[[ITR1_LOAD]] : (!cir.ptr<!rec_CtorDtor>, !u64i) -> !cir.ptr<!rec_CtorDtor>
 // CHECK-NEXT: %[[BOUND2_STRIDE_LOAD_TO:.*]] = cir.load %[[BOUND2_STRIDE_TO]] : !cir.ptr<!cir.ptr<!rec_CtorDtor>>, !cir.ptr<!rec_CtorDtor>
 // CHECK-NEXT: %[[STRIDE_TO:.*]] = cir.ptr_stride %[[BOUND2_STRIDE_LOAD_TO]], %[[ITR1_LOAD]] : (!cir.ptr<!rec_CtorDtor>, !u64i) -> !cir.ptr<!rec_CtorDtor>
-// CHECK-NEXT: cir.call @_ZN8CtorDtorC1ERKS_(%[[BOUND1_STRIDE_TO]], %[[BOUND1_STRIDE_FROM]]) nothrow : (!cir.ptr<!rec_CtorDtor>, !cir.ptr<!rec_CtorDtor>) -> ()
+// CHECK-NEXT: cir.copy %[[STRIDE_FROM]] to %[[STRIDE_TO]] : !cir.ptr<!rec_CtorDtor>
 // CHECK-NEXT: cir.yield
 // CHECK-NEXT: } step {
 // CHECK-NEXT: %[[ITR1_LOAD]] = cir.load %[[ITR1]] : !cir.ptr<!u64i>, !u64i
diff --git a/clang/test/CIR/CodeGenOpenACC/openacc-not-implemented-global.cpp b/clang/test/CIR/CodeGenOpenACC/openacc-not-implemented-global.cpp
deleted file mode 100644
index a5e4694c6f5e6..0000000000000
--- a/clang/test/CIR/CodeGenOpenACC/openacc-not-implemented-global.cpp
+++ /dev/null
@@ -1,6 +0,0 @@
-// RUN: %clang_cc1 -std=c++17 -triple x86_64-unknown-linux-gnu -fopenacc -fclangir -emit-cir %s -o %t.cir -verify
-// RUN: %clang_cc1 -std=c++17 -triple x86_64-unknown-linux-gnu -fopenacc -fclangir -emit-llvm %s -o %t-cir.ll -verify
-
-void foo() {}
-// expected-error@+1{{ClangIR code gen Not Yet Implemented: OpenACC Global Routine Construct}}
-#pragma acc routine(foo) seq
diff --git a/clang/test/CIR/CodeGenOpenACC/routine-anon-ns.cpp b/clang/test/CIR/CodeGenOpenACC/routine-anon-ns.cpp
new file mode 100644
index 0000000000000..7c0a2edee5257
--- /dev/null
+++ b/clang/test/CIR/CodeGenOpenACC/routine-anon-ns.cpp
@@ -0,0 +1,27 @@
+// RUN: %clang_cc1 -fopenacc -Wno-openacc-self-if-potential-conflict -emit-cir -fclangir %s -o - | FileCheck %s
+
+namespace {
+#pragma acc routine seq
+  void NSFunc1(){}
+#pragma acc routine seq
+  auto Lambda1 = [](){};
+
+  auto Lambda2 = [](){};
+} // namespace 
+
+#pragma acc routine(NSFunc1) seq
+#pragma acc routine(Lambda2) seq
+void force_emit() {
+  NSFunc1();
+  Lambda1();
+  Lambda2();
+}
+
+// CHECK: cir.func{{.*}} @[[F1_NAME:[^\(]*]]({{.*}}){{.*}} attributes {acc.routine_info = #acc.routine_info<[@[[F1_R_NAME:.*]], @[[F1_R2_NAME:.*]]]>}
+// CHECK: cir.func {{.*}}lambda{{.*}} @[[L1_NAME:[^\(]*]]({{.*}}){{.*}} attributes {acc.routine_info = #acc.routine_info<[@[[L1_R_NAME:.*]]]>}
+// CHECK: cir.func {{.*}}lambda{{.*}} @[[L2_NAME:[^\(]*]]({{.*}}){{.*}} attributes {acc.routine_info = #acc.routine_info<[@[[L2_R_NAME:.*]]]>}
+//
+// CHECK: acc.routine @[[F1_R_NAME]] func(@[[F1_NAME]]) seq
+// CHECK: acc.routine @[[L1_R_NAME]] func(@[[L1_NAME]]) seq
+// CHECK: acc.routine @[[F1_R2_NAME]] func(@[[F1_NAME]]) seq
+// CHECK: acc.routine @[[L2_R_NAME]] func(@[[L2_NAME]]) seq
diff --git a/clang/test/CIR/CodeGenOpenACC/routine-clauses.cpp b/clang/test/CIR/CodeGenOpenACC/routine-clauses.cpp
new file mode 100644
index 0000000000000..81437e7e02ab1
--- /dev/null
+++ b/clang/test/CIR/CodeGenOpenACC/routine-clauses.cpp
@@ -0,0 +1,38 @@
+// RUN: %clang_cc1 -fopenacc -Wno-openacc-self-if-potential-conflict -emit-cir -fclangir %s -o - | FileCheck %s
+
+#pragma acc routine seq nohost
+void Func1() {}
+
+void Func2() {}
+#pragma acc routine(Func2) seq
+
+#pragma acc routine worker
+void Func3() {}
+
+void Func4() {}
+#pragma acc routine(Func4) worker nohost
+
+#pragma acc routine nohost vector
+void Func5() {}
+
+void Func6() {}
+#pragma acc routine(Func6) nohost vector
+
+// CHECK: cir.func{{.*}} @[[F1_NAME:.*Func1[^\(]*]]({{.*}}){{.*}} attributes {acc.routine_info = #acc.routine_info<[@[[F1_R_NAME:.*]]]>}
+// CHECK: acc.routine @[[F1_R_NAME]] func(@[[F1_NAME]]) seq nohost
+
+// CHECK: cir.func{{.*}} @[[F2_NAME:.*Func2[^\(]*]]({{.*}}){{.*}} attributes {acc.routine_info = #acc.routine_info<[@[[F2_R_NAME:.*]]]>}
+
+// CHECK: cir.func{{.*}} @[[F3_NAME:.*Func3[^\(]*]]({{.*}}){{.*}} attributes {acc.routine_info = #acc.routine_info<[@[[F3_R_NAME:.*]]]>}
+// CHECK: acc.routine @[[F3_R_NAME]] func(@[[F3_NAME]]) worker
+
+// CHECK: cir.func{{.*}} @[[F4_NAME:.*Func4[^\(]*]]({{.*}}){{.*}} attributes {acc.routine_info = #acc.routine_info<[@[[F4_R_NAME:.*]]]>}
+
+// CHECK: cir.func{{.*}} @[[F5_NAME:.*Func5[^\(]*]]({{.*}}){{.*}} attributes {acc.routine_info = #acc.routine_info<[@[[F5_R_NAME:.*]]]>}
+// CHECK: acc.routine @[[F5_R_NAME]] func(@[[F5_NAME]]) vector
+
+// CHECK: cir.func{{.*}} @[[F6_NAME:.*Func6[^\(]*]]({{.*}}){{.*}} attributes {acc.routine_info = #acc.routine_info<[@[[F6_R_NAME:.*]]]>}
+
+// CHECK: acc.routine @[[F2_R_NAME]] func(@[[F2_NAME]]) seq
+// CHECK: acc.routine @[[F4_R_NAME]] func(@[[F4_NAME]]) worker nohost
+// CHECK: acc.routine @[[F6_R_NAME]] func(@[[F6_NAME]]) vector nohost
diff --git a/clang/test/CIR/CodeGenOpenACC/routine-globals.cpp b/clang/test/CIR/CodeGenOpenACC/routine-globals.cpp
new file mode 100644
index 0000000000000..5f125bbce6cb8
--- /dev/null
+++ b/clang/test/CIR/CodeGenOpenACC/routine-globals.cpp
@@ -0,0 +1,35 @@
+// RUN: %clang_cc1 -fopenacc -Wno-openacc-self-if-potential-conflict -emit-cir -fclangir %s -o - | FileCheck %s
+
+#pragma acc routine seq
+auto Lambda1 = [](){};
+
+auto Lambda2 = [](){};
+#pragma acc routine(Lambda2) seq
+#pragma acc routine(Lambda2) seq
+
+#pragma acc routine seq
+int GlobalFunc1();
+
+int GlobalFunc2();
+#pragma acc routine(GlobalFunc2) seq
+#pragma acc routine(GlobalFunc1) seq
+
+void force_emit() {
+  Lambda1();
+  Lambda2();
+  GlobalFunc1();
+  GlobalFunc2();
+}
+
+// CHECK: cir.func {{.*}}lambda{{.*}} @[[L1_NAME:[^\(]*]]({{.*}}){{.*}} attributes {acc.routine_info = #acc.routine_info<[@[[L1_R_NAME:.*]]]>}
+// CHECK: cir.func {{.*}}lambda{{.*}} @[[L2_NAME:[^\(]*]]({{.*}}){{.*}} attributes {acc.routine_info = #acc.routine_info<[@[[L2_R_NAME:.*]], @[[L2_R2_NAME:.*]]]>}
+//
+// CHECK: cir.func{{.*}} @[[G1_NAME:[^\(]*]]({{.*}}){{.*}} attributes {acc.routine_info = #acc.routine_info<[@[[G1_R_NAME:.*]], @[[G1_R2_NAME:.*]]]>}
+// CHECK: cir.func{{.*}} @[[G2_NAME:[^\(]*]]({{.*}}){{.*}} attributes {acc.routine_info = #acc.routine_info<[@[[G2_R_NAME:.*]]]>}
+
+// CHECK: acc.routine @[[L1_R_NAME]] func(@[[L1_NAME]]) seq
+// CHECK: acc.routine @[[G1_R_NAME]] func(@[[G1_NAME]]) seq
+// CHECK: acc.routine @[[L2_R_NAME]] func(@[[L2_NAME]]) seq
+// CHECK: acc.routine @[[L2_R2_NAME]] func(@[[L2_NAME]]) seq
+// CHECK: acc.routine @[[G2_R_NAME]] func(@[[G2_NAME]]) seq
+// CHECK: acc.routine @[[G1_R2_NAME]] func(@[[G1_NAME]]) seq
diff --git a/clang/test/CIR/CodeGenOpenACC/routine-globals2.cpp b/clang/test/CIR/CodeGenOpenACC/routine-globals2.cpp
new file mode 100644
index 0000000000000..e1aa5046684da
--- /dev/null
+++ b/clang/test/CIR/CodeGenOpenACC/routine-globals2.cpp
@@ -0,0 +1,44 @@
+// RUN: %clang_cc1 -fopenacc -Wno-openacc-self-if-potential-conflict -emit-cir -fclangir %s -o - | FileCheck %s
+
+#pragma acc routine seq
+void GlobalFunc4();
+#pragma acc routine(GlobalFunc4) seq
+
+#pragma acc routine seq
+#pragma acc routine seq
+void GlobalFunc5();
+#pragma acc routine(GlobalFunc5) seq
+#pragma acc routine(GlobalFunc5) seq
+
+void GlobalFunc6();
+void GlobalFunc6();
+#pragma acc routine(GlobalFunc6) seq
+void GlobalFunc6(){}
+
+void GlobalFunc7(){}
+#pragma acc routine(GlobalFunc7) seq
+
+void force_emit() {
+  GlobalFunc4();
+  GlobalFunc5();
+  GlobalFunc6();
+  GlobalFunc7();
+}
+
+// CHECK: cir.func{{.*}} @[[G6_NAME:[^\(]*]]({{.*}}){{.*}} attributes {acc.routine_info = #acc.routine_info<[@[[G6_R_NAME:.*]]]>}
+// CHECK: cir.func{{.*}} @[[G7_NAME:[^\(]*]]({{.*}}){{.*}} attributes {acc.routine_info = #acc.routine_info<[@[[G7_R_NAME:.*]]]>}
+
+// CHECK: cir.func{{.*}} @[[G4_NAME:[^\(]*]]({{.*}}){{.*}} attributes {acc.routine_info = #acc.routine_info<[@[[G4_R_NAME:.*]], @[[G4_R2_NAME:.*]]]>}
+// CHECK: cir.func{{.*}} @[[G5_NAME:[^\(]*]]({{.*}}){{.*}} attributes {acc.routine_info = #acc.routine_info<[@[[G5_R_NAME:.*]], @[[G5_R1_NAME:.*]], @[[G5_R2_NAME:.*]], @[[G5_R3_NAME:.*]]]>}
+
+// CHECK: acc.routine @[[G4_R_NAME]] func(@[[G4_NAME]]) seq
+// CHECK: acc.routine @[[G5_R_NAME]] func(@[[G5_NAME]]) seq
+// CHECK: acc.routine @[[G5_R1_NAME]] func(@[[G5_NAME]]) seq
+//
+// CHECK: acc.routine @[[G4_R2_NAME]] func(@[[G4_NAME]]) seq
+//
+// CHECK: acc.routine @[[G5_R2_NAME]] func(@[[G5_NAME]]) seq
+// CHECK: acc.routine @[[G5_R3_NAME]] func(@[[G5_NAME]]) seq
+//
+// CHECK: acc.routine @[[G6_R_NAME]] func(@[[G6_NAME]]) seq
+// CHECK: acc.routine @[[G7_R_NAME]] func(@[[G7_NAME]]) seq
diff --git a/clang/test/CIR/CodeGenOpenACC/routine-locals.cpp b/clang/test/CIR/CodeGenOpenACC/routine-locals.cpp
new file mode 100644
index 0000000000000..d338a9cea0d09
--- /dev/null
+++ b/clang/test/CIR/CodeGenOpenACC/routine-locals.cpp
@@ -0,0 +1,24 @@
+// RUN: %clang_cc1 -fopenacc -Wno-openacc-self-if-potential-conflict -emit-cir -fclangir %s -o - | FileCheck %s
+
+void GlobalFunc();
+void InFunc() {
+
+#pragma acc routine(GlobalFunc) seq
+  GlobalFunc();
+
+#pragma acc routine seq
+  auto Lambda1 = [](){};
+  Lambda1();
+
+  auto Lambda2 = [](){};
+#pragma acc routine(Lambda2) seq
+  Lambda2();
+};
+
+// CHECK: cir.func{{.*}} @[[G1_NAME:[^\(]*]]({{.*}}){{.*}} attributes {acc.routine_info = #acc.routine_info<[@[[G1_R_NAME:.*]]]>}
+// CHECK: cir.func {{.*}}lambda{{.*}} @[[L1_NAME:[^\(]*]]({{.*}}){{.*}} attributes {acc.routine_info = #acc.routine_info<[@[[L1_R_NAME:.*]]]>}
+// CHECK: cir.func {{.*}}lambda{{.*}} @[[L2_NAME:[^\(]*]]({{.*}}){{.*}} attributes {acc.routine_info = #acc.routine_info<[@[[L2_R_NAME:.*]]]>}
+
+// CHECK: acc.routine @[[L1_R_NAME]] func(@[[L1_NAME]]) seq
+// CHECK: acc.routine @[[G1_R_NAME]] func(@[[G1_NAME]]) seq
+// CHECK: acc.routine @[[L2_R_NAME]] func(@[[L2_NAME]]) seq
diff --git a/clang/test/CIR/CodeGenOpenACC/routine-members.cpp b/clang/test/CIR/CodeGenOpenACC/routine-members.cpp
new file mode 100644
index 0000000000000..713500cfe3868
--- /dev/null
+++ b/clang/test/CIR/CodeGenOpenACC/routine-members.cpp
@@ -0,0 +1,55 @@
+// RUN: %clang_cc1 -fopenacc -Wno-openacc-self-if-potential-conflict -emit-cir -fclangir %s -o - | FileCheck %s
+
+struct S {
+#pragma acc routine seq
+  void MemFunc1();
+  void MemFunc2();
+#pragma acc routine(S::MemFunc2) seq
+  void MemFunc3();
+#pragma acc routine(S::MemFunc3) seq
+
+#pragma acc routine seq
+  static void StaticMemFunc1();
+  static void StaticMemFunc2();
+  static void StaticMemFunc3();
+#pragma acc routine(StaticMemFunc3) seq
+
+#pragma acc routine seq
+  static constexpr auto StaticLambda1 = [](){};
+ static constexpr auto StaticLambda2 = [](){};
+};
+#pragma acc routine(S::MemFunc2) seq
+#pragma acc routine(S::StaticLambda2) seq
+#pragma acc routine(S::StaticMemFunc2) seq
+
+void force_emit() {
+  S{}.MemFunc1();
+  S{}.MemFunc2();
+  S{}.MemFunc3();
+  S::StaticMemFunc1();
+  S::StaticMemFunc2();
+  S::StaticMemFunc3();
+  S::StaticLambda1();
+  S::StaticLambda2();
+}
+
+// CHECK: cir.func{{.*}} @[[MEM1_NAME:[^\(]*]]({{.*}}){{.*}} attributes {acc.routine_info = #acc.routine_info<[@[[MEM1_R_NAME:.*]]]>}
+// CHECK: cir.func{{.*}} @[[MEM2_NAME:[^\(]*]]({{.*}}){{.*}} attributes {acc.routine_info = #acc.routine_info<[@[[MEM2_R_NAME:.*]], @[[MEM2_R2_NAME:.*]]]>}
+// CHECK: cir.func{{.*}} @[[MEM3_NAME:[^\(]*]]({{.*}}){{.*}} attributes {acc.routine_info = #acc.routine_info<[@[[MEM3_R_NAME:.*]]]>}
+//
+// CHECK: cir.func{{.*}} @[[STATICMEM1_NAME:[^\(]*]]({{.*}}){{.*}} attributes {acc.routine_info = #acc.routine_info<[@[[STATICMEM1_R_NAME:.*]]]>}
+// CHECK: cir.func{{.*}} @[[STATICMEM2_NAME:[^\(]*]]({{.*}}){{.*}} attributes {acc.routine_info = #acc.routine_info<[@[[STATICMEM2_R_NAME:.*]]]>}
+// CHECK: cir.func{{.*}} @[[STATICMEM3_NAME:[^\(]*]]({{.*}}){{.*}} attributes {acc.routine_info = #acc.routine_info<[@[[STATICMEM3_R_NAME:.*]]]>}
+//
+// CHECK: cir.func {{.*}}lambda{{.*}} @[[L1_NAME:[^\(]*]]({{.*}}){{.*}} attributes {acc.routine_info = #acc.routine_info<[@[[L1_R_NAME:.*]]]>}
+// CHECK: cir.func {{.*}}lambda{{.*}} @[[L2_NAME:[^\(]*]]({{.*}}){{.*}} attributes {acc.routine_info = #acc.routine_info<[@[[L2_R_NAME:.*]]]>}
+//
+// CHECK: acc.routine @[[MEM1_R_NAME]] func(@[[MEM1_NAME]]) seq
+// CHECK: acc.routine @[[STATICMEM1_R_NAME]] func(@[[STATICMEM1_NAME]]) seq
+// CHECK: acc.routine @[[L1_R_NAME]] func(@[[L1_NAME]]) seq
+// CHECK: acc.routine @[[MEM2_R_NAME]] func(@[[MEM2_NAME]]) seq
+// CHECK: acc.routine @[[MEM3_R_NAME]] func(@[[MEM3_NAME]]) seq
+// CHECK: acc.routine @[[STATICMEM3_R_NAME]] func(@[[STATICMEM3_NAME]]) seq
+// CHECK: acc.routine @[[MEM2_R2_NAME]] func(@[[MEM2_NAME]]) seq
+// CHECK: acc.routine @[[L2_R_NAME]] func(@[[L2_NAME]]) seq
+// CHECK: acc.routine @[[STATICMEM2_R_NAME]] func(@[[STATICMEM2_NAME]]) seq
diff --git a/clang/test/CIR/CodeGenOpenACC/routine-ns.cpp b/clang/test/CIR/CodeGenOpenACC/routine-ns.cpp
new file mode 100644
index 0000000000000..9d1d677e79db8
--- /dev/null
+++ b/clang/test/CIR/CodeGenOpenACC/routine-ns.cpp
@@ -0,0 +1,28 @@
+// RUN: %clang_cc1 -fopenacc -Wno-openacc-self-if-potential-conflict -emit-cir -fclangir %s -o - | FileCheck %s
+
+namespace NS1 {
+#pragma acc routine seq
+  int NSFunc1();
+#pragma acc routine seq
+  auto Lambda1 = [](){};
+
+  auto Lambda2 = [](){};
+} // namespace NS1
+
+#pragma acc routine(NS1::NSFunc1) seq
+#pragma acc routine(NS1::Lambda2) seq
+
+void force_emit() {
+  NS1::NSFunc1();
+  NS1::Lambda1();
+  NS1::Lambda2();
+}
+
+// CHECK: cir.func{{.*}} @[[F1_NAME:[^\(]*]]({{.*}}){{.*}} attributes {acc.routine_info = #acc.routine_info<[@[[F1_R_NAME:.*]], @[[F1_R2_NAME:.*]]]>}
+// CHECK: cir.func {{.*}}lambda{{.*}} @[[L1_NAME:[^\(]*]]({{.*}}){{.*}} attributes {acc.routine_info = #acc.routine_info<[@[[L1_R_NAME:.*]]]>}
+// CHECK: cir.func {{.*}}lambda{{.*}} @[[L2_NAME:[^\(]*]]({{.*}}){{.*}} attributes {acc.routine_info = #acc.routine_info<[@[[L2_R_NAME:.*]]]>}
+//
+// CHECK: acc.routine @[[F1_R_NAME]] func(@[[F1_NAME]]) seq 
+// CHECK: acc.routine @[[L1_R_NAME]] func(@[[L1_NAME]]) seq 
+// CHECK: acc.routine @[[F1_R2_NAME]] func(@[[F1_NAME]]) seq  
+// CHECK: acc.routine @[[L2_R_NAME]] func(@[[L2_NAME]]) seq 
diff --git a/clang/test/CIR/CodeGenOpenACC/routine-templ.cpp b/clang/test/CIR/CodeGenOpenACC/routine-templ.cpp
new file mode 100644
index 0000000000000..419442220a1ba
--- /dev/null
+++ b/clang/test/CIR/CodeGenOpenACC/routine-templ.cpp
@@ -0,0 +1,16 @@
+// RUN: %clang_cc1 -fopenacc -Wno-openacc-self-if-potential-conflict -emit-cir -fclangir %s -o - | FileCheck %s
+
+#pragma acc routine seq
+template<typename T>
+void func(){}
+
+void use() {
+  func<int>();
+  func<float>();
+}
+
+// CHECK: cir.func{{.*}} @[[T1_NAME:[^\(]*]]({{.*}}){{.*}} attributes {acc.routine_info = #acc.routine_info<[@[[T1_R_NAME:.*]]]>}
+// CHECK: cir.func{{.*}} @[[T2_NAME:[^\(]*]]({{.*}}){{.*}} attributes {acc.routine_info = #acc.routine_info<[@[[T2_R_NAME:.*]]]>}
+//
+// CHECK: acc.routine @[[T1_R_NAME]] func(@[[T1_NAME]]) seq
+// CHECK: acc.routine @[[T2_R_NAME]] func(@[[T2_NAME]]) seq
diff --git a/clang/test/CodeGen/arm-mve-intrinsics/vmaxnmaq.c b/clang/test/CodeGen/arm-mve-intrinsics/vmaxnmaq.c
index 613a390bc6d36..04834ece3a4a6 100644
--- a/clang/test/CodeGen/arm-mve-intrinsics/vmaxnmaq.c
+++ b/clang/test/CodeGen/arm-mve-intrinsics/vmaxnmaq.c
@@ -1,17 +1,26 @@
 // NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py
-// RUN: %clang_cc1 -triple thumbv8.1m.main-none-none-eabi -target-feature +mve.fp -mfloat-abi hard -disable-O0-optnone -emit-llvm -o - %s | opt -S -passes=mem2reg | FileCheck %s
-// RUN: %clang_cc1 -triple thumbv8.1m.main-none-none-eabi -target-feature +mve.fp -mfloat-abi hard -disable-O0-optnone -DPOLYMORPHIC -emit-llvm -o - %s | opt -S -passes=mem2reg | FileCheck %s
+// RUN: %clang_cc1 -triple thumbv8.1m.main-none-none-eabi -target-feature +mve.fp -mfloat-abi hard -disable-O0-optnone -emit-llvm -o - %s | opt -S -passes=mem2reg | FileCheck %s --check-prefixes=CHECK,CHECK-NOSTRICT
+// RUN: %clang_cc1 -triple thumbv8.1m.main-none-none-eabi -target-feature +mve.fp -mfloat-abi hard -disable-O0-optnone -DPOLYMORPHIC -emit-llvm -o - %s | opt -S -passes=mem2reg | FileCheck %s --check-prefixes=CHECK,CHECK-NOSTRICT
+// RUN: %clang_cc1 -triple thumbv8.1m.main-none-none-eabi -target-feature +mve.fp -mfloat-abi hard -disable-O0-optnone -frounding-math -fexperimental-strict-floating-point -emit-llvm -o - %s | opt -S -passes=mem2reg | FileCheck %s --check-prefixes=CHECK,CHECK-STRICT
+// RUN: %clang_cc1 -triple thumbv8.1m.main-none-none-eabi -target-feature +mve.fp -mfloat-abi hard -disable-O0-optnone -frounding-math -fexperimental-strict-floating-point -DPOLYMORPHIC -emit-llvm -o - %s | opt -S -passes=mem2reg | FileCheck %s --check-prefixes=CHECK,CHECK-STRICT
 
 // REQUIRES: aarch64-registered-target || arm-registered-target
 
 #include <arm_mve.h>
 
-// CHECK-LABEL: @test_vmaxnmaq_f16(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = call <8 x half> @llvm.fabs.v8f16(<8 x half> [[A:%.*]])
-// CHECK-NEXT:    [[TMP1:%.*]] = call <8 x half> @llvm.fabs.v8f16(<8 x half> [[B:%.*]])
-// CHECK-NEXT:    [[TMP2:%.*]] = call <8 x half> @llvm.maxnum.v8f16(<8 x half> [[TMP0]], <8 x half> [[TMP1]])
-// CHECK-NEXT:    ret <8 x half> [[TMP2]]
+// CHECK-NOSTRICT-LABEL: @test_vmaxnmaq_f16(
+// CHECK-NOSTRICT-NEXT:  entry:
+// CHECK-NOSTRICT-NEXT:    [[TMP0:%.*]] = call <8 x half> @llvm.fabs.v8f16(<8 x half> [[A:%.*]])
+// CHECK-NOSTRICT-NEXT:    [[TMP1:%.*]] = call <8 x half> @llvm.fabs.v8f16(<8 x half> [[B:%.*]])
+// CHECK-NOSTRICT-NEXT:    [[TMP2:%.*]] = call <8 x half> @llvm.maxnum.v8f16(<8 x half> [[TMP0]], <8 x half> [[TMP1]])
+// CHECK-NOSTRICT-NEXT:    ret <8 x half> [[TMP2]]
+//
+// CHECK-STRICT-LABEL: @test_vmaxnmaq_f16(
+// CHECK-STRICT-NEXT:  entry:
+// CHECK-STRICT-NEXT:    [[TMP0:%.*]] = call <8 x half> @llvm.fabs.v8f16(<8 x half> [[A:%.*]]) #[[ATTR3:[0-9]+]]
+// CHECK-STRICT-NEXT:    [[TMP1:%.*]] = call <8 x half> @llvm.fabs.v8f16(<8 x half> [[B:%.*]]) #[[ATTR3]]
+// CHECK-STRICT-NEXT:    [[TMP2:%.*]] = call <8 x half> @llvm.arm.mve.vmaxnm.v8f16(<8 x half> [[TMP0]], <8 x half> [[TMP1]]) #[[ATTR3]]
+// CHECK-STRICT-NEXT:    ret <8 x half> [[TMP2]]
 //
 float16x8_t test_vmaxnmaq_f16(float16x8_t a, float16x8_t b)
 {
@@ -22,12 +31,19 @@ float16x8_t test_vmaxnmaq_f16(float16x8_t a, float16x8_t b)
 #endif /* POLYMORPHIC */
 }
 
-// CHECK-LABEL: @test_vmaxnmaq_f32(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = call <4 x float> @llvm.fabs.v4f32(<4 x float> [[A:%.*]])
-// CHECK-NEXT:    [[TMP1:%.*]] = call <4 x float> @llvm.fabs.v4f32(<4 x float> [[B:%.*]])
-// CHECK-NEXT:    [[TMP2:%.*]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[TMP0]], <4 x float> [[TMP1]])
-// CHECK-NEXT:    ret <4 x float> [[TMP2]]
+// CHECK-NOSTRICT-LABEL: @test_vmaxnmaq_f32(
+// CHECK-NOSTRICT-NEXT:  entry:
+// CHECK-NOSTRICT-NEXT:    [[TMP0:%.*]] = call <4 x float> @llvm.fabs.v4f32(<4 x float> [[A:%.*]])
+// CHECK-NOSTRICT-NEXT:    [[TMP1:%.*]] = call <4 x float> @llvm.fabs.v4f32(<4 x float> [[B:%.*]])
+// CHECK-NOSTRICT-NEXT:    [[TMP2:%.*]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[TMP0]], <4 x float> [[TMP1]])
+// CHECK-NOSTRICT-NEXT:    ret <4 x float> [[TMP2]]
+//
+// CHECK-STRICT-LABEL: @test_vmaxnmaq_f32(
+// CHECK-STRICT-NEXT:  entry:
+// CHECK-STRICT-NEXT:    [[TMP0:%.*]] = call <4 x float> @llvm.fabs.v4f32(<4 x float> [[A:%.*]]) #[[ATTR3]]
+// CHECK-STRICT-NEXT:    [[TMP1:%.*]] = call <4 x float> @llvm.fabs.v4f32(<4 x float> [[B:%.*]]) #[[ATTR3]]
+// CHECK-STRICT-NEXT:    [[TMP2:%.*]] = call <4 x float> @llvm.arm.mve.vmaxnm.v4f32(<4 x float> [[TMP0]], <4 x float> [[TMP1]]) #[[ATTR3]]
+// CHECK-STRICT-NEXT:    ret <4 x float> [[TMP2]]
 //
 float32x4_t test_vmaxnmaq_f32(float32x4_t a, float32x4_t b)
 {
@@ -38,12 +54,19 @@ float32x4_t test_vmaxnmaq_f32(float32x4_t a, float32x4_t b)
 #endif /* POLYMORPHIC */
 }
 
-// CHECK-LABEL: @test_vmaxnmaq_m_f16(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = zext i16 [[P:%.*]] to i32
-// CHECK-NEXT:    [[TMP1:%.*]] = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 [[TMP0]])
-// CHECK-NEXT:    [[TMP2:%.*]] = call <8 x half> @llvm.arm.mve.vmaxnma.predicated.v8f16.v8i1(<8 x half> [[A:%.*]], <8 x half> [[B:%.*]], <8 x i1> [[TMP1]])
-// CHECK-NEXT:    ret <8 x half> [[TMP2]]
+// CHECK-NOSTRICT-LABEL: @test_vmaxnmaq_m_f16(
+// CHECK-NOSTRICT-NEXT:  entry:
+// CHECK-NOSTRICT-NEXT:    [[TMP0:%.*]] = zext i16 [[P:%.*]] to i32
+// CHECK-NOSTRICT-NEXT:    [[TMP1:%.*]] = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 [[TMP0]])
+// CHECK-NOSTRICT-NEXT:    [[TMP2:%.*]] = call <8 x half> @llvm.arm.mve.vmaxnma.predicated.v8f16.v8i1(<8 x half> [[A:%.*]], <8 x half> [[B:%.*]], <8 x i1> [[TMP1]])
+// CHECK-NOSTRICT-NEXT:    ret <8 x half> [[TMP2]]
+//
+// CHECK-STRICT-LABEL: @test_vmaxnmaq_m_f16(
+// CHECK-STRICT-NEXT:  entry:
+// CHECK-STRICT-NEXT:    [[TMP0:%.*]] = zext i16 [[P:%.*]] to i32
+// CHECK-STRICT-NEXT:    [[TMP1:%.*]] = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 [[TMP0]]) #[[ATTR3]]
+// CHECK-STRICT-NEXT:    [[TMP2:%.*]] = call <8 x half> @llvm.arm.mve.vmaxnma.predicated.v8f16.v8i1(<8 x half> [[A:%.*]], <8 x half> [[B:%.*]], <8 x i1> [[TMP1]]) #[[ATTR3]]
+// CHECK-STRICT-NEXT:    ret <8 x half> [[TMP2]]
 //
 float16x8_t test_vmaxnmaq_m_f16(float16x8_t a, float16x8_t b, mve_pred16_t p)
 {
@@ -54,12 +77,19 @@ float16x8_t test_vmaxnmaq_m_f16(float16x8_t a, float16x8_t b, mve_pred16_t p)
 #endif /* POLYMORPHIC */
 }
 
-// CHECK-LABEL: @test_vmaxnmaq_m_f32(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = zext i16 [[P:%.*]] to i32
-// CHECK-NEXT:    [[TMP1:%.*]] = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 [[TMP0]])
-// CHECK-NEXT:    [[TMP2:%.*]] = call <4 x float> @llvm.arm.mve.vmaxnma.predicated.v4f32.v4i1(<4 x float> [[A:%.*]], <4 x float> [[B:%.*]], <4 x i1> [[TMP1]])
-// CHECK-NEXT:    ret <4 x float> [[TMP2]]
+// CHECK-NOSTRICT-LABEL: @test_vmaxnmaq_m_f32(
+// CHECK-NOSTRICT-NEXT:  entry:
+// CHECK-NOSTRICT-NEXT:    [[TMP0:%.*]] = zext i16 [[P:%.*]] to i32
+// CHECK-NOSTRICT-NEXT:    [[TMP1:%.*]] = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 [[TMP0]])
+// CHECK-NOSTRICT-NEXT:    [[TMP2:%.*]] = call <4 x float> @llvm.arm.mve.vmaxnma.predicated.v4f32.v4i1(<4 x float> [[A:%.*]], <4 x float> [[B:%.*]], <4 x i1> [[TMP1]])
+// CHECK-NOSTRICT-NEXT:    ret <4 x float> [[TMP2]]
+//
+// CHECK-STRICT-LABEL: @test_vmaxnmaq_m_f32(
+// CHECK-STRICT-NEXT:  entry:
+// CHECK-STRICT-NEXT:    [[TMP0:%.*]] = zext i16 [[P:%.*]] to i32
+// CHECK-STRICT-NEXT:    [[TMP1:%.*]] = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 [[TMP0]]) #[[ATTR3]]
+// CHECK-STRICT-NEXT:    [[TMP2:%.*]] = call <4 x float> @llvm.arm.mve.vmaxnma.predicated.v4f32.v4i1(<4 x float> [[A:%.*]], <4 x float> [[B:%.*]], <4 x i1> [[TMP1]]) #[[ATTR3]]
+// CHECK-STRICT-NEXT:    ret <4 x float> [[TMP2]]
 //
 float32x4_t test_vmaxnmaq_m_f32(float32x4_t a, float32x4_t b, mve_pred16_t p)
 {
@@ -69,3 +99,5 @@ float32x4_t test_vmaxnmaq_m_f32(float32x4_t a, float32x4_t b, mve_pred16_t p)
     return vmaxnmaq_m_f32(a, b, p);
 #endif /* POLYMORPHIC */
 }
+//// NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+// CHECK: {{.*}}
diff --git a/clang/test/CodeGen/arm-mve-intrinsics/vmaxnmq.c b/clang/test/CodeGen/arm-mve-intrinsics/vmaxnmq.c
index bad7cd903ab16..1225353a5a9d2 100644
--- a/clang/test/CodeGen/arm-mve-intrinsics/vmaxnmq.c
+++ b/clang/test/CodeGen/arm-mve-intrinsics/vmaxnmq.c
@@ -1,15 +1,22 @@
 // NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py
-// RUN: %clang_cc1 -triple thumbv8.1m.main-none-none-eabi -target-feature +mve.fp -mfloat-abi hard -disable-O0-optnone -emit-llvm -o - %s | opt -S -passes=mem2reg | FileCheck %s
-// RUN: %clang_cc1 -triple thumbv8.1m.main-none-none-eabi -target-feature +mve.fp -mfloat-abi hard -disable-O0-optnone -DPOLYMORPHIC -emit-llvm -o - %s | opt -S -passes=mem2reg | FileCheck %s
+// RUN: %clang_cc1 -triple thumbv8.1m.main-none-none-eabi -target-feature +mve.fp -mfloat-abi hard -disable-O0-optnone -emit-llvm -o - %s | opt -S -passes=mem2reg | FileCheck %s --check-prefixes=CHECK,CHECK-NOSTRICT
+// RUN: %clang_cc1 -triple thumbv8.1m.main-none-none-eabi -target-feature +mve.fp -mfloat-abi hard -disable-O0-optnone -DPOLYMORPHIC -emit-llvm -o - %s | opt -S -passes=mem2reg | FileCheck %s --check-prefixes=CHECK,CHECK-NOSTRICT
+// RUN: %clang_cc1 -triple thumbv8.1m.main-none-none-eabi -target-feature +mve.fp -mfloat-abi hard -disable-O0-optnone -frounding-math -fexperimental-strict-floating-point -emit-llvm -o - %s | opt -S -passes=mem2reg | FileCheck %s --check-prefixes=CHECK,CHECK-STRICT
+// RUN: %clang_cc1 -triple thumbv8.1m.main-none-none-eabi -target-feature +mve.fp -mfloat-abi hard -disable-O0-optnone -frounding-math -fexperimental-strict-floating-point -DPOLYMORPHIC -emit-llvm -o - %s | opt -S -passes=mem2reg | FileCheck %s --check-prefixes=CHECK,CHECK-STRICT
 
 // REQUIRES: aarch64-registered-target || arm-registered-target
 
 #include <arm_mve.h>
 
-// CHECK-LABEL: @test_vmaxnmq_f16(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = call <8 x half> @llvm.maxnum.v8f16(<8 x half> [[A:%.*]], <8 x half> [[B:%.*]])
-// CHECK-NEXT:    ret <8 x half> [[TMP0]]
+// CHECK-NOSTRICT-LABEL: @test_vmaxnmq_f16(
+// CHECK-NOSTRICT-NEXT:  entry:
+// CHECK-NOSTRICT-NEXT:    [[TMP0:%.*]] = call <8 x half> @llvm.maxnum.v8f16(<8 x half> [[A:%.*]], <8 x half> [[B:%.*]])
+// CHECK-NOSTRICT-NEXT:    ret <8 x half> [[TMP0]]
+//
+// CHECK-STRICT-LABEL: @test_vmaxnmq_f16(
+// CHECK-STRICT-NEXT:  entry:
+// CHECK-STRICT-NEXT:    [[TMP0:%.*]] = call <8 x half> @llvm.arm.mve.vmaxnm.v8f16(<8 x half> [[A:%.*]], <8 x half> [[B:%.*]]) #[[ATTR2:[0-9]+]]
+// CHECK-STRICT-NEXT:    ret <8 x half> [[TMP0]]
 //
 float16x8_t test_vmaxnmq_f16(float16x8_t a, float16x8_t b)
 {
@@ -20,10 +27,15 @@ float16x8_t test_vmaxnmq_f16(float16x8_t a, float16x8_t b)
 #endif /* POLYMORPHIC */
 }
 
-// CHECK-LABEL: @test_vmaxnmq_f32(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[A:%.*]], <4 x float> [[B:%.*]])
-// CHECK-NEXT:    ret <4 x float> [[TMP0]]
+// CHECK-NOSTRICT-LABEL: @test_vmaxnmq_f32(
+// CHECK-NOSTRICT-NEXT:  entry:
+// CHECK-NOSTRICT-NEXT:    [[TMP0:%.*]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[A:%.*]], <4 x float> [[B:%.*]])
+// CHECK-NOSTRICT-NEXT:    ret <4 x float> [[TMP0]]
+//
+// CHECK-STRICT-LABEL: @test_vmaxnmq_f32(
+// CHECK-STRICT-NEXT:  entry:
+// CHECK-STRICT-NEXT:    [[TMP0:%.*]] = call <4 x float> @llvm.arm.mve.vmaxnm.v4f32(<4 x float> [[A:%.*]], <4 x float> [[B:%.*]]) #[[ATTR2]]
+// CHECK-STRICT-NEXT:    ret <4 x float> [[TMP0]]
 //
 float32x4_t test_vmaxnmq_f32(float32x4_t a, float32x4_t b)
 {
@@ -34,12 +46,19 @@ float32x4_t test_vmaxnmq_f32(float32x4_t a, float32x4_t b)
 #endif /* POLYMORPHIC */
 }
 
-// CHECK-LABEL: @test_vmaxnmq_m_f16(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = zext i16 [[P:%.*]] to i32
-// CHECK-NEXT:    [[TMP1:%.*]] = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 [[TMP0]])
-// CHECK-NEXT:    [[TMP2:%.*]] = call <8 x half> @llvm.arm.mve.max.predicated.v8f16.v8i1(<8 x half> [[A:%.*]], <8 x half> [[B:%.*]], i32 0, <8 x i1> [[TMP1]], <8 x half> [[INACTIVE:%.*]])
-// CHECK-NEXT:    ret <8 x half> [[TMP2]]
+// CHECK-NOSTRICT-LABEL: @test_vmaxnmq_m_f16(
+// CHECK-NOSTRICT-NEXT:  entry:
+// CHECK-NOSTRICT-NEXT:    [[TMP0:%.*]] = zext i16 [[P:%.*]] to i32
+// CHECK-NOSTRICT-NEXT:    [[TMP1:%.*]] = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 [[TMP0]])
+// CHECK-NOSTRICT-NEXT:    [[TMP2:%.*]] = call <8 x half> @llvm.arm.mve.max.predicated.v8f16.v8i1(<8 x half> [[A:%.*]], <8 x half> [[B:%.*]], i32 0, <8 x i1> [[TMP1]], <8 x half> [[INACTIVE:%.*]])
+// CHECK-NOSTRICT-NEXT:    ret <8 x half> [[TMP2]]
+//
+// CHECK-STRICT-LABEL: @test_vmaxnmq_m_f16(
+// CHECK-STRICT-NEXT:  entry:
+// CHECK-STRICT-NEXT:    [[TMP0:%.*]] = zext i16 [[P:%.*]] to i32
+// CHECK-STRICT-NEXT:    [[TMP1:%.*]] = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 [[TMP0]]) #[[ATTR2]]
+// CHECK-STRICT-NEXT:    [[TMP2:%.*]] = call <8 x half> @llvm.arm.mve.max.predicated.v8f16.v8i1(<8 x half> [[A:%.*]], <8 x half> [[B:%.*]], i32 0, <8 x i1> [[TMP1]], <8 x half> [[INACTIVE:%.*]]) #[[ATTR2]]
+// CHECK-STRICT-NEXT:    ret <8 x half> [[TMP2]]
 //
 float16x8_t test_vmaxnmq_m_f16(float16x8_t inactive, float16x8_t a, float16x8_t b, mve_pred16_t p)
 {
@@ -50,12 +69,19 @@ float16x8_t test_vmaxnmq_m_f16(float16x8_t inactive, float16x8_t a, float16x8_t
 #endif /* POLYMORPHIC */
 }
 
-// CHECK-LABEL: @test_vmaxnmq_m_f32(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = zext i16 [[P:%.*]] to i32
-// CHECK-NEXT:    [[TMP1:%.*]] = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 [[TMP0]])
-// CHECK-NEXT:    [[TMP2:%.*]] = call <4 x float> @llvm.arm.mve.max.predicated.v4f32.v4i1(<4 x float> [[A:%.*]], <4 x float> [[B:%.*]], i32 0, <4 x i1> [[TMP1]], <4 x float> [[INACTIVE:%.*]])
-// CHECK-NEXT:    ret <4 x float> [[TMP2]]
+// CHECK-NOSTRICT-LABEL: @test_vmaxnmq_m_f32(
+// CHECK-NOSTRICT-NEXT:  entry:
+// CHECK-NOSTRICT-NEXT:    [[TMP0:%.*]] = zext i16 [[P:%.*]] to i32
+// CHECK-NOSTRICT-NEXT:    [[TMP1:%.*]] = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 [[TMP0]])
+// CHECK-NOSTRICT-NEXT:    [[TMP2:%.*]] = call <4 x float> @llvm.arm.mve.max.predicated.v4f32.v4i1(<4 x float> [[A:%.*]], <4 x float> [[B:%.*]], i32 0, <4 x i1> [[TMP1]], <4 x float> [[INACTIVE:%.*]])
+// CHECK-NOSTRICT-NEXT:    ret <4 x float> [[TMP2]]
+//
+// CHECK-STRICT-LABEL: @test_vmaxnmq_m_f32(
+// CHECK-STRICT-NEXT:  entry:
+// CHECK-STRICT-NEXT:    [[TMP0:%.*]] = zext i16 [[P:%.*]] to i32
+// CHECK-STRICT-NEXT:    [[TMP1:%.*]] = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 [[TMP0]]) #[[ATTR2]]
+// CHECK-STRICT-NEXT:    [[TMP2:%.*]] = call <4 x float> @llvm.arm.mve.max.predicated.v4f32.v4i1(<4 x float> [[A:%.*]], <4 x float> [[B:%.*]], i32 0, <4 x i1> [[TMP1]], <4 x float> [[INACTIVE:%.*]]) #[[ATTR2]]
+// CHECK-STRICT-NEXT:    ret <4 x float> [[TMP2]]
 //
 float32x4_t test_vmaxnmq_m_f32(float32x4_t inactive, float32x4_t a, float32x4_t b, mve_pred16_t p)
 {
@@ -66,12 +92,19 @@ float32x4_t test_vmaxnmq_m_f32(float32x4_t inactive, float32x4_t a, float32x4_t
 #endif /* POLYMORPHIC */
 }
 
-// CHECK-LABEL: @test_vmaxnmq_x_f16(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = zext i16 [[P:%.*]] to i32
-// CHECK-NEXT:    [[TMP1:%.*]] = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 [[TMP0]])
-// CHECK-NEXT:    [[TMP2:%.*]] = call <8 x half> @llvm.arm.mve.max.predicated.v8f16.v8i1(<8 x half> [[A:%.*]], <8 x half> [[B:%.*]], i32 0, <8 x i1> [[TMP1]], <8 x half> undef)
-// CHECK-NEXT:    ret <8 x half> [[TMP2]]
+// CHECK-NOSTRICT-LABEL: @test_vmaxnmq_x_f16(
+// CHECK-NOSTRICT-NEXT:  entry:
+// CHECK-NOSTRICT-NEXT:    [[TMP0:%.*]] = zext i16 [[P:%.*]] to i32
+// CHECK-NOSTRICT-NEXT:    [[TMP1:%.*]] = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 [[TMP0]])
+// CHECK-NOSTRICT-NEXT:    [[TMP2:%.*]] = call <8 x half> @llvm.arm.mve.max.predicated.v8f16.v8i1(<8 x half> [[A:%.*]], <8 x half> [[B:%.*]], i32 0, <8 x i1> [[TMP1]], <8 x half> undef)
+// CHECK-NOSTRICT-NEXT:    ret <8 x half> [[TMP2]]
+//
+// CHECK-STRICT-LABEL: @test_vmaxnmq_x_f16(
+// CHECK-STRICT-NEXT:  entry:
+// CHECK-STRICT-NEXT:    [[TMP0:%.*]] = zext i16 [[P:%.*]] to i32
+// CHECK-STRICT-NEXT:    [[TMP1:%.*]] = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 [[TMP0]]) #[[ATTR2]]
+// CHECK-STRICT-NEXT:    [[TMP2:%.*]] = call <8 x half> @llvm.arm.mve.max.predicated.v8f16.v8i1(<8 x half> [[A:%.*]], <8 x half> [[B:%.*]], i32 0, <8 x i1> [[TMP1]], <8 x half> undef) #[[ATTR2]]
+// CHECK-STRICT-NEXT:    ret <8 x half> [[TMP2]]
 //
 float16x8_t test_vmaxnmq_x_f16(float16x8_t a, float16x8_t b, mve_pred16_t p)
 {
@@ -82,12 +115,19 @@ float16x8_t test_vmaxnmq_x_f16(float16x8_t a, float16x8_t b, mve_pred16_t p)
 #endif /* POLYMORPHIC */
 }
 
-// CHECK-LABEL: @test_vmaxnmq_x_f32(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = zext i16 [[P:%.*]] to i32
-// CHECK-NEXT:    [[TMP1:%.*]] = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 [[TMP0]])
-// CHECK-NEXT:    [[TMP2:%.*]] = call <4 x float> @llvm.arm.mve.max.predicated.v4f32.v4i1(<4 x float> [[A:%.*]], <4 x float> [[B:%.*]], i32 0, <4 x i1> [[TMP1]], <4 x float> undef)
-// CHECK-NEXT:    ret <4 x float> [[TMP2]]
+// CHECK-NOSTRICT-LABEL: @test_vmaxnmq_x_f32(
+// CHECK-NOSTRICT-NEXT:  entry:
+// CHECK-NOSTRICT-NEXT:    [[TMP0:%.*]] = zext i16 [[P:%.*]] to i32
+// CHECK-NOSTRICT-NEXT:    [[TMP1:%.*]] = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 [[TMP0]])
+// CHECK-NOSTRICT-NEXT:    [[TMP2:%.*]] = call <4 x float> @llvm.arm.mve.max.predicated.v4f32.v4i1(<4 x float> [[A:%.*]], <4 x float> [[B:%.*]], i32 0, <4 x i1> [[TMP1]], <4 x float> undef)
+// CHECK-NOSTRICT-NEXT:    ret <4 x float> [[TMP2]]
+//
+// CHECK-STRICT-LABEL: @test_vmaxnmq_x_f32(
+// CHECK-STRICT-NEXT:  entry:
+// CHECK-STRICT-NEXT:    [[TMP0:%.*]] = zext i16 [[P:%.*]] to i32
+// CHECK-STRICT-NEXT:    [[TMP1:%.*]] = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 [[TMP0]]) #[[ATTR2]]
+// CHECK-STRICT-NEXT:    [[TMP2:%.*]] = call <4 x float> @llvm.arm.mve.max.predicated.v4f32.v4i1(<4 x float> [[A:%.*]], <4 x float> [[B:%.*]], i32 0, <4 x i1> [[TMP1]], <4 x float> undef) #[[ATTR2]]
+// CHECK-STRICT-NEXT:    ret <4 x float> [[TMP2]]
 //
 float32x4_t test_vmaxnmq_x_f32(float32x4_t a, float32x4_t b, mve_pred16_t p)
 {
@@ -97,3 +137,5 @@ float32x4_t test_vmaxnmq_x_f32(float32x4_t a, float32x4_t b, mve_pred16_t p)
     return vmaxnmq_x_f32(a, b, p);
 #endif /* POLYMORPHIC */
 }
+//// NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+// CHECK: {{.*}}
diff --git a/clang/test/CodeGen/arm-mve-intrinsics/vminnmaq.c b/clang/test/CodeGen/arm-mve-intrinsics/vminnmaq.c
index 0182cf7c5b6b3..fc0dc5701e4d9 100644
--- a/clang/test/CodeGen/arm-mve-intrinsics/vminnmaq.c
+++ b/clang/test/CodeGen/arm-mve-intrinsics/vminnmaq.c
@@ -1,17 +1,26 @@
 // NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py
-// RUN: %clang_cc1 -triple thumbv8.1m.main-none-none-eabi -target-feature +mve.fp -mfloat-abi hard -disable-O0-optnone -emit-llvm -o - %s | opt -S -passes=mem2reg | FileCheck %s
-// RUN: %clang_cc1 -triple thumbv8.1m.main-none-none-eabi -target-feature +mve.fp -mfloat-abi hard -disable-O0-optnone -DPOLYMORPHIC -emit-llvm -o - %s | opt -S -passes=mem2reg | FileCheck %s
+// RUN: %clang_cc1 -triple thumbv8.1m.main-none-none-eabi -target-feature +mve.fp -mfloat-abi hard -disable-O0-optnone -emit-llvm -o - %s | opt -S -passes=mem2reg | FileCheck %s --check-prefixes=CHECK,CHECK-NOSTRICT
+// RUN: %clang_cc1 -triple thumbv8.1m.main-none-none-eabi -target-feature +mve.fp -mfloat-abi hard -disable-O0-optnone -DPOLYMORPHIC -emit-llvm -o - %s | opt -S -passes=mem2reg | FileCheck %s --check-prefixes=CHECK,CHECK-NOSTRICT
+// RUN: %clang_cc1 -triple thumbv8.1m.main-none-none-eabi -target-feature +mve.fp -mfloat-abi hard -disable-O0-optnone -frounding-math -fexperimental-strict-floating-point -emit-llvm -o - %s | opt -S -passes=mem2reg | FileCheck %s --check-prefixes=CHECK,CHECK-STRICT
+// RUN: %clang_cc1 -triple thumbv8.1m.main-none-none-eabi -target-feature +mve.fp -mfloat-abi hard -disable-O0-optnone -frounding-math -fexperimental-strict-floating-point -DPOLYMORPHIC -emit-llvm -o - %s | opt -S -passes=mem2reg | FileCheck %s --check-prefixes=CHECK,CHECK-STRICT
 
 // REQUIRES: aarch64-registered-target || arm-registered-target
 
 #include <arm_mve.h>
 
-// CHECK-LABEL: @test_vminnmaq_f16(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = call <8 x half> @llvm.fabs.v8f16(<8 x half> [[A:%.*]])
-// CHECK-NEXT:    [[TMP1:%.*]] = call <8 x half> @llvm.fabs.v8f16(<8 x half> [[B:%.*]])
-// CHECK-NEXT:    [[TMP2:%.*]] = call <8 x half> @llvm.minnum.v8f16(<8 x half> [[TMP0]], <8 x half> [[TMP1]])
-// CHECK-NEXT:    ret <8 x half> [[TMP2]]
+// CHECK-NOSTRICT-LABEL: @test_vminnmaq_f16(
+// CHECK-NOSTRICT-NEXT:  entry:
+// CHECK-NOSTRICT-NEXT:    [[TMP0:%.*]] = call <8 x half> @llvm.fabs.v8f16(<8 x half> [[A:%.*]])
+// CHECK-NOSTRICT-NEXT:    [[TMP1:%.*]] = call <8 x half> @llvm.fabs.v8f16(<8 x half> [[B:%.*]])
+// CHECK-NOSTRICT-NEXT:    [[TMP2:%.*]] = call <8 x half> @llvm.minnum.v8f16(<8 x half> [[TMP0]], <8 x half> [[TMP1]])
+// CHECK-NOSTRICT-NEXT:    ret <8 x half> [[TMP2]]
+//
+// CHECK-STRICT-LABEL: @test_vminnmaq_f16(
+// CHECK-STRICT-NEXT:  entry:
+// CHECK-STRICT-NEXT:    [[TMP0:%.*]] = call <8 x half> @llvm.fabs.v8f16(<8 x half> [[A:%.*]]) #[[ATTR3:[0-9]+]]
+// CHECK-STRICT-NEXT:    [[TMP1:%.*]] = call <8 x half> @llvm.fabs.v8f16(<8 x half> [[B:%.*]]) #[[ATTR3]]
+// CHECK-STRICT-NEXT:    [[TMP2:%.*]] = call <8 x half> @llvm.arm.mve.vminnm.v8f16(<8 x half> [[TMP0]], <8 x half> [[TMP1]]) #[[ATTR3]]
+// CHECK-STRICT-NEXT:    ret <8 x half> [[TMP2]]
 //
 float16x8_t test_vminnmaq_f16(float16x8_t a, float16x8_t b)
 {
@@ -22,12 +31,19 @@ float16x8_t test_vminnmaq_f16(float16x8_t a, float16x8_t b)
 #endif /* POLYMORPHIC */
 }
 
-// CHECK-LABEL: @test_vminnmaq_f32(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = call <4 x float> @llvm.fabs.v4f32(<4 x float> [[A:%.*]])
-// CHECK-NEXT:    [[TMP1:%.*]] = call <4 x float> @llvm.fabs.v4f32(<4 x float> [[B:%.*]])
-// CHECK-NEXT:    [[TMP2:%.*]] = call <4 x float> @llvm.minnum.v4f32(<4 x float> [[TMP0]], <4 x float> [[TMP1]])
-// CHECK-NEXT:    ret <4 x float> [[TMP2]]
+// CHECK-NOSTRICT-LABEL: @test_vminnmaq_f32(
+// CHECK-NOSTRICT-NEXT:  entry:
+// CHECK-NOSTRICT-NEXT:    [[TMP0:%.*]] = call <4 x float> @llvm.fabs.v4f32(<4 x float> [[A:%.*]])
+// CHECK-NOSTRICT-NEXT:    [[TMP1:%.*]] = call <4 x float> @llvm.fabs.v4f32(<4 x float> [[B:%.*]])
+// CHECK-NOSTRICT-NEXT:    [[TMP2:%.*]] = call <4 x float> @llvm.minnum.v4f32(<4 x float> [[TMP0]], <4 x float> [[TMP1]])
+// CHECK-NOSTRICT-NEXT:    ret <4 x float> [[TMP2]]
+//
+// CHECK-STRICT-LABEL: @test_vminnmaq_f32(
+// CHECK-STRICT-NEXT:  entry:
+// CHECK-STRICT-NEXT:    [[TMP0:%.*]] = call <4 x float> @llvm.fabs.v4f32(<4 x float> [[A:%.*]]) #[[ATTR3]]
+// CHECK-STRICT-NEXT:    [[TMP1:%.*]] = call <4 x float> @llvm.fabs.v4f32(<4 x float> [[B:%.*]]) #[[ATTR3]]
+// CHECK-STRICT-NEXT:    [[TMP2:%.*]] = call <4 x float> @llvm.arm.mve.vminnm.v4f32(<4 x float> [[TMP0]], <4 x float> [[TMP1]]) #[[ATTR3]]
+// CHECK-STRICT-NEXT:    ret <4 x float> [[TMP2]]
 //
 float32x4_t test_vminnmaq_f32(float32x4_t a, float32x4_t b)
 {
@@ -38,12 +54,19 @@ float32x4_t test_vminnmaq_f32(float32x4_t a, float32x4_t b)
 #endif /* POLYMORPHIC */
 }
 
-// CHECK-LABEL: @test_vminnmaq_m_f16(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = zext i16 [[P:%.*]] to i32
-// CHECK-NEXT:    [[TMP1:%.*]] = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 [[TMP0]])
-// CHECK-NEXT:    [[TMP2:%.*]] = call <8 x half> @llvm.arm.mve.vminnma.predicated.v8f16.v8i1(<8 x half> [[A:%.*]], <8 x half> [[B:%.*]], <8 x i1> [[TMP1]])
-// CHECK-NEXT:    ret <8 x half> [[TMP2]]
+// CHECK-NOSTRICT-LABEL: @test_vminnmaq_m_f16(
+// CHECK-NOSTRICT-NEXT:  entry:
+// CHECK-NOSTRICT-NEXT:    [[TMP0:%.*]] = zext i16 [[P:%.*]] to i32
+// CHECK-NOSTRICT-NEXT:    [[TMP1:%.*]] = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 [[TMP0]])
+// CHECK-NOSTRICT-NEXT:    [[TMP2:%.*]] = call <8 x half> @llvm.arm.mve.vminnma.predicated.v8f16.v8i1(<8 x half> [[A:%.*]], <8 x half> [[B:%.*]], <8 x i1> [[TMP1]])
+// CHECK-NOSTRICT-NEXT:    ret <8 x half> [[TMP2]]
+//
+// CHECK-STRICT-LABEL: @test_vminnmaq_m_f16(
+// CHECK-STRICT-NEXT:  entry:
+// CHECK-STRICT-NEXT:    [[TMP0:%.*]] = zext i16 [[P:%.*]] to i32
+// CHECK-STRICT-NEXT:    [[TMP1:%.*]] = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 [[TMP0]]) #[[ATTR3]]
+// CHECK-STRICT-NEXT:    [[TMP2:%.*]] = call <8 x half> @llvm.arm.mve.vminnma.predicated.v8f16.v8i1(<8 x half> [[A:%.*]], <8 x half> [[B:%.*]], <8 x i1> [[TMP1]]) #[[ATTR3]]
+// CHECK-STRICT-NEXT:    ret <8 x half> [[TMP2]]
 //
 float16x8_t test_vminnmaq_m_f16(float16x8_t a, float16x8_t b, mve_pred16_t p)
 {
@@ -54,12 +77,19 @@ float16x8_t test_vminnmaq_m_f16(float16x8_t a, float16x8_t b, mve_pred16_t p)
 #endif /* POLYMORPHIC */
 }
 
-// CHECK-LABEL: @test_vminnmaq_m_f32(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = zext i16 [[P:%.*]] to i32
-// CHECK-NEXT:    [[TMP1:%.*]] = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 [[TMP0]])
-// CHECK-NEXT:    [[TMP2:%.*]] = call <4 x float> @llvm.arm.mve.vminnma.predicated.v4f32.v4i1(<4 x float> [[A:%.*]], <4 x float> [[B:%.*]], <4 x i1> [[TMP1]])
-// CHECK-NEXT:    ret <4 x float> [[TMP2]]
+// CHECK-NOSTRICT-LABEL: @test_vminnmaq_m_f32(
+// CHECK-NOSTRICT-NEXT:  entry:
+// CHECK-NOSTRICT-NEXT:    [[TMP0:%.*]] = zext i16 [[P:%.*]] to i32
+// CHECK-NOSTRICT-NEXT:    [[TMP1:%.*]] = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 [[TMP0]])
+// CHECK-NOSTRICT-NEXT:    [[TMP2:%.*]] = call <4 x float> @llvm.arm.mve.vminnma.predicated.v4f32.v4i1(<4 x float> [[A:%.*]], <4 x float> [[B:%.*]], <4 x i1> [[TMP1]])
+// CHECK-NOSTRICT-NEXT:    ret <4 x float> [[TMP2]]
+//
+// CHECK-STRICT-LABEL: @test_vminnmaq_m_f32(
+// CHECK-STRICT-NEXT:  entry:
+// CHECK-STRICT-NEXT:    [[TMP0:%.*]] = zext i16 [[P:%.*]] to i32
+// CHECK-STRICT-NEXT:    [[TMP1:%.*]] = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 [[TMP0]]) #[[ATTR3]]
+// CHECK-STRICT-NEXT:    [[TMP2:%.*]] = call <4 x float> @llvm.arm.mve.vminnma.predicated.v4f32.v4i1(<4 x float> [[A:%.*]], <4 x float> [[B:%.*]], <4 x i1> [[TMP1]]) #[[ATTR3]]
+// CHECK-STRICT-NEXT:    ret <4 x float> [[TMP2]]
 //
 float32x4_t test_vminnmaq_m_f32(float32x4_t a, float32x4_t b, mve_pred16_t p)
 {
@@ -69,3 +99,5 @@ float32x4_t test_vminnmaq_m_f32(float32x4_t a, float32x4_t b, mve_pred16_t p)
     return vminnmaq_m_f32(a, b, p);
 #endif /* POLYMORPHIC */
 }
+//// NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+// CHECK: {{.*}}
diff --git a/clang/test/CodeGen/arm-mve-intrinsics/vminnmq.c b/clang/test/CodeGen/arm-mve-intrinsics/vminnmq.c
index b48ff9d84b8f6..7dbad94c77674 100644
--- a/clang/test/CodeGen/arm-mve-intrinsics/vminnmq.c
+++ b/clang/test/CodeGen/arm-mve-intrinsics/vminnmq.c
@@ -1,15 +1,22 @@
 // NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py
-// RUN: %clang_cc1 -triple thumbv8.1m.main-none-none-eabi -target-feature +mve.fp -mfloat-abi hard -disable-O0-optnone -emit-llvm -o - %s | opt -S -passes=mem2reg | FileCheck %s
-// RUN: %clang_cc1 -triple thumbv8.1m.main-none-none-eabi -target-feature +mve.fp -mfloat-abi hard -disable-O0-optnone -DPOLYMORPHIC -emit-llvm -o - %s | opt -S -passes=mem2reg | FileCheck %s
+// RUN: %clang_cc1 -triple thumbv8.1m.main-none-none-eabi -target-feature +mve.fp -mfloat-abi hard -disable-O0-optnone -emit-llvm -o - %s | opt -S -passes=mem2reg | FileCheck %s --check-prefixes=CHECK,CHECK-NOSTRICT
+// RUN: %clang_cc1 -triple thumbv8.1m.main-none-none-eabi -target-feature +mve.fp -mfloat-abi hard -disable-O0-optnone -DPOLYMORPHIC -emit-llvm -o - %s | opt -S -passes=mem2reg | FileCheck %s --check-prefixes=CHECK,CHECK-NOSTRICT
+// RUN: %clang_cc1 -triple thumbv8.1m.main-none-none-eabi -target-feature +mve.fp -mfloat-abi hard -disable-O0-optnone -frounding-math -fexperimental-strict-floating-point -emit-llvm -o - %s | opt -S -passes=mem2reg | FileCheck %s --check-prefixes=CHECK,CHECK-STRICT
+// RUN: %clang_cc1 -triple thumbv8.1m.main-none-none-eabi -target-feature +mve.fp -mfloat-abi hard -disable-O0-optnone -frounding-math -fexperimental-strict-floating-point -DPOLYMORPHIC -emit-llvm -o - %s | opt -S -passes=mem2reg | FileCheck %s --check-prefixes=CHECK,CHECK-STRICT
 
 // REQUIRES: aarch64-registered-target || arm-registered-target
 
 #include <arm_mve.h>
 
-// CHECK-LABEL: @test_vminnmq_f16(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = call <8 x half> @llvm.minnum.v8f16(<8 x half> [[A:%.*]], <8 x half> [[B:%.*]])
-// CHECK-NEXT:    ret <8 x half> [[TMP0]]
+// CHECK-NOSTRICT-LABEL: @test_vminnmq_f16(
+// CHECK-NOSTRICT-NEXT:  entry:
+// CHECK-NOSTRICT-NEXT:    [[TMP0:%.*]] = call <8 x half> @llvm.minnum.v8f16(<8 x half> [[A:%.*]], <8 x half> [[B:%.*]])
+// CHECK-NOSTRICT-NEXT:    ret <8 x half> [[TMP0]]
+//
+// CHECK-STRICT-LABEL: @test_vminnmq_f16(
+// CHECK-STRICT-NEXT:  entry:
+// CHECK-STRICT-NEXT:    [[TMP0:%.*]] = call <8 x half> @llvm.arm.mve.vminnm.v8f16(<8 x half> [[A:%.*]], <8 x half> [[B:%.*]]) #[[ATTR2:[0-9]+]]
+// CHECK-STRICT-NEXT:    ret <8 x half> [[TMP0]]
 //
 float16x8_t test_vminnmq_f16(float16x8_t a, float16x8_t b)
 {
@@ -20,10 +27,15 @@ float16x8_t test_vminnmq_f16(float16x8_t a, float16x8_t b)
 #endif /* POLYMORPHIC */
 }
 
-// CHECK-LABEL: @test_vminnmq_f32(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = call <4 x float> @llvm.minnum.v4f32(<4 x float> [[A:%.*]], <4 x float> [[B:%.*]])
-// CHECK-NEXT:    ret <4 x float> [[TMP0]]
+// CHECK-NOSTRICT-LABEL: @test_vminnmq_f32(
+// CHECK-NOSTRICT-NEXT:  entry:
+// CHECK-NOSTRICT-NEXT:    [[TMP0:%.*]] = call <4 x float> @llvm.minnum.v4f32(<4 x float> [[A:%.*]], <4 x float> [[B:%.*]])
+// CHECK-NOSTRICT-NEXT:    ret <4 x float> [[TMP0]]
+//
+// CHECK-STRICT-LABEL: @test_vminnmq_f32(
+// CHECK-STRICT-NEXT:  entry:
+// CHECK-STRICT-NEXT:    [[TMP0:%.*]] = call <4 x float> @llvm.arm.mve.vminnm.v4f32(<4 x float> [[A:%.*]], <4 x float> [[B:%.*]]) #[[ATTR2]]
+// CHECK-STRICT-NEXT:    ret <4 x float> [[TMP0]]
 //
 float32x4_t test_vminnmq_f32(float32x4_t a, float32x4_t b)
 {
@@ -34,12 +46,19 @@ float32x4_t test_vminnmq_f32(float32x4_t a, float32x4_t b)
 #endif /* POLYMORPHIC */
 }
 
-// CHECK-LABEL: @test_vminnmq_m_f16(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = zext i16 [[P:%.*]] to i32
-// CHECK-NEXT:    [[TMP1:%.*]] = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 [[TMP0]])
-// CHECK-NEXT:    [[TMP2:%.*]] = call <8 x half> @llvm.arm.mve.min.predicated.v8f16.v8i1(<8 x half> [[A:%.*]], <8 x half> [[B:%.*]], i32 0, <8 x i1> [[TMP1]], <8 x half> [[INACTIVE:%.*]])
-// CHECK-NEXT:    ret <8 x half> [[TMP2]]
+// CHECK-NOSTRICT-LABEL: @test_vminnmq_m_f16(
+// CHECK-NOSTRICT-NEXT:  entry:
+// CHECK-NOSTRICT-NEXT:    [[TMP0:%.*]] = zext i16 [[P:%.*]] to i32
+// CHECK-NOSTRICT-NEXT:    [[TMP1:%.*]] = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 [[TMP0]])
+// CHECK-NOSTRICT-NEXT:    [[TMP2:%.*]] = call <8 x half> @llvm.arm.mve.min.predicated.v8f16.v8i1(<8 x half> [[A:%.*]], <8 x half> [[B:%.*]], i32 0, <8 x i1> [[TMP1]], <8 x half> [[INACTIVE:%.*]])
+// CHECK-NOSTRICT-NEXT:    ret <8 x half> [[TMP2]]
+//
+// CHECK-STRICT-LABEL: @test_vminnmq_m_f16(
+// CHECK-STRICT-NEXT:  entry:
+// CHECK-STRICT-NEXT:    [[TMP0:%.*]] = zext i16 [[P:%.*]] to i32
+// CHECK-STRICT-NEXT:    [[TMP1:%.*]] = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 [[TMP0]]) #[[ATTR2]]
+// CHECK-STRICT-NEXT:    [[TMP2:%.*]] = call <8 x half> @llvm.arm.mve.min.predicated.v8f16.v8i1(<8 x half> [[A:%.*]], <8 x half> [[B:%.*]], i32 0, <8 x i1> [[TMP1]], <8 x half> [[INACTIVE:%.*]]) #[[ATTR2]]
+// CHECK-STRICT-NEXT:    ret <8 x half> [[TMP2]]
 //
 float16x8_t test_vminnmq_m_f16(float16x8_t inactive, float16x8_t a, float16x8_t b, mve_pred16_t p)
 {
@@ -50,12 +69,19 @@ float16x8_t test_vminnmq_m_f16(float16x8_t inactive, float16x8_t a, float16x8_t
 #endif /* POLYMORPHIC */
 }
 
-// CHECK-LABEL: @test_vminnmq_m_f32(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = zext i16 [[P:%.*]] to i32
-// CHECK-NEXT:    [[TMP1:%.*]] = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 [[TMP0]])
-// CHECK-NEXT:    [[TMP2:%.*]] = call <4 x float> @llvm.arm.mve.min.predicated.v4f32.v4i1(<4 x float> [[A:%.*]], <4 x float> [[B:%.*]], i32 0, <4 x i1> [[TMP1]], <4 x float> [[INACTIVE:%.*]])
-// CHECK-NEXT:    ret <4 x float> [[TMP2]]
+// CHECK-NOSTRICT-LABEL: @test_vminnmq_m_f32(
+// CHECK-NOSTRICT-NEXT:  entry:
+// CHECK-NOSTRICT-NEXT:    [[TMP0:%.*]] = zext i16 [[P:%.*]] to i32
+// CHECK-NOSTRICT-NEXT:    [[TMP1:%.*]] = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 [[TMP0]])
+// CHECK-NOSTRICT-NEXT:    [[TMP2:%.*]] = call <4 x float> @llvm.arm.mve.min.predicated.v4f32.v4i1(<4 x float> [[A:%.*]], <4 x float> [[B:%.*]], i32 0, <4 x i1> [[TMP1]], <4 x float> [[INACTIVE:%.*]])
+// CHECK-NOSTRICT-NEXT:    ret <4 x float> [[TMP2]]
+//
+// CHECK-STRICT-LABEL: @test_vminnmq_m_f32(
+// CHECK-STRICT-NEXT:  entry:
+// CHECK-STRICT-NEXT:    [[TMP0:%.*]] = zext i16 [[P:%.*]] to i32
+// CHECK-STRICT-NEXT:    [[TMP1:%.*]] = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 [[TMP0]]) #[[ATTR2]]
+// CHECK-STRICT-NEXT:    [[TMP2:%.*]] = call <4 x float> @llvm.arm.mve.min.predicated.v4f32.v4i1(<4 x float> [[A:%.*]], <4 x float> [[B:%.*]], i32 0, <4 x i1> [[TMP1]], <4 x float> [[INACTIVE:%.*]]) #[[ATTR2]]
+// CHECK-STRICT-NEXT:    ret <4 x float> [[TMP2]]
 //
 float32x4_t test_vminnmq_m_f32(float32x4_t inactive, float32x4_t a, float32x4_t b, mve_pred16_t p)
 {
@@ -66,12 +92,19 @@ float32x4_t test_vminnmq_m_f32(float32x4_t inactive, float32x4_t a, float32x4_t
 #endif /* POLYMORPHIC */
 }
 
-// CHECK-LABEL: @test_vminnmq_x_f16(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = zext i16 [[P:%.*]] to i32
-// CHECK-NEXT:    [[TMP1:%.*]] = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 [[TMP0]])
-// CHECK-NEXT:    [[TMP2:%.*]] = call <8 x half> @llvm.arm.mve.min.predicated.v8f16.v8i1(<8 x half> [[A:%.*]], <8 x half> [[B:%.*]], i32 0, <8 x i1> [[TMP1]], <8 x half> undef)
-// CHECK-NEXT:    ret <8 x half> [[TMP2]]
+// CHECK-NOSTRICT-LABEL: @test_vminnmq_x_f16(
+// CHECK-NOSTRICT-NEXT:  entry:
+// CHECK-NOSTRICT-NEXT:    [[TMP0:%.*]] = zext i16 [[P:%.*]] to i32
+// CHECK-NOSTRICT-NEXT:    [[TMP1:%.*]] = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 [[TMP0]])
+// CHECK-NOSTRICT-NEXT:    [[TMP2:%.*]] = call <8 x half> @llvm.arm.mve.min.predicated.v8f16.v8i1(<8 x half> [[A:%.*]], <8 x half> [[B:%.*]], i32 0, <8 x i1> [[TMP1]], <8 x half> undef)
+// CHECK-NOSTRICT-NEXT:    ret <8 x half> [[TMP2]]
+//
+// CHECK-STRICT-LABEL: @test_vminnmq_x_f16(
+// CHECK-STRICT-NEXT:  entry:
+// CHECK-STRICT-NEXT:    [[TMP0:%.*]] = zext i16 [[P:%.*]] to i32
+// CHECK-STRICT-NEXT:    [[TMP1:%.*]] = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 [[TMP0]]) #[[ATTR2]]
+// CHECK-STRICT-NEXT:    [[TMP2:%.*]] = call <8 x half> @llvm.arm.mve.min.predicated.v8f16.v8i1(<8 x half> [[A:%.*]], <8 x half> [[B:%.*]], i32 0, <8 x i1> [[TMP1]], <8 x half> undef) #[[ATTR2]]
+// CHECK-STRICT-NEXT:    ret <8 x half> [[TMP2]]
 //
 float16x8_t test_vminnmq_x_f16(float16x8_t a, float16x8_t b, mve_pred16_t p)
 {
@@ -82,12 +115,19 @@ float16x8_t test_vminnmq_x_f16(float16x8_t a, float16x8_t b, mve_pred16_t p)
 #endif /* POLYMORPHIC */
 }
 
-// CHECK-LABEL: @test_vminnmq_x_f32(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = zext i16 [[P:%.*]] to i32
-// CHECK-NEXT:    [[TMP1:%.*]] = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 [[TMP0]])
-// CHECK-NEXT:    [[TMP2:%.*]] = call <4 x float> @llvm.arm.mve.min.predicated.v4f32.v4i1(<4 x float> [[A:%.*]], <4 x float> [[B:%.*]], i32 0, <4 x i1> [[TMP1]], <4 x float> undef)
-// CHECK-NEXT:    ret <4 x float> [[TMP2]]
+// CHECK-NOSTRICT-LABEL: @test_vminnmq_x_f32(
+// CHECK-NOSTRICT-NEXT:  entry:
+// CHECK-NOSTRICT-NEXT:    [[TMP0:%.*]] = zext i16 [[P:%.*]] to i32
+// CHECK-NOSTRICT-NEXT:    [[TMP1:%.*]] = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 [[TMP0]])
+// CHECK-NOSTRICT-NEXT:    [[TMP2:%.*]] = call <4 x float> @llvm.arm.mve.min.predicated.v4f32.v4i1(<4 x float> [[A:%.*]], <4 x float> [[B:%.*]], i32 0, <4 x i1> [[TMP1]], <4 x float> undef)
+// CHECK-NOSTRICT-NEXT:    ret <4 x float> [[TMP2]]
+//
+// CHECK-STRICT-LABEL: @test_vminnmq_x_f32(
+// CHECK-STRICT-NEXT:  entry:
+// CHECK-STRICT-NEXT:    [[TMP0:%.*]] = zext i16 [[P:%.*]] to i32
+// CHECK-STRICT-NEXT:    [[TMP1:%.*]] = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 [[TMP0]]) #[[ATTR2]]
+// CHECK-STRICT-NEXT:    [[TMP2:%.*]] = call <4 x float> @llvm.arm.mve.min.predicated.v4f32.v4i1(<4 x float> [[A:%.*]], <4 x float> [[B:%.*]], i32 0, <4 x i1> [[TMP1]], <4 x float> undef) #[[ATTR2]]
+// CHECK-STRICT-NEXT:    ret <4 x float> [[TMP2]]
 //
 float32x4_t test_vminnmq_x_f32(float32x4_t a, float32x4_t b, mve_pred16_t p)
 {
@@ -97,3 +137,5 @@ float32x4_t test_vminnmq_x_f32(float32x4_t a, float32x4_t b, mve_pred16_t p)
     return vminnmq_x_f32(a, b, p);
 #endif /* POLYMORPHIC */
 }
+//// NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+// CHECK: {{.*}}
diff --git a/clang/test/CodeGen/distributed-thin-lto/memprof-pgho.cpp b/clang/test/CodeGen/distributed-thin-lto/memprof-pgho.cpp
index ed05962846aff..317efd1b3a138 100644
--- a/clang/test/CodeGen/distributed-thin-lto/memprof-pgho.cpp
+++ b/clang/test/CodeGen/distributed-thin-lto/memprof-pgho.cpp
@@ -1,6 +1,8 @@
 // Test end-to-end ThinLTO optimization pipeline with PGHO, that it does not
 // interfere with other allocation instrumentation features.
 //
+// REQUIRES: x86-registered-target
+//
 // RUN: split-file %s %t
 // RUN: llvm-profdata merge %t/memprof.yaml -o %t/use.memprofdata
 //
diff --git a/clang/test/CodeGenHLSL/BasicFeatures/MatrixElementTypeCast.hlsl b/clang/test/CodeGenHLSL/BasicFeatures/MatrixElementTypeCast.hlsl
new file mode 100644
index 0000000000000..3bd7636212862
--- /dev/null
+++ b/clang/test/CodeGenHLSL/BasicFeatures/MatrixElementTypeCast.hlsl
@@ -0,0 +1,219 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 6
+// RUN: %clang_cc1 -finclude-default-header -triple dxil-pc-shadermodel6.3-library -x hlsl -emit-llvm -disable-llvm-passes -fnative-half-type -fnative-int16-type -o - %s | FileCheck %s
+
+
+// CHECK-LABEL: define hidden noundef <6 x i32> @_Z22elementwise_type_cast0u11matrix_typeILm3ELm2EfE(
+// CHECK-SAME: <6 x float> noundef nofpclass(nan inf) [[F32:%.*]]) #[[ATTR0:[0-9]+]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[F32_ADDR:%.*]] = alloca [6 x float], align 4
+// CHECK-NEXT:    [[I32:%.*]] = alloca [6 x i32], align 4
+// CHECK-NEXT:    store <6 x float> [[F32]], ptr [[F32_ADDR]], align 4
+// CHECK-NEXT:    [[TMP0:%.*]] = load <6 x float>, ptr [[F32_ADDR]], align 4
+// CHECK-NEXT:    [[CONV:%.*]] = fptosi <6 x float> [[TMP0]] to <6 x i32>
+// CHECK-NEXT:    store <6 x i32> [[CONV]], ptr [[I32]], align 4
+// CHECK-NEXT:    [[TMP1:%.*]] = load <6 x i32>, ptr [[I32]], align 4
+// CHECK-NEXT:    ret <6 x i32> [[TMP1]]
+//
+int3x2 elementwise_type_cast0(float3x2 f32) {
+    int3x2 i32 = (int3x2)f32;
+    return i32;
+}
+
+// CHECK-LABEL: define hidden noundef <6 x i32> @_Z22elementwise_type_cast1u11matrix_typeILm3ELm2EsE(
+// CHECK-SAME: <6 x i16> noundef [[I16_32:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[I16_32_ADDR:%.*]] = alloca [6 x i16], align 2
+// CHECK-NEXT:    [[I32:%.*]] = alloca [6 x i32], align 4
+// CHECK-NEXT:    store <6 x i16> [[I16_32]], ptr [[I16_32_ADDR]], align 2
+// CHECK-NEXT:    [[TMP0:%.*]] = load <6 x i16>, ptr [[I16_32_ADDR]], align 2
+// CHECK-NEXT:    [[CONV:%.*]] = sext <6 x i16> [[TMP0]] to <6 x i32>
+// CHECK-NEXT:    store <6 x i32> [[CONV]], ptr [[I32]], align 4
+// CHECK-NEXT:    [[TMP1:%.*]] = load <6 x i32>, ptr [[I32]], align 4
+// CHECK-NEXT:    ret <6 x i32> [[TMP1]]
+//
+int3x2 elementwise_type_cast1(int16_t3x2 i16_32) {
+    int3x2 i32 = (int3x2)i16_32;
+    return i32;
+}
+
+// CHECK-LABEL: define hidden noundef <6 x i32> @_Z22elementwise_type_cast2u11matrix_typeILm3ELm2ElE(
+// CHECK-SAME: <6 x i64> noundef [[I64_32:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[I64_32_ADDR:%.*]] = alloca [6 x i64], align 8
+// CHECK-NEXT:    [[I32:%.*]] = alloca [6 x i32], align 4
+// CHECK-NEXT:    store <6 x i64> [[I64_32]], ptr [[I64_32_ADDR]], align 8
+// CHECK-NEXT:    [[TMP0:%.*]] = load <6 x i64>, ptr [[I64_32_ADDR]], align 8
+// CHECK-NEXT:    [[CONV:%.*]] = trunc <6 x i64> [[TMP0]] to <6 x i32>
+// CHECK-NEXT:    store <6 x i32> [[CONV]], ptr [[I32]], align 4
+// CHECK-NEXT:    [[TMP1:%.*]] = load <6 x i32>, ptr [[I32]], align 4
+// CHECK-NEXT:    ret <6 x i32> [[TMP1]]
+//
+int3x2 elementwise_type_cast2(int64_t3x2 i64_32) {
+    int3x2 i32 = (int3x2)i64_32;
+    return i32;
+}
+
+// CHECK-LABEL: define hidden noundef <6 x i16> @_Z22elementwise_type_cast3u11matrix_typeILm2ELm3EDhE(
+// CHECK-SAME: <6 x half> noundef nofpclass(nan inf) [[H23:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[H23_ADDR:%.*]] = alloca [6 x half], align 2
+// CHECK-NEXT:    [[I23:%.*]] = alloca [6 x i16], align 2
+// CHECK-NEXT:    store <6 x half> [[H23]], ptr [[H23_ADDR]], align 2
+// CHECK-NEXT:    [[TMP0:%.*]] = load <6 x half>, ptr [[H23_ADDR]], align 2
+// CHECK-NEXT:    [[CONV:%.*]] = fptosi <6 x half> [[TMP0]] to <6 x i16>
+// CHECK-NEXT:    store <6 x i16> [[CONV]], ptr [[I23]], align 2
+// CHECK-NEXT:    [[TMP1:%.*]] = load <6 x i16>, ptr [[I23]], align 2
+// CHECK-NEXT:    ret <6 x i16> [[TMP1]]
+//
+int16_t2x3 elementwise_type_cast3(half2x3 h23) {
+    int16_t2x3 i23 = (int16_t2x3)h23;
+    return i23;
+}
+
+// CHECK-LABEL: define hidden noundef <6 x i32> @_Z22elementwise_type_cast4u11matrix_typeILm3ELm2EdE(
+// CHECK-SAME: <6 x double> noundef nofpclass(nan inf) [[D32:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[D32_ADDR:%.*]] = alloca [6 x double], align 8
+// CHECK-NEXT:    [[I32:%.*]] = alloca [6 x i32], align 4
+// CHECK-NEXT:    store <6 x double> [[D32]], ptr [[D32_ADDR]], align 8
+// CHECK-NEXT:    [[TMP0:%.*]] = load <6 x double>, ptr [[D32_ADDR]], align 8
+// CHECK-NEXT:    [[CONV:%.*]] = fptosi <6 x double> [[TMP0]] to <6 x i32>
+// CHECK-NEXT:    store <6 x i32> [[CONV]], ptr [[I32]], align 4
+// CHECK-NEXT:    [[TMP1:%.*]] = load <6 x i32>, ptr [[I32]], align 4
+// CHECK-NEXT:    ret <6 x i32> [[TMP1]]
+//
+int3x2 elementwise_type_cast4(double3x2 d32) {
+    int3x2 i32 = (int3x2)d32;
+    return i32;
+}
+
+// CHECK-LABEL: define hidden void @_Z5call2v(
+// CHECK-SAME: ) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[A:%.*]] = alloca [2 x [1 x i32]], align 4
+// CHECK-NEXT:    [[B:%.*]] = alloca [2 x i32], align 4
+// CHECK-NEXT:    [[AGG_TEMP:%.*]] = alloca [2 x [1 x i32]], align 4
+// CHECK-NEXT:    [[FLATCAST_TMP:%.*]] = alloca <2 x i32>, align 4
+// CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[A]], ptr align 4 @__const._Z5call2v.A, i32 8, i1 false)
+// CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[AGG_TEMP]], ptr align 4 [[A]], i32 8, i1 false)
+// CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds [2 x [1 x i32]], ptr [[AGG_TEMP]], i32 0, i32 0, i32 0
+// CHECK-NEXT:    [[GEP1:%.*]] = getelementptr inbounds [2 x [1 x i32]], ptr [[AGG_TEMP]], i32 0, i32 1, i32 0
+// CHECK-NEXT:    [[TMP0:%.*]] = load <2 x i32>, ptr [[FLATCAST_TMP]], align 4
+// CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr [[GEP]], align 4
+// CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x i32> [[TMP0]], i32 [[TMP1]], i64 0
+// CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr [[GEP1]], align 4
+// CHECK-NEXT:    [[TMP4:%.*]] = insertelement <2 x i32> [[TMP2]], i32 [[TMP3]], i64 1
+// CHECK-NEXT:    store <2 x i32> [[TMP4]], ptr [[B]], align 4
+// CHECK-NEXT:    ret void
+//
+void call2() {
+  int A[2][1] = {{1},{2}};
+  int2x1 B = (int2x1)A;
+}
+
+struct S {
+  int X;
+  float Y;
+};
+
+// CHECK-LABEL: define hidden void @_Z5call3v(
+// CHECK-SAME: ) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[S:%.*]] = alloca [[STRUCT_S:%.*]], align 1
+// CHECK-NEXT:    [[A:%.*]] = alloca [2 x i32], align 4
+// CHECK-NEXT:    [[AGG_TEMP:%.*]] = alloca [[STRUCT_S]], align 1
+// CHECK-NEXT:    [[FLATCAST_TMP:%.*]] = alloca <2 x i32>, align 4
+// CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 1 [[S]], ptr align 1 @__const._Z5call3v.s, i32 8, i1 false)
+// CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 1 [[AGG_TEMP]], ptr align 1 [[S]], i32 8, i1 false)
+// CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[AGG_TEMP]], i32 0, i32 0
+// CHECK-NEXT:    [[GEP1:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[AGG_TEMP]], i32 0, i32 1
+// CHECK-NEXT:    [[TMP0:%.*]] = load <2 x i32>, ptr [[FLATCAST_TMP]], align 4
+// CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr [[GEP]], align 4
+// CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x i32> [[TMP0]], i32 [[TMP1]], i64 0
+// CHECK-NEXT:    [[TMP3:%.*]] = load float, ptr [[GEP1]], align 4
+// CHECK-NEXT:    [[CONV:%.*]] = fptosi float [[TMP3]] to i32
+// CHECK-NEXT:    [[TMP4:%.*]] = insertelement <2 x i32> [[TMP2]], i32 [[CONV]], i64 1
+// CHECK-NEXT:    store <2 x i32> [[TMP4]], ptr [[A]], align 4
+// CHECK-NEXT:    ret void
+//
+void call3() {
+  S s = {1, 2.0};
+  int2x1 A = (int2x1)s;
+}
+
+struct BFields {
+  double D;
+  int E: 15;
+  int : 8;
+  float F;
+};
+
+struct Derived : BFields {
+  int G;
+};
+
+// CHECK-LABEL: define hidden void @_Z5call47Derived(
+// CHECK-SAME: ptr noundef byval([[STRUCT_DERIVED:%.*]]) align 1 [[D:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[A:%.*]] = alloca [4 x i32], align 4
+// CHECK-NEXT:    [[AGG_TEMP:%.*]] = alloca [[STRUCT_DERIVED]], align 1
+// CHECK-NEXT:    [[FLATCAST_TMP:%.*]] = alloca <4 x i32>, align 4
+// CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 1 [[AGG_TEMP]], ptr align 1 [[D]], i32 19, i1 false)
+// CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds [[STRUCT_DERIVED]], ptr [[AGG_TEMP]], i32 0, i32 0
+// CHECK-NEXT:    [[E:%.*]] = getelementptr inbounds nuw [[STRUCT_BFIELDS:%.*]], ptr [[GEP]], i32 0, i32 1
+// CHECK-NEXT:    [[GEP1:%.*]] = getelementptr inbounds [[STRUCT_DERIVED]], ptr [[AGG_TEMP]], i32 0, i32 0, i32 0
+// CHECK-NEXT:    [[GEP2:%.*]] = getelementptr inbounds [[STRUCT_DERIVED]], ptr [[AGG_TEMP]], i32 0, i32 0, i32 2
+// CHECK-NEXT:    [[GEP3:%.*]] = getelementptr inbounds [[STRUCT_DERIVED]], ptr [[AGG_TEMP]], i32 0, i32 1
+// CHECK-NEXT:    [[TMP0:%.*]] = load <4 x i32>, ptr [[FLATCAST_TMP]], align 4
+// CHECK-NEXT:    [[TMP1:%.*]] = load double, ptr [[GEP1]], align 8
+// CHECK-NEXT:    [[CONV:%.*]] = fptosi double [[TMP1]] to i32
+// CHECK-NEXT:    [[TMP2:%.*]] = insertelement <4 x i32> [[TMP0]], i32 [[CONV]], i64 0
+// CHECK-NEXT:    [[TMP3:%.*]] = load float, ptr [[GEP2]], align 4
+// CHECK-NEXT:    [[CONV4:%.*]] = fptosi float [[TMP3]] to i32
+// CHECK-NEXT:    [[TMP4:%.*]] = insertelement <4 x i32> [[TMP2]], i32 [[CONV4]], i64 1
+// CHECK-NEXT:    [[BF_LOAD:%.*]] = load i24, ptr [[E]], align 1
+// CHECK-NEXT:    [[BF_SHL:%.*]] = shl i24 [[BF_LOAD]], 9
+// CHECK-NEXT:    [[BF_ASHR:%.*]] = ashr i24 [[BF_SHL]], 9
+// CHECK-NEXT:    [[BF_CAST:%.*]] = sext i24 [[BF_ASHR]] to i32
+// CHECK-NEXT:    [[TMP5:%.*]] = insertelement <4 x i32> [[TMP4]], i32 [[BF_CAST]], i64 2
+// CHECK-NEXT:    [[TMP6:%.*]] = load i32, ptr [[GEP3]], align 4
+// CHECK-NEXT:    [[TMP7:%.*]] = insertelement <4 x i32> [[TMP5]], i32 [[TMP6]], i64 3
+// CHECK-NEXT:    store <4 x i32> [[TMP7]], ptr [[A]], align 4
+// CHECK-NEXT:    ret void
+//
+void call4(Derived D) {
+  int2x2 A = (int2x2)D;
+}
+
+// CHECK-LABEL: define hidden noundef nofpclass(nan inf) <4 x float> @_Z5call5Dv4_f(
+// CHECK-SAME: <4 x float> noundef nofpclass(nan inf) [[V:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[V_ADDR:%.*]] = alloca <4 x float>, align 16
+// CHECK-NEXT:    [[M:%.*]] = alloca [4 x float], align 4
+// CHECK-NEXT:    [[HLSL_EWCAST_SRC:%.*]] = alloca <4 x float>, align 16
+// CHECK-NEXT:    [[FLATCAST_TMP:%.*]] = alloca <4 x float>, align 4
+// CHECK-NEXT:    store <4 x float> [[V]], ptr [[V_ADDR]], align 16
+// CHECK-NEXT:    [[TMP0:%.*]] = load <4 x float>, ptr [[V_ADDR]], align 16
+// CHECK-NEXT:    store <4 x float> [[TMP0]], ptr [[HLSL_EWCAST_SRC]], align 16
+// CHECK-NEXT:    [[VECTOR_GEP:%.*]] = getelementptr inbounds <4 x float>, ptr [[HLSL_EWCAST_SRC]], i32 0
+// CHECK-NEXT:    [[TMP1:%.*]] = load <4 x float>, ptr [[FLATCAST_TMP]], align 4
+// CHECK-NEXT:    [[TMP2:%.*]] = load <4 x float>, ptr [[VECTOR_GEP]], align 16
+// CHECK-NEXT:    [[VECEXT:%.*]] = extractelement <4 x float> [[TMP2]], i32 0
+// CHECK-NEXT:    [[TMP3:%.*]] = insertelement <4 x float> [[TMP1]], float [[VECEXT]], i64 0
+// CHECK-NEXT:    [[TMP4:%.*]] = load <4 x float>, ptr [[VECTOR_GEP]], align 16
+// CHECK-NEXT:    [[VECEXT1:%.*]] = extractelement <4 x float> [[TMP4]], i32 2
+// CHECK-NEXT:    [[TMP5:%.*]] = insertelement <4 x float> [[TMP3]], float [[VECEXT1]], i64 1
+// CHECK-NEXT:    [[TMP6:%.*]] = load <4 x float>, ptr [[VECTOR_GEP]], align 16
+// CHECK-NEXT:    [[VECEXT2:%.*]] = extractelement <4 x float> [[TMP6]], i32 1
+// CHECK-NEXT:    [[TMP7:%.*]] = insertelement <4 x float> [[TMP5]], float [[VECEXT2]], i64 2
+// CHECK-NEXT:    [[TMP8:%.*]] = load <4 x float>, ptr [[VECTOR_GEP]], align 16
+// CHECK-NEXT:    [[VECEXT3:%.*]] = extractelement <4 x float> [[TMP8]], i32 3
+// CHECK-NEXT:    [[TMP9:%.*]] = insertelement <4 x float> [[TMP7]], float [[VECEXT3]], i64 3
+// CHECK-NEXT:    store <4 x float> [[TMP9]], ptr [[M]], align 4
+// CHECK-NEXT:    [[TMP10:%.*]] = load <4 x float>, ptr [[M]], align 4
+// CHECK-NEXT:    ret <4 x float> [[TMP10]]
+//
+float2x2 call5(float4 v) {
+    float2x2 m = (float2x2)v;
+    return m;
+}
diff --git a/clang/test/CodeGenHLSL/BasicFeatures/MatrixExplicitTruncation.hlsl b/clang/test/CodeGenHLSL/BasicFeatures/MatrixExplicitTruncation.hlsl
new file mode 100644
index 0000000000000..f3c4bc496d5a4
--- /dev/null
+++ b/clang/test/CodeGenHLSL/BasicFeatures/MatrixExplicitTruncation.hlsl
@@ -0,0 +1,156 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 6
+// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.7-library -disable-llvm-passes -emit-llvm -finclude-default-header -o - %s | FileCheck %s
+
+// CHECK-LABEL: define hidden noundef <12 x i32> @_Z10trunc_castu11matrix_typeILm4ELm4EiE(
+// CHECK-SAME: <16 x i32> noundef [[I44:%.*]]) #[[ATTR0:[0-9]+]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[I44_ADDR:%.*]] = alloca [16 x i32], align 4
+// CHECK-NEXT:    [[I34:%.*]] = alloca [12 x i32], align 4
+// CHECK-NEXT:    store <16 x i32> [[I44]], ptr [[I44_ADDR]], align 4
+// CHECK-NEXT:    [[TMP0:%.*]] = load <16 x i32>, ptr [[I44_ADDR]], align 4
+// CHECK-NEXT:    [[TRUNC:%.*]] = shufflevector <16 x i32> [[TMP0]], <16 x i32> poison, <12 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
+// CHECK-NEXT:    store <12 x i32> [[TRUNC]], ptr [[I34]], align 4
+// CHECK-NEXT:    [[TMP1:%.*]] = load <12 x i32>, ptr [[I34]], align 4
+// CHECK-NEXT:    ret <12 x i32> [[TMP1]]
+//
+ int3x4 trunc_cast(int4x4 i44) {
+    int3x4 i34 = (int3x4)i44;
+    return i34;
+}
+
+// CHECK-LABEL: define hidden noundef <12 x i32> @_Z11trunc_cast0u11matrix_typeILm4ELm4EiE(
+// CHECK-SAME: <16 x i32> noundef [[I44:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[I44_ADDR:%.*]] = alloca [16 x i32], align 4
+// CHECK-NEXT:    [[I43:%.*]] = alloca [12 x i32], align 4
+// CHECK-NEXT:    store <16 x i32> [[I44]], ptr [[I44_ADDR]], align 4
+// CHECK-NEXT:    [[TMP0:%.*]] = load <16 x i32>, ptr [[I44_ADDR]], align 4
+// CHECK-NEXT:    [[TRUNC:%.*]] = shufflevector <16 x i32> [[TMP0]], <16 x i32> poison, <12 x i32> <i32 0, i32 1, i32 2, i32 4, i32 5, i32 6, i32 8, i32 9, i32 10, i32 12, i32 13, i32 14>
+// CHECK-NEXT:    store <12 x i32> [[TRUNC]], ptr [[I43]], align 4
+// CHECK-NEXT:    [[TMP1:%.*]] = load <12 x i32>, ptr [[I43]], align 4
+// CHECK-NEXT:    ret <12 x i32> [[TMP1]]
+//
+ int4x3 trunc_cast0(int4x4 i44) {
+    int4x3 i43 = (int4x3)i44;
+    return i43;
+}
+
+// CHECK-LABEL: define hidden noundef <9 x i32> @_Z11trunc_cast1u11matrix_typeILm4ELm4EiE(
+// CHECK-SAME: <16 x i32> noundef [[I44:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[I44_ADDR:%.*]] = alloca [16 x i32], align 4
+// CHECK-NEXT:    [[I33:%.*]] = alloca [9 x i32], align 4
+// CHECK-NEXT:    store <16 x i32> [[I44]], ptr [[I44_ADDR]], align 4
+// CHECK-NEXT:    [[TMP0:%.*]] = load <16 x i32>, ptr [[I44_ADDR]], align 4
+// CHECK-NEXT:    [[TRUNC:%.*]] = shufflevector <16 x i32> [[TMP0]], <16 x i32> poison, <9 x i32> <i32 0, i32 1, i32 2, i32 4, i32 5, i32 6, i32 8, i32 9, i32 10>
+// CHECK-NEXT:    store <9 x i32> [[TRUNC]], ptr [[I33]], align 4
+// CHECK-NEXT:    [[TMP1:%.*]] = load <9 x i32>, ptr [[I33]], align 4
+// CHECK-NEXT:    ret <9 x i32> [[TMP1]]
+//
+ int3x3 trunc_cast1(int4x4 i44) {
+    int3x3 i33 = (int3x3)i44;
+    return i33;
+}
+
+// CHECK-LABEL: define hidden noundef <6 x i32> @_Z11trunc_cast2u11matrix_typeILm4ELm4EiE(
+// CHECK-SAME: <16 x i32> noundef [[I44:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[I44_ADDR:%.*]] = alloca [16 x i32], align 4
+// CHECK-NEXT:    [[I32:%.*]] = alloca [6 x i32], align 4
+// CHECK-NEXT:    store <16 x i32> [[I44]], ptr [[I44_ADDR]], align 4
+// CHECK-NEXT:    [[TMP0:%.*]] = load <16 x i32>, ptr [[I44_ADDR]], align 4
+// CHECK-NEXT:    [[TRUNC:%.*]] = shufflevector <16 x i32> [[TMP0]], <16 x i32> poison, <6 x i32> <i32 0, i32 1, i32 4, i32 5, i32 8, i32 9>
+// CHECK-NEXT:    store <6 x i32> [[TRUNC]], ptr [[I32]], align 4
+// CHECK-NEXT:    [[TMP1:%.*]] = load <6 x i32>, ptr [[I32]], align 4
+// CHECK-NEXT:    ret <6 x i32> [[TMP1]]
+//
+ int3x2 trunc_cast2(int4x4 i44) {
+    int3x2 i32 = (int3x2)i44;
+    return i32;
+}
+
+// CHECK-LABEL: define hidden noundef <6 x i32> @_Z11trunc_cast3u11matrix_typeILm4ELm4EiE(
+// CHECK-SAME: <16 x i32> noundef [[I44:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[I44_ADDR:%.*]] = alloca [16 x i32], align 4
+// CHECK-NEXT:    [[I23:%.*]] = alloca [6 x i32], align 4
+// CHECK-NEXT:    store <16 x i32> [[I44]], ptr [[I44_ADDR]], align 4
+// CHECK-NEXT:    [[TMP0:%.*]] = load <16 x i32>, ptr [[I44_ADDR]], align 4
+// CHECK-NEXT:    [[TRUNC:%.*]] = shufflevector <16 x i32> [[TMP0]], <16 x i32> poison, <6 x i32> <i32 0, i32 1, i32 2, i32 4, i32 5, i32 6>
+// CHECK-NEXT:    store <6 x i32> [[TRUNC]], ptr [[I23]], align 4
+// CHECK-NEXT:    [[TMP1:%.*]] = load <6 x i32>, ptr [[I23]], align 4
+// CHECK-NEXT:    ret <6 x i32> [[TMP1]]
+//
+ int2x3 trunc_cast3(int4x4 i44) {
+    int2x3 i23 = (int2x3)i44;
+    return i23;
+}
+
+// CHECK-LABEL: define hidden noundef <4 x i32> @_Z11trunc_cast4u11matrix_typeILm4ELm4EiE(
+// CHECK-SAME: <16 x i32> noundef [[I44:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[I44_ADDR:%.*]] = alloca [16 x i32], align 4
+// CHECK-NEXT:    [[I22:%.*]] = alloca [4 x i32], align 4
+// CHECK-NEXT:    store <16 x i32> [[I44]], ptr [[I44_ADDR]], align 4
+// CHECK-NEXT:    [[TMP0:%.*]] = load <16 x i32>, ptr [[I44_ADDR]], align 4
+// CHECK-NEXT:    [[TRUNC:%.*]] = shufflevector <16 x i32> [[TMP0]], <16 x i32> poison, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
+// CHECK-NEXT:    store <4 x i32> [[TRUNC]], ptr [[I22]], align 4
+// CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr [[I22]], align 4
+// CHECK-NEXT:    ret <4 x i32> [[TMP1]]
+//
+ int2x2 trunc_cast4(int4x4 i44) {
+    int2x2 i22 = (int2x2)i44;
+    return i22;
+}
+
+// CHECK-LABEL: define hidden noundef <2 x i32> @_Z11trunc_cast5u11matrix_typeILm4ELm4EiE(
+// CHECK-SAME: <16 x i32> noundef [[I44:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[I44_ADDR:%.*]] = alloca [16 x i32], align 4
+// CHECK-NEXT:    [[I21:%.*]] = alloca [2 x i32], align 4
+// CHECK-NEXT:    store <16 x i32> [[I44]], ptr [[I44_ADDR]], align 4
+// CHECK-NEXT:    [[TMP0:%.*]] = load <16 x i32>, ptr [[I44_ADDR]], align 4
+// CHECK-NEXT:    [[TRUNC:%.*]] = shufflevector <16 x i32> [[TMP0]], <16 x i32> poison, <2 x i32> <i32 0, i32 4>
+// CHECK-NEXT:    store <2 x i32> [[TRUNC]], ptr [[I21]], align 4
+// CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i32>, ptr [[I21]], align 4
+// CHECK-NEXT:    ret <2 x i32> [[TMP1]]
+//
+ int2x1 trunc_cast5(int4x4 i44) {
+    int2x1 i21 = (int2x1)i44;
+    return i21;
+}
+
+// CHECK-LABEL: define hidden noundef i32 @_Z11trunc_cast6u11matrix_typeILm4ELm4EiE(
+// CHECK-SAME: <16 x i32> noundef [[I44:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[I44_ADDR:%.*]] = alloca [16 x i32], align 4
+// CHECK-NEXT:    [[I1:%.*]] = alloca i32, align 4
+// CHECK-NEXT:    store <16 x i32> [[I44]], ptr [[I44_ADDR]], align 4
+// CHECK-NEXT:    [[TMP0:%.*]] = load <16 x i32>, ptr [[I44_ADDR]], align 4
+// CHECK-NEXT:    [[CAST_MTRUNC:%.*]] = extractelement <16 x i32> [[TMP0]], i32 0
+// CHECK-NEXT:    store i32 [[CAST_MTRUNC]], ptr [[I1]], align 4
+// CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr [[I1]], align 4
+// CHECK-NEXT:    ret i32 [[TMP1]]
+//
+ int trunc_cast6(int4x4 i44) {
+    int i1 = (int)i44;
+    return i1;
+}
+
+// CHECK-LABEL: define hidden noundef i32 @_Z16trunc_multi_castu11matrix_typeILm4ELm4EiE(
+// CHECK-SAME: <16 x i32> noundef [[I44:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[I44_ADDR:%.*]] = alloca [16 x i32], align 4
+// CHECK-NEXT:    [[I1:%.*]] = alloca i32, align 4
+// CHECK-NEXT:    store <16 x i32> [[I44]], ptr [[I44_ADDR]], align 4
+// CHECK-NEXT:    [[TMP0:%.*]] = load <16 x i32>, ptr [[I44_ADDR]], align 4
+// CHECK-NEXT:    [[TRUNC:%.*]] = shufflevector <16 x i32> [[TMP0]], <16 x i32> poison, <12 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
+// CHECK-NEXT:    [[CAST_MTRUNC:%.*]] = extractelement <12 x i32> [[TRUNC]], i32 0
+// CHECK-NEXT:    store i32 [[CAST_MTRUNC]], ptr [[I1]], align 4
+// CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr [[I1]], align 4
+// CHECK-NEXT:    ret i32 [[TMP1]]
+//
+ int trunc_multi_cast(int4x4 i44) {
+    int i1 = (int)(int3x4)i44;
+    return i1;
+}
diff --git a/clang/test/CodeGenHLSL/BasicFeatures/MatrixImplicitTruncation.hlsl b/clang/test/CodeGenHLSL/BasicFeatures/MatrixImplicitTruncation.hlsl
new file mode 100644
index 0000000000000..e621f68623bd1
--- /dev/null
+++ b/clang/test/CodeGenHLSL/BasicFeatures/MatrixImplicitTruncation.hlsl
@@ -0,0 +1,138 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 6
+// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.7-library -disable-llvm-passes -emit-llvm -finclude-default-header -o - %s | FileCheck %s
+
+// CHECK-LABEL: define hidden noundef <12 x i32> @_Z10trunc_castu11matrix_typeILm4ELm4EiE(
+// CHECK-SAME: <16 x i32> noundef [[I44:%.*]]) #[[ATTR0:[0-9]+]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[I44_ADDR:%.*]] = alloca [16 x i32], align 4
+// CHECK-NEXT:    [[I34:%.*]] = alloca [12 x i32], align 4
+// CHECK-NEXT:    store <16 x i32> [[I44]], ptr [[I44_ADDR]], align 4
+// CHECK-NEXT:    [[TMP0:%.*]] = load <16 x i32>, ptr [[I44_ADDR]], align 4
+// CHECK-NEXT:    [[TRUNC:%.*]] = shufflevector <16 x i32> [[TMP0]], <16 x i32> poison, <12 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
+// CHECK-NEXT:    store <12 x i32> [[TRUNC]], ptr [[I34]], align 4
+// CHECK-NEXT:    [[TMP1:%.*]] = load <12 x i32>, ptr [[I34]], align 4
+// CHECK-NEXT:    ret <12 x i32> [[TMP1]]
+//
+ int3x4 trunc_cast(int4x4 i44) {
+    int3x4 i34 = i44;
+    return i34;
+}
+
+// CHECK-LABEL: define hidden noundef <12 x i32> @_Z11trunc_cast0u11matrix_typeILm4ELm4EiE(
+// CHECK-SAME: <16 x i32> noundef [[I44:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[I44_ADDR:%.*]] = alloca [16 x i32], align 4
+// CHECK-NEXT:    [[I43:%.*]] = alloca [12 x i32], align 4
+// CHECK-NEXT:    store <16 x i32> [[I44]], ptr [[I44_ADDR]], align 4
+// CHECK-NEXT:    [[TMP0:%.*]] = load <16 x i32>, ptr [[I44_ADDR]], align 4
+// CHECK-NEXT:    [[TRUNC:%.*]] = shufflevector <16 x i32> [[TMP0]], <16 x i32> poison, <12 x i32> <i32 0, i32 1, i32 2, i32 4, i32 5, i32 6, i32 8, i32 9, i32 10, i32 12, i32 13, i32 14>
+// CHECK-NEXT:    store <12 x i32> [[TRUNC]], ptr [[I43]], align 4
+// CHECK-NEXT:    [[TMP1:%.*]] = load <12 x i32>, ptr [[I43]], align 4
+// CHECK-NEXT:    ret <12 x i32> [[TMP1]]
+//
+ int4x3 trunc_cast0(int4x4 i44) {
+    int4x3 i43 = i44;
+    return i43;
+}
+
+// CHECK-LABEL: define hidden noundef <9 x i32> @_Z11trunc_cast1u11matrix_typeILm4ELm4EiE(
+// CHECK-SAME: <16 x i32> noundef [[I44:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[I44_ADDR:%.*]] = alloca [16 x i32], align 4
+// CHECK-NEXT:    [[I33:%.*]] = alloca [9 x i32], align 4
+// CHECK-NEXT:    store <16 x i32> [[I44]], ptr [[I44_ADDR]], align 4
+// CHECK-NEXT:    [[TMP0:%.*]] = load <16 x i32>, ptr [[I44_ADDR]], align 4
+// CHECK-NEXT:    [[TRUNC:%.*]] = shufflevector <16 x i32> [[TMP0]], <16 x i32> poison, <9 x i32> <i32 0, i32 1, i32 2, i32 4, i32 5, i32 6, i32 8, i32 9, i32 10>
+// CHECK-NEXT:    store <9 x i32> [[TRUNC]], ptr [[I33]], align 4
+// CHECK-NEXT:    [[TMP1:%.*]] = load <9 x i32>, ptr [[I33]], align 4
+// CHECK-NEXT:    ret <9 x i32> [[TMP1]]
+//
+ int3x3 trunc_cast1(int4x4 i44) {
+    int3x3 i33 = i44;
+    return i33;
+}
+
+// CHECK-LABEL: define hidden noundef <6 x i32> @_Z11trunc_cast2u11matrix_typeILm4ELm4EiE(
+// CHECK-SAME: <16 x i32> noundef [[I44:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[I44_ADDR:%.*]] = alloca [16 x i32], align 4
+// CHECK-NEXT:    [[I32:%.*]] = alloca [6 x i32], align 4
+// CHECK-NEXT:    store <16 x i32> [[I44]], ptr [[I44_ADDR]], align 4
+// CHECK-NEXT:    [[TMP0:%.*]] = load <16 x i32>, ptr [[I44_ADDR]], align 4
+// CHECK-NEXT:    [[TRUNC:%.*]] = shufflevector <16 x i32> [[TMP0]], <16 x i32> poison, <6 x i32> <i32 0, i32 1, i32 4, i32 5, i32 8, i32 9>
+// CHECK-NEXT:    store <6 x i32> [[TRUNC]], ptr [[I32]], align 4
+// CHECK-NEXT:    [[TMP1:%.*]] = load <6 x i32>, ptr [[I32]], align 4
+// CHECK-NEXT:    ret <6 x i32> [[TMP1]]
+//
+ int3x2 trunc_cast2(int4x4 i44) {
+    int3x2 i32 = i44;
+    return i32;
+}
+
+// CHECK-LABEL: define hidden noundef <6 x i32> @_Z11trunc_cast3u11matrix_typeILm4ELm4EiE(
+// CHECK-SAME: <16 x i32> noundef [[I44:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[I44_ADDR:%.*]] = alloca [16 x i32], align 4
+// CHECK-NEXT:    [[I23:%.*]] = alloca [6 x i32], align 4
+// CHECK-NEXT:    store <16 x i32> [[I44]], ptr [[I44_ADDR]], align 4
+// CHECK-NEXT:    [[TMP0:%.*]] = load <16 x i32>, ptr [[I44_ADDR]], align 4
+// CHECK-NEXT:    [[TRUNC:%.*]] = shufflevector <16 x i32> [[TMP0]], <16 x i32> poison, <6 x i32> <i32 0, i32 1, i32 2, i32 4, i32 5, i32 6>
+// CHECK-NEXT:    store <6 x i32> [[TRUNC]], ptr [[I23]], align 4
+// CHECK-NEXT:    [[TMP1:%.*]] = load <6 x i32>, ptr [[I23]], align 4
+// CHECK-NEXT:    ret <6 x i32> [[TMP1]]
+//
+ int2x3 trunc_cast3(int4x4 i44) {
+    int2x3 i23 = i44;
+    return i23;
+}
+
+// CHECK-LABEL: define hidden noundef <4 x i32> @_Z11trunc_cast4u11matrix_typeILm4ELm4EiE(
+// CHECK-SAME: <16 x i32> noundef [[I44:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[I44_ADDR:%.*]] = alloca [16 x i32], align 4
+// CHECK-NEXT:    [[I22:%.*]] = alloca [4 x i32], align 4
+// CHECK-NEXT:    store <16 x i32> [[I44]], ptr [[I44_ADDR]], align 4
+// CHECK-NEXT:    [[TMP0:%.*]] = load <16 x i32>, ptr [[I44_ADDR]], align 4
+// CHECK-NEXT:    [[TRUNC:%.*]] = shufflevector <16 x i32> [[TMP0]], <16 x i32> poison, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
+// CHECK-NEXT:    store <4 x i32> [[TRUNC]], ptr [[I22]], align 4
+// CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr [[I22]], align 4
+// CHECK-NEXT:    ret <4 x i32> [[TMP1]]
+//
+ int2x2 trunc_cast4(int4x4 i44) {
+    int2x2 i22 = i44;
+    return i22;
+}
+
+// CHECK-LABEL: define hidden noundef <2 x i32> @_Z11trunc_cast5u11matrix_typeILm4ELm4EiE(
+// CHECK-SAME: <16 x i32> noundef [[I44:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[I44_ADDR:%.*]] = alloca [16 x i32], align 4
+// CHECK-NEXT:    [[I21:%.*]] = alloca [2 x i32], align 4
+// CHECK-NEXT:    store <16 x i32> [[I44]], ptr [[I44_ADDR]], align 4
+// CHECK-NEXT:    [[TMP0:%.*]] = load <16 x i32>, ptr [[I44_ADDR]], align 4
+// CHECK-NEXT:    [[TRUNC:%.*]] = shufflevector <16 x i32> [[TMP0]], <16 x i32> poison, <2 x i32> <i32 0, i32 4>
+// CHECK-NEXT:    store <2 x i32> [[TRUNC]], ptr [[I21]], align 4
+// CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i32>, ptr [[I21]], align 4
+// CHECK-NEXT:    ret <2 x i32> [[TMP1]]
+//
+ int2x1 trunc_cast5(int4x4 i44) {
+    int2x1 i21 = i44;
+    return i21;
+}
+
+// CHECK-LABEL: define hidden noundef i32 @_Z11trunc_cast6u11matrix_typeILm4ELm4EiE(
+// CHECK-SAME: <16 x i32> noundef [[I44:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[I44_ADDR:%.*]] = alloca [16 x i32], align 4
+// CHECK-NEXT:    [[I1:%.*]] = alloca i32, align 4
+// CHECK-NEXT:    store <16 x i32> [[I44]], ptr [[I44_ADDR]], align 4
+// CHECK-NEXT:    [[TMP0:%.*]] = load <16 x i32>, ptr [[I44_ADDR]], align 4
+// CHECK-NEXT:    [[CAST_MTRUNC:%.*]] = extractelement <16 x i32> [[TMP0]], i32 0
+// CHECK-NEXT:    store i32 [[CAST_MTRUNC]], ptr [[I1]], align 4
+// CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr [[I1]], align 4
+// CHECK-NEXT:    ret i32 [[TMP1]]
+//
+ int trunc_cast6(int4x4 i44) {
+    int i1 = i44;
+    return i1;
+}
diff --git a/clang/test/CodeGenOpenCL/address-spaces.cl b/clang/test/CodeGenOpenCL/address-spaces.cl
index 5b2a95c6ac16a..b9f01069fa26c 100644
--- a/clang/test/CodeGenOpenCL/address-spaces.cl
+++ b/clang/test/CodeGenOpenCL/address-spaces.cl
@@ -2,9 +2,10 @@
 // RUN: %clang_cc1 %s -O0 -cl-std=CL3.0 -cl-ext=-all -ffake-address-space-map -emit-llvm -o - | FileCheck %s --check-prefixes=CHECK,SPIR
 // RUN: %clang_cc1 %s -O0 -cl-std=clc++2021 -cl-ext=-all -ffake-address-space-map -emit-llvm -o - | FileCheck %s --check-prefixes=CHECK,SPIR
 // RUN: %clang_cc1 %s -O0 -DCL20 -cl-std=CL2.0 -ffake-address-space-map -emit-llvm -o - | FileCheck %s --check-prefixes=CL20,CL20SPIR
-// RUN: %clang_cc1 %s -O0 -triple amdgcn-amd-amdhsa -emit-llvm -o - | FileCheck --check-prefixes=CHECK,AMDGCN %s
-// RUN: %clang_cc1 %s -O0 -triple amdgcn-amd-amdhsa -cl-std=CL3.0 -emit-llvm -o - | FileCheck --check-prefixes=CHECK,AMDGCN %s
+// RUN: %clang_cc1 %s -O0 -triple amdgcn-amd-mesa3d -emit-llvm -o - | FileCheck --check-prefixes=CHECK,AMDGCN %s
+// RUN: %clang_cc1 %s -O0 -triple amdgcn-amd-mesa3d -cl-std=CL3.0 -emit-llvm -o - | FileCheck --check-prefixes=CHECK,AMDGCN %s
 // RUN: %clang_cc1 %s -O0 -triple amdgcn-amd-amdhsa -DCL20 -cl-std=CL2.0 -emit-llvm -o - | FileCheck %s --check-prefixes=CL20,CL20AMDGCN
+// RUN: %clang_cc1 %s -O0 -triple amdgcn-amd-amdhsa -DCL20 -cl-std=CL3.0 -emit-llvm -o - | FileCheck %s --check-prefixes=CL20,CL20AMDGCN
 // RUN: %clang_cc1 %s -O0 -triple amdgcn-mesa-mesa3d -emit-llvm -o - | FileCheck --check-prefixes=CHECK,AMDGCN %s
 // RUN: %clang_cc1 %s -O0 -triple amdgcn-mesa-mesa3d -cl-std=CL3.0 -emit-llvm -o - | FileCheck --check-prefixes=CHECK,AMDGCN %s
 // RUN: %clang_cc1 %s -O0 -triple r600-- -emit-llvm -o - | FileCheck --check-prefixes=CHECK,AMDGCN %s
diff --git a/clang/test/CodeGenOpenCL/builtins-alloca.cl b/clang/test/CodeGenOpenCL/builtins-alloca.cl
index ce7da3aba9e45..51da8e3b3badb 100644
--- a/clang/test/CodeGenOpenCL/builtins-alloca.cl
+++ b/clang/test/CodeGenOpenCL/builtins-alloca.cl
@@ -3,9 +3,9 @@
 // RUN:     -emit-llvm -o - | FileCheck --check-prefixes=OPENCL12 %s
 // RUN: %clang_cc1 %s -O0 -triple amdgcn-amd-amdhsa -cl-std=CL2.0 \
 // RUN:     -emit-llvm -o - | FileCheck --check-prefixes=OPENCL20 %s
-// RUN: %clang_cc1 %s -O0 -triple amdgcn-amd-amdhsa -cl-std=CL3.0 \
+// RUN: %clang_cc1 %s -O0 -triple amdgcn-amd-mesa3d -target-cpu gfx600 -cl-std=CL3.0 \
 // RUN:     -emit-llvm -o - | FileCheck --check-prefixes=OPENCL30 %s
-// RUN: %clang_cc1 %s -O0 -triple amdgcn-amd-amdhsa -cl-std=CL3.0 -cl-ext=+__opencl_c_generic_address_space \
+// RUN: %clang_cc1 %s -O0 -triple amdgcn-amd-amdhsa -cl-std=CL3.0 \
 // RUN:     -emit-llvm -o - | FileCheck --check-prefixes=OPENCL30GAS %s
 
 // OPENCL-LABEL: define dso_local void @test1_builtin_alloca(
diff --git a/clang/test/Driver/autocomplete.c b/clang/test/Driver/autocomplete.c
index 4983b71496834..1fd60929751ee 100644
--- a/clang/test/Driver/autocomplete.c
+++ b/clang/test/Driver/autocomplete.c
@@ -117,6 +117,8 @@
 // WARNING-NEXT: -Wmany-braces-around-scalar-init
 // WARNING-NEXT: -Wmath-errno-enabled-with-veclib
 // WARNING-NEXT: -Wmathematical-notation-identifier-extension
+// WARNING-NEXT: -Wmatrix-conversion
+// WARNING-NEXT: -Wmatrix-conversions
 // WARNING-NEXT: -Wmax-tokens
 // WARNING-NEXT: -Wmax-unsigned-zero
 // RUN: %clang --autocomplete=-Wno-invalid-pp- | FileCheck %s -check-prefix=NOWARNING
diff --git a/clang/test/Misc/amdgcn.languageOptsOpenCL.cl b/clang/test/Misc/amdgcn.languageOptsOpenCL.cl
index 80c0825895c86..08715fc5a1f4a 100644
--- a/clang/test/Misc/amdgcn.languageOptsOpenCL.cl
+++ b/clang/test/Misc/amdgcn.languageOptsOpenCL.cl
@@ -11,6 +11,9 @@
 // RUN: %clang_cc1 -x cl -cl-std=CL3.0 %s -verify -triple amdgcn-unknown-unknown -Wpedantic-core-features -DTEST_CORE_FEATURES
 // RUN: %clang_cc1 -x cl -cl-std=CL3.0 %s -verify -triple amdgcn-unknown-unknown -target-cpu gfx700 -Wpedantic-core-features -DTEST_CORE_FEATURES -DFLAT_SUPPORT
 
+// Test none target with amdhsa triple, which implies >= gfx700
+// RUN: %clang_cc1 -x cl -cl-std=CL3.0 %s -verify -triple amdgcn-unknown-amdhsa -Wpedantic-core-features -DTEST_CORE_FEATURES -DFLAT_SUPPORT
+
 // Extensions in all versions
 #ifndef cl_clang_storage_class_specifiers
 #error "Missing cl_clang_storage_class_specifiers define"
@@ -162,6 +165,10 @@
   #ifndef __opencl_c_program_scope_global_variables
     #error "Missing __opencl_c_program_scope_global_variables define"
   #endif
+
+  #ifndef __opencl_c_read_write_images
+    #error "Missing __opencl_c_read_write_images define"
+  #endif
 #endif
 
 #if (__OPENCL_C_VERSION__ >= 300)
diff --git a/clang/test/OpenMP/amdgcn_weak_alias.c b/clang/test/OpenMP/amdgcn_weak_alias.c
index a9d5c1737b321..33c7dc0041810 100644
--- a/clang/test/OpenMP/amdgcn_weak_alias.c
+++ b/clang/test/OpenMP/amdgcn_weak_alias.c
@@ -94,10 +94,3 @@ int Three(void) __attribute__ ((weak, alias("__Three")));
 int Three_(void) __attribute__ ((alias("__Three")));
 extern int __attribute__((weak, alias("__Three_var"))) Three_var;
 extern int __attribute__((alias("__Three_var"))) Three_var_;
-//.
-// HOST: [[META0:![0-9]+]] = !{i32 1, !"__Two_var", i32 0, i32 0}
-// HOST: [[META1:![0-9]+]] = !{i32 1, !"__Three_var", i32 0, i32 1}
-//.
-// DEVICE: [[META0:![0-9]+]] = !{i32 1, !"__Two_var", i32 0, i32 0}
-// DEVICE: [[META1:![0-9]+]] = !{i32 1, !"__Three_var", i32 0, i32 1}
-//.
diff --git a/clang/test/SemaCXX/type-trait-common-reference.cpp b/clang/test/SemaCXX/type-trait-common-reference.cpp
deleted file mode 100644
index cc4fefc49886d..0000000000000
--- a/clang/test/SemaCXX/type-trait-common-reference.cpp
+++ /dev/null
@@ -1,136 +0,0 @@
-// RUN: %clang_cc1 -triple x86_64-apple-darwin10 -fsyntax-only -verify -std=c++17 -Wno-vla-cxx-extension %s
-// RUN: %clang_cc1 -triple x86_64-apple-darwin10 -fsyntax-only -verify -std=c++20 -Wno-vla-cxx-extension %s
-
-#if !__has_builtin(__builtin_common_reference)
-#  error
-#endif
-
-// expected-note@*:* {{template <template <class, class, template <class> class, template <class> class> class, template <class ...> class, template <class> class, class, class ...>}}
-
-void test() {
-  __builtin_common_reference<> a; // expected-error {{too few template arguments for template '__builtin_common_reference'}}
-  __builtin_common_reference<1> b; // expected-error {{template argument for template template parameter must be a class template or type alias template}}
-  __builtin_common_reference<int, 1> c; // expected-error {{template argument for template template parameter must be a class template or type alias template}}
-}
-
-struct empty_type {};
-
-template <class T>
-struct type_identity {
-  using type = T;
-};
-
-template <class...>
-struct common_type;
-
-template <class... Args>
-using common_type_t = typename common_type<Args...>::type;
-
-template <class, class, template <class> class, template <class> class>
-struct basic_common_reference {};
-
-template <class T, class U, template <class> class TX, template <class> class UX>
-using basic_common_reference_t = typename basic_common_reference<T, U, TX, UX>::type;
-
-void test_vla() {
-  int i = 4;
-  int VLA[i];
-  __builtin_common_reference<basic_common_reference_t, common_type_t, type_identity, empty_type, decltype(VLA)> d; // expected-error {{variably modified type 'decltype(VLA)' (aka 'int[i]') cannot be used as a template argument}}
-}
-
-template <class... Args>
-using common_reference_base = __builtin_common_reference<basic_common_reference_t, common_type_t, type_identity, empty_type, Args...>;
-
-template <class... Args>
-struct common_reference : common_reference_base<Args...> {};
-
-template <class... Args>
-using common_reference_t = typename __builtin_common_reference<basic_common_reference_t, common_type_t, type_identity, empty_type, Args...>::type;
-
-struct Incomplete;
-
-template<>
-struct common_type<Incomplete, Incomplete>;
-
-static_assert(__is_same(common_reference_base<>, empty_type));
-
-static_assert(__is_same(common_reference_base<Incomplete>, type_identity<Incomplete>));
-static_assert(__is_same(common_reference_base<char>, type_identity<char>));
-static_assert(__is_same(common_reference_base<int>, type_identity<int>));
-static_assert(__is_same(common_reference_base<const int>, type_identity<const int>));
-static_assert(__is_same(common_reference_base<volatile int>, type_identity<volatile int>));
-static_assert(__is_same(common_reference_base<const volatile int>, type_identity<const volatile int>));
-static_assert(__is_same(common_reference_base<int[]>, type_identity<int[]>));
-static_assert(__is_same(common_reference_base<const int[]>, type_identity<const int[]>));
-static_assert(__is_same(common_reference_base<void(&)()>, type_identity<void(&)()>));
-
-static_assert(__is_same(common_reference_base<int[], int[]>, type_identity<int*>));
-static_assert(__is_same(common_reference_base<int, int>, type_identity<int>));
-static_assert(__is_same(common_reference_base<int, long>, type_identity<long>));
-static_assert(__is_same(common_reference_base<long, int>, type_identity<long>));
-static_assert(__is_same(common_reference_base<long, long>, type_identity<long>));
-
-static_assert(__is_same(common_reference_base<const int, long>, type_identity<long>));
-static_assert(__is_same(common_reference_base<const volatile int, long>, type_identity<long>));
-static_assert(__is_same(common_reference_base<int, const long>, type_identity<long>));
-static_assert(__is_same(common_reference_base<int, const volatile long>, type_identity<long>));
-
-static_assert(__is_same(common_reference_base<int*, long*>, empty_type));
-static_assert(__is_same(common_reference_base<const unsigned int *const &, const unsigned int *const &>, type_identity<const unsigned int *const &>));
-
-static_assert(__is_same(common_reference_base<int, long, float>, type_identity<float>));
-static_assert(__is_same(common_reference_base<unsigned, char, long>, type_identity<long>));
-static_assert(__is_same(common_reference_base<long long, long long, long>, type_identity<long long>));
-
-static_assert(__is_same(common_reference_base<int [[clang::address_space(1)]]>, type_identity<int [[clang::address_space(1)]]>));
-static_assert(__is_same(common_reference_base<int [[clang::address_space(1)]], int>, type_identity<int>));
-static_assert(__is_same(common_reference_base<long [[clang::address_space(1)]], int>, type_identity<long>));
-static_assert(__is_same(common_reference_base<long [[clang::address_space(1)]], int [[clang::address_space(1)]]>, type_identity<long>));
-static_assert(__is_same(common_reference_base<long [[clang::address_space(1)]], long [[clang::address_space(1)]]>, type_identity<long>));
-static_assert(__is_same(common_reference_base<long [[clang::address_space(1)]], long [[clang::address_space(2)]]>, type_identity<long>));
-
-struct S {};
-struct T : S {};
-struct U {};
-
-static_assert(__is_same(common_reference_base<S&&, T&&>, type_identity<S&&>));
-
-static_assert(__is_same(common_reference_base<int S::*, int S::*>, type_identity<int S::*>));
-static_assert(__is_same(common_reference_base<int S::*, int T::*>, type_identity<int T::*>));
-static_assert(__is_same(common_reference_base<int S::*, long S::*>, empty_type));
-
-static_assert(__is_same(common_reference_base<int (S::*)(), int (S::*)()>, type_identity<int (S::*)()>));
-static_assert(__is_same(common_reference_base<int (S::*)(), int (T::*)()>, type_identity<int (T::*)()>));
-static_assert(__is_same(common_reference_base<int (S::*)(), long (S::*)()>, empty_type));
-
-static_assert(__is_same(common_reference_base<int&, int&>, type_identity<int&>));
-static_assert(__is_same(common_reference_base<int&, const int&>, type_identity<const int&>));
-static_assert(__is_same(common_reference_base<volatile int&, const int&>, type_identity<const volatile int&>));
-
-template <class T, class U>
-struct my_pair;
-
-template <class T1, class U1, class T2, class U2, template <class> class TX, template <class> class UX>
-struct basic_common_reference<my_pair<T1, U1>, my_pair<T2, U2>, TX, UX> {
-  using type = my_pair<common_reference_t<TX<T1>, UX<T2>>, common_reference_t<TX<U1>, UX<U2>>>;
-};
-
-static_assert(__is_same(common_reference_base<my_pair<const int&, int&>, my_pair<int&, volatile int&>>, type_identity<my_pair<const int&, volatile int&>>));
-static_assert(__is_same(common_reference_base<const my_pair<int, int>&, my_pair<int&, volatile int&>>, type_identity<my_pair<const int&, const volatile int&>>));
-static_assert(__is_same(common_reference_base<const int&, const volatile int&>, type_identity<const volatile int&>));
-static_assert(__is_same(common_reference_base<int&&, const volatile int&>, type_identity<int>));
-static_assert(__is_same(common_reference_base<my_pair<int, int>&&, my_pair<int&, volatile int&>>, type_identity<my_pair<const int&, int>>));
-static_assert(__is_same(common_reference_base<my_pair<int, int>&&, my_pair<int&, int>&&>, type_identity<my_pair<const int&, int&&>>));
-
-struct conversion_operator {
-  operator volatile int&&() volatile;
-};
-
-static_assert(__is_same(common_reference_base<volatile conversion_operator&&, volatile int&&>, type_identity<volatile int&&>));
-
-struct reference_wrapper {
-  reference_wrapper(int&);
-  operator int&() const;
-};
-
-static_assert(__is_same(common_reference_base<const reference_wrapper&, int&>, empty_type));
diff --git a/clang/test/SemaHLSL/MatrixElementOverloadResolution.hlsl b/clang/test/SemaHLSL/MatrixElementOverloadResolution.hlsl
new file mode 100644
index 0000000000000..04149e176edbd
--- /dev/null
+++ b/clang/test/SemaHLSL/MatrixElementOverloadResolution.hlsl
@@ -0,0 +1,293 @@
+// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.3-library -fnative-half-type -finclude-default-header -verify -o - %s -DERROR=1
+// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.3-library -fnative-half-type -finclude-default-header -ast-dump %s | FileCheck %s
+
+// This test verifies floating point type implicit conversion ranks for overload
+// resolution. In HLSL the built-in type ranks are half < float < double. This
+// applies to both scalar and matrix types.
+
+// HLSL allows implicit truncation fo types, so it differentiates between
+// promotions (converting to larger types) and conversions (converting to
+// smaller types). Promotions are preferred over conversions. Promotions prefer
+// promoting to the next lowest type in the ranking order. Conversions prefer
+// converting to the next highest type in the ranking order.
+
+void HalfFloatDouble(double2x2 D);
+void HalfFloatDouble(float2x2 F);
+void HalfFloatDouble(half2x2 H);
+
+// CHECK: FunctionDecl {{.*}} used HalfFloatDouble 'void (double2x2)'
+// CHECK: FunctionDecl {{.*}} used HalfFloatDouble 'void (float2x2)'
+// CHECK: FunctionDecl {{.*}} used HalfFloatDouble 'void (half2x2)'
+
+void FloatDouble(double2x2 D); // expected-note {{candidate function}}
+void FloatDouble(float2x2 F); // expected-note {{candidate function}}
+
+// CHECK: FunctionDecl {{.*}} used FloatDouble 'void (double2x2)'
+// CHECK: FunctionDecl {{.*}} used FloatDouble 'void (float2x2)'
+
+void HalfDouble(double2x2 D);
+void HalfDouble(half2x2 H);
+
+// CHECK: FunctionDecl {{.*}} used HalfDouble 'void (double2x2)'
+// CHECK: FunctionDecl {{.*}} used HalfDouble 'void (half2x2)'
+
+void HalfFloat(float2x2 F); // expected-note {{candidate function}}
+void HalfFloat(half2x2 H); // expected-note {{candidate function}}
+
+// CHECK: FunctionDecl {{.*}} used HalfFloat 'void (float2x2)'
+// CHECK: FunctionDecl {{.*}} used HalfFloat 'void (half2x2)'
+
+void Double(double2x2 D);
+void Float(float2x2 F);
+void Half(half2x2 H);
+
+// CHECK: FunctionDecl {{.*}} used Double 'void (double2x2)'
+// CHECK: FunctionDecl {{.*}} used Float 'void (float2x2)'
+// CHECK: FunctionDecl {{.*}} used Half 'void (half2x2)'
+
+// Case 1: A function declared with overloads for half float and double types.
+//   (a) When called with half, it will resolve to half because half is an exact
+//   match.
+//   (b) When called with float it will resolve to float because float is an
+//   exact match.
+//   (c) When called with double it will resolve to double because it is an
+//   exact match.
+
+// CHECK-LABEL: FunctionDecl {{.*}} Case1 'void (half2x2, float2x2, double2x2)'
+void Case1(half2x2 H, float2x2 F, double2x2 D) {
+  // CHECK: CallExpr {{.*}} 'void'
+  // CHECK-NEXT: ImplicitCastExpr {{.*}} 'void (*)(half2x2)' <FunctionToPointerDecay>
+  // CHECK-NEXT: DeclRefExpr {{.*}} 'void (half2x2)' lvalue Function {{.*}} 'HalfFloatDouble' 'void (half2x2)'
+  HalfFloatDouble(H);
+
+  // CHECK: CallExpr {{.*}} 'void'
+  // CHECK-NEXT: ImplicitCastExpr {{.*}} 'void (*)(float2x2)' <FunctionToPointerDecay>
+  // CHECK-NEXT: DeclRefExpr {{.*}} 'void (float2x2)' lvalue Function {{.*}} 'HalfFloatDouble' 'void (float2x2)'
+  HalfFloatDouble(F);
+
+  // CHECK: CallExpr {{.*}} 'void'
+  // CHECK-NEXT: ImplicitCastExpr {{.*}} 'void (*)(double2x2)' <FunctionToPointerDecay>
+  // CHECK-NEXT: DeclRefExpr {{.*}} 'void (double2x2)' lvalue Function {{.*}} 'HalfFloatDouble' 'void (double2x2)'
+  HalfFloatDouble(D);
+}
+
+// Case 2: A function declared with double and float overlaods.
+//   (a) When called with half, it fails to resulve the ambiguous promotion.
+//   (b) When called with float it will resolve to float because float is an
+//   exact match.
+//   (c) When called with double it will resolve to double because it is an
+//   exact match.
+
+// CHECK-LABEL: FunctionDecl {{.*}} Case2 'void (half2x2, float2x2, double2x2)'
+void Case2(half2x2 H, float2x2 F, double2x2 D) {
+#if ERROR
+  FloatDouble(H); // expected-error {{call to 'FloatDouble' is ambiguous}}
+#endif
+
+  // CHECK: CallExpr {{.*}} 'void'
+  // CHECK-NEXT: ImplicitCastExpr {{.*}} 'void (*)(float2x2)' <FunctionToPointerDecay>
+  // CHECK-NEXT: DeclRefExpr {{.*}} 'void (float2x2)' lvalue Function {{.*}} 'FloatDouble' 'void (float2x2)'
+  FloatDouble(F);
+
+  // CHECK: CallExpr {{.*}} 'void'
+  // CHECK-NEXT: ImplicitCastExpr {{.*}} 'void (*)(double2x2)' <FunctionToPointerDecay>
+  // CHECK-NEXT: DeclRefExpr {{.*}} 'void (double2x2)' lvalue Function {{.*}} 'FloatDouble' 'void (double2x2)'
+  FloatDouble(D);
+}
+
+// Case 3: A function declared with half and double overloads
+//   (a) When called with half, it will resolve to half because it is an exact
+//   match.
+//   (b) When called with flaot, it will resolve to double because double is a
+//   valid promotion.
+//   (c) When called with double, it will resolve to double because it is an
+//   exact match.
+
+// CHECK-LABEL: FunctionDecl {{.*}} Case3 'void (half2x2, float2x2, double2x2)'
+void Case3(half2x2 H, float2x2 F, double2x2 D) {
+  // CHECK: CallExpr {{.*}} 'void'
+  // CHECK-NEXT: ImplicitCastExpr {{.*}} 'void (*)(half2x2)' <FunctionToPointerDecay>
+  // CHECK-NEXT: DeclRefExpr {{.*}} 'void (half2x2)' lvalue Function {{.*}} 'HalfDouble' 'void (half2x2)'
+  HalfDouble(H);
+
+  // CHECK: CallExpr {{.*}} 'void'
+  // CHECK-NEXT: ImplicitCastExpr {{.*}} 'void (*)(double2x2)' <FunctionToPointerDecay>
+  // CHECK-NEXT: DeclRefExpr {{.*}} 'void (double2x2)' lvalue Function {{.*}} 'HalfDouble' 'void (double2x2)'
+  HalfDouble(F);
+
+  // CHECK: CallExpr {{.*}} 'void'
+  // CHECK-NEXT: ImplicitCastExpr {{.*}} 'void (*)(double2x2)' <FunctionToPointerDecay>
+  // CHECK-NEXT: DeclRefExpr {{.*}} 'void (double2x2)' lvalue Function {{.*}} 'HalfDouble' 'void (double2x2)'
+  HalfDouble(D);
+}
+
+// Case 4: A function declared with half and float overloads.
+//   (a) When called with half, it will resolve to half because half is an exact
+//   match.
+//   (b) When called with float it will resolve to float because float is an
+//   exact match.
+//   (c) When called with double it fails to resolve the ambigjuous conversion.
+
+// CHECK-LABEL: FunctionDecl {{.*}} Case4 'void (half2x2, float2x2, double2x2)'
+void Case4(half2x2 H, float2x2 F, double2x2 D) {
+  // CHECK: CallExpr {{.*}} 'void'
+  // CHECK-NEXT: ImplicitCastExpr {{.*}} 'void (*)(half2x2)' <FunctionToPointerDecay>
+  // CHECK-NEXT: DeclRefExpr {{.*}} 'void (half2x2)' lvalue Function {{.*}} 'HalfFloat' 'void (half2x2)'
+  HalfFloat(H);
+
+  // CHECK: CallExpr {{.*}} 'void'
+  // CHECK-NEXT: ImplicitCastExpr {{.*}} 'void (*)(float2x2)' <FunctionToPointerDecay>
+  // CHECK-NEXT: DeclRefExpr {{.*}} 'void (float2x2)' lvalue Function {{.*}} 'HalfFloat' 'void (float2x2)'
+  HalfFloat(F);
+
+#if ERROR
+  HalfFloat(D); // expected-error{{call to 'HalfFloat' is ambiguous}}
+#endif
+}
+
+// Case 5: A function declared with only a double overload.
+//   (a) When called with half, it will resolve to double because double is a
+//   valid promotion.
+//   (b) When called with float it will resolve to double because double is a
+//   valid promotion.
+//   (c) When called with double it will resolve to double because it is an
+//   exact match.
+
+// CHECK-LABEL: FunctionDecl {{.*}} Case5 'void (half2x2, float2x2, double2x2)'
+void Case5(half2x2 H, float2x2 F, double2x2 D) {
+  // CHECK: CallExpr {{.*}} 'void'
+  // CHECK-NEXT: ImplicitCastExpr {{.*}} 'void (*)(double2x2)' <FunctionToPointerDecay>
+  // CHECK-NEXT: DeclRefExpr {{.*}} 'void (double2x2)' lvalue Function {{.*}} 'Double' 'void (double2x2)'
+  Double(H);
+
+  // CHECK: CallExpr {{.*}} 'void'
+  // CHECK-NEXT: ImplicitCastExpr {{.*}} 'void (*)(double2x2)' <FunctionToPointerDecay>
+  // CHECK-NEXT: DeclRefExpr {{.*}} 'void (double2x2)' lvalue Function {{.*}} 'Double' 'void (double2x2)'
+  Double(F);
+
+  // CHECK: CallExpr {{.*}} 'void'
+  // CHECK-NEXT: ImplicitCastExpr {{.*}} 'void (*)(double2x2)' <FunctionToPointerDecay>
+  // CHECK-NEXT: DeclRefExpr {{.*}} 'void (double2x2)' lvalue Function {{.*}} 'Double' 'void (double2x2)'
+  Double(D);
+}
+
+// Case 6: A function declared with only a float overload.
+//   (a) When called with half, it will resolve to float because float is a
+//   valid promotion.
+//   (b) When called with float it will resolve to float because float is an
+//   exact match.
+//   (c) When called with double it will resolve to float because it is a
+//   valid conversion.
+
+// CHECK-LABEL: FunctionDecl {{.*}} Case6 'void (half2x2, float2x2, double2x2)'
+void Case6(half2x2 H, float2x2 F, double2x2 D) {
+  // CHECK: CallExpr {{.*}} 'void'
+  // CHECK-NEXT: ImplicitCastExpr {{.*}} 'void (*)(float2x2)' <FunctionToPointerDecay>
+  // CHECK-NEXT: DeclRefExpr {{.*}} 'void (float2x2)' lvalue Function {{.*}} 'Float' 'void (float2x2)'
+  Float(H);
+
+  // CHECK: CallExpr {{.*}} 'void'
+  // CHECK-NEXT: ImplicitCastExpr {{.*}} 'void (*)(float2x2)' <FunctionToPointerDecay>
+  // CHECK-NEXT: DeclRefExpr {{.*}} 'void (float2x2)' lvalue Function {{.*}} 'Float' 'void (float2x2)'
+  Float(F);
+
+  // CHECK: CallExpr {{.*}} 'void'
+  // CHECK-NEXT: ImplicitCastExpr {{.*}} 'void (*)(float2x2)' <FunctionToPointerDecay>
+  // CHECK-NEXT: DeclRefExpr {{.*}} 'void (float2x2)' lvalue Function {{.*}} 'Float' 'void (float2x2)'
+  Float(D); // TODO: See #168944. Make this an expect warning. {{implicit conversion loses floating-point precision: 'double2x2' (aka 'matrix<double, 2, 2>') to 'matrix<float, 2, 2>' (matrix of 2 'float' values)}}
+}
+
+// Case 7: A function declared with only a half overload.
+//   (a) When called with half, it will resolve to half because half is an
+//   exact match
+//   (b) When called with float it will resolve to half because half is a
+//   valid conversion.
+//   (c) When called with double it will resolve to float because it is a
+//   valid conversion.
+
+// CHECK-LABEL: FunctionDecl {{.*}} Case7 'void (half2x2, float2x2, double2x2)'
+void Case7(half2x2 H, float2x2 F, double2x2 D) {
+  // CHECK: CallExpr {{.*}} 'void'
+  // CHECK-NEXT: ImplicitCastExpr {{.*}} 'void (*)(half2x2)' <FunctionToPointerDecay>
+  // CHECK-NEXT: DeclRefExpr {{.*}} 'void (half2x2)' lvalue Function {{.*}} 'Half' 'void (half2x2)'
+  Half(H);
+
+  // CHECK: CallExpr {{.*}} 'void'
+  // CHECK-NEXT: ImplicitCastExpr {{.*}} 'void (*)(half2x2)' <FunctionToPointerDecay>
+  // CHECK-NEXT: DeclRefExpr {{.*}} 'void (half2x2)' lvalue Function {{.*}} 'Half' 'void (half2x2)'
+  Half(F); // TODO: See #168944. Make this an expect warning. {{implicit conversion loses floating-point precision: 'float2x2' (aka 'matrix<float, 2, 2>') to 'matrix<half, 2, 2>' (matrix of 4 'half' values)}}
+
+  // CHECK: CallExpr {{.*}} 'void'
+  // CHECK-NEXT: ImplicitCastExpr {{.*}} 'void (*)(half2x2)' <FunctionToPointerDecay>
+  // CHECK-NEXT: DeclRefExpr {{.*}} 'void (half2x2)' lvalue Function {{.*}} 'Half' 'void (half2x2)'
+  Half(D); // TODO: See #168944. Make this an expect warning.  {{implicit conversion loses floating-point precision: 'double2x2' (aka 'matrix<double, 2, 2>') to 'matrix<half, 2, 2>' (matrix of 4 'half' values)}}
+}
+
+void fn3x2(float3x2) {} // expected-note{{candidate function}}
+void fn2x2(float2x2) {}
+void fn2x2IO(inout float2x2) {}
+void fnI2x2IO(inout int2x2) {}
+
+void matOrVec(float4 F) {}
+void matOrVec(float2x2 F) {}
+
+void matOrVec2(float3 F) {} // expected-note{{candidate function}}
+void matOrVec2(float2x3 F) {} // expected-note{{candidate function}}
+
+export void Case8(float2x3 f23, float4x4 f44, float3x3 f33, float3x2 f32) {
+  int2x2 i22 = f23;
+  // expected-warning@-1{{implicit conversion truncates matrix: 'float2x3' (aka 'matrix<float, 2, 3>') to 'int2x2' (aka 'matrix<int, 2, 2>')}}
+  //CHECK: VarDecl {{.*}} i22 'int2x2':'matrix<int, 2, 2>' cinit
+  //CHECK-NEXT: ImplicitCastExpr {{.*}} 'int2x2':'matrix<int, 2, 2>' <FloatingToIntegral>
+  //CHECK-NEXT: ImplicitCastExpr {{.*}} 'float2x3':'matrix<float, 2, 3>' <LValueToRValue>
+#ifdef ERROR
+  int3x2 i32 = f23; // expected-error{{cannot initialize a variable of type 'matrix<int, 3, 2>' with an lvalue of type 'matrix<float, 2, 3>'}}
+  fn3x2(f23); // expected-error{{no matching function for call to 'fn3x2'}}
+#endif
+  
+  fn2x2(f23);
+  // expected-warning@-1{{implicit conversion truncates matrix: 'float2x3' (aka 'matrix<float, 2, 3>') to 'matrix<float, 2, 2>'}}
+  //CHECK: DeclRefExpr {{.*}} 'void (float2x2)' lvalue Function {{.*}} 'fn2x2' 'void (float2x2)'
+  //CHECK-NEXT: ImplicitCastExpr {{.*}} 'matrix<float, 2, 2>' <HLSLMatrixTruncation>
+  //CHECK-NEXT: ImplicitCastExpr {{.*}} 'float2x3':'matrix<float, 2, 3>' <LValueToRValue>
+
+#ifdef ERROR
+  fn2x2IO(f23); // expected-error{{assigning to 'matrix<[2 * ...], 3>' from incompatible type 'matrix<[2 * ...], 2>'}}
+  fnI2x2IO(f23); // expected-error{{assigning to 'matrix<float, [...], 3>' from incompatible type 'matrix<int, [...], 2>'}}
+#endif
+
+  matOrVec(f23);
+  // expected-warning@-1{{implicit conversion truncates matrix: 'float2x3' (aka 'matrix<float, 2, 3>') to 'matrix<float, 2, 2>'}}
+  //CHECK: DeclRefExpr {{.*}} 'void (float2x2)' lvalue Function {{.*}} 'matOrVec' 'void (float2x2)'
+  //CHECK-NEXT: ImplicitCastExpr {{.*}} 'matrix<float, 2, 2>' <HLSLMatrixTruncation>
+  //CHECK-NEXT: ImplicitCastExpr {{.*}} 'float2x3':'matrix<float, 2, 3>' <LValueToRValue>
+
+  matOrVec(f44);
+  // expected-warning@-1{{implicit conversion truncates matrix: 'float4x4' (aka 'matrix<float, 4, 4>') to 'matrix<float, 2, 2>'}}
+  //CHECK: DeclRefExpr {{.*}} 'void (float2x2)' lvalue Function {{.*}} 'matOrVec' 'void (float2x2)'
+  //CHECK-NEXT: ImplicitCastExpr {{.*}} 'matrix<float, 2, 2>' <HLSLMatrixTruncation>
+  //CHECK-NEXT: ImplicitCastExpr {{.*}} 'float4x4':'matrix<float, 4, 4>' <LValueToRValue>
+
+#ifdef ERROR
+  matOrVec(2.0); // TODO: See #168960 this should be ambiguous once we implement ICK_HLSL_Matrix_Splat.
+#endif
+  matOrVec2(f23);
+  //CHECK: DeclRefExpr {{.*}} 'void (float2x3)' lvalue Function {{.*}} 'matOrVec2' 'void (float2x3)'
+  //CHECK-NEXT: ImplicitCastExpr {{.*}} 'float2x3':'matrix<float, 2, 3>' <LValueToRValue>
+
+  matOrVec2(f44);
+  // expected-warning@-1{{implicit conversion truncates matrix: 'float4x4' (aka 'matrix<float, 4, 4>') to 'matrix<float, 2, 3>'}}
+  //CHECK: DeclRefExpr {{.*}} 'void (float2x3)' lvalue Function {{.*}} 'matOrVec2' 'void (float2x3)'
+  //CHECK-NEXT: ImplicitCastExpr {{.*}} 'matrix<float, 2, 3>' <HLSLMatrixTruncation>
+  //CHECK-NEXT: ImplicitCastExpr {{.*}} 'float4x4':'matrix<float, 4, 4>' <LValueToRValue>
+
+  matOrVec2(f33);
+  // expected-warning@-1{{implicit conversion truncates matrix: 'float3x3' (aka 'matrix<float, 3, 3>') to 'matrix<float, 2, 3>'}}
+  //CHECK: DeclRefExpr {{.*}} 'void (float2x3)' lvalue Function {{.*}} 'matOrVec2' 'void (float2x3)'
+  //CHECK-NEXT: ImplicitCastExpr {{.*}} 'matrix<float, 2, 3>' <HLSLMatrixTruncation>
+  //CHECK-NEXT: ImplicitCastExpr {{.*}} 'float3x3':'matrix<float, 3, 3>' <LValueToRValue>
+  
+#ifdef ERROR
+  matOrVec2(f32); // expected-error{{no matching function for call to 'matOrVec2'}}
+#endif
+}
diff --git a/clang/test/SemaHLSL/Types/BuiltinMatrix/MatrixCastErrors.hlsl b/clang/test/SemaHLSL/Types/BuiltinMatrix/MatrixCastErrors.hlsl
new file mode 100644
index 0000000000000..59d432cd3eb00
--- /dev/null
+++ b/clang/test/SemaHLSL/Types/BuiltinMatrix/MatrixCastErrors.hlsl
@@ -0,0 +1,21 @@
+// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.6-library -finclude-default-header -std=hlsl202x -verify %s
+
+// Note column is too large
+export int3x2 shape_cast_error(float2x3 f23) {
+    int3x2 i32 = (int3x2)f23;
+    // expected-error@-1 {{conversion between matrix types 'int3x2' (aka 'matrix<int, 3, 2>') and 'matrix<float, 2, 3>' of different size is not allowed}}
+    return i32;
+}
+// Note row is too large
+export int2x3 shape_cast_error2(float3x2 f32) {
+    int2x3 i23 = (int2x3)f32;
+    // expected-error@-1 {{conversion between matrix types 'int2x3' (aka 'matrix<int, 2, 3>') and 'matrix<float, 3, 2>' of different size is not allowed}}
+    return i23;
+}
+
+// Note do the type change independent of the shape should still error
+export int2x3 shape_cast_error3(float3x2 f32) {
+    int2x3 i23 = (int3x2)f32;
+    // expected-error@-1 {{cannot initialize a variable of type 'matrix<[...], 2, 3>' with an rvalue of type 'matrix<[...], 3, 2>}}
+    return i23;
+}
diff --git a/clang/test/SemaHLSL/Types/BuiltinMatrix/MatrixImplicitTruncCastWarnings.hlsl b/clang/test/SemaHLSL/Types/BuiltinMatrix/MatrixImplicitTruncCastWarnings.hlsl
new file mode 100644
index 0000000000000..2c50b957578ec
--- /dev/null
+++ b/clang/test/SemaHLSL/Types/BuiltinMatrix/MatrixImplicitTruncCastWarnings.hlsl
@@ -0,0 +1,50 @@
+// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.6-library -finclude-default-header -verify %s
+
+export int3x4 trunc_cast(int4x4 i44) {
+    int3x4 i34 = i44;
+    // expected-warning@-1{{implicit conversion truncates matrix: 'int4x4' (aka 'matrix<int, 4, 4>') to 'matrix<int, 3, 4>'}}
+    return i34;
+}
+
+export int4x3 trunc_cast0(int4x4 i44) {
+    int4x3 i43 = i44;
+    // expected-warning@-1{{implicit conversion truncates matrix: 'int4x4' (aka 'matrix<int, 4, 4>') to 'matrix<int, 4, 3>'}}
+    return i43;
+}
+
+export int3x3 trunc_cast1(int4x4 i44) {
+    int3x3 i33 = i44;
+    // expected-warning@-1{{implicit conversion truncates matrix: 'int4x4' (aka 'matrix<int, 4, 4>') to 'matrix<int, 3, 3>'}}
+    return i33;
+}
+
+export int3x2 trunc_cast2(int4x4 i44) {
+    int3x2 i32 = i44;
+    // expected-warning@-1{{implicit conversion truncates matrix: 'int4x4' (aka 'matrix<int, 4, 4>') to 'matrix<int, 3, 2>'}}
+    return i32;
+}
+
+export int2x3 trunc_cast3(int4x4 i44) {
+    int2x3 i23 = i44;
+    // expected-warning@-1{{implicit conversion truncates matrix: 'int4x4' (aka 'matrix<int, 4, 4>') to 'matrix<int, 2, 3>'}}
+    return i23;
+}
+
+export int2x2 trunc_cast4(int4x4 i44) {
+    int2x2 i22 = i44;
+    // expected-warning@-1{{implicit conversion truncates matrix: 'int4x4' (aka 'matrix<int, 4, 4>') to 'matrix<int, 2, 2>'}}
+    return i22;
+}
+
+export int2x1 trunc_cast5(int4x4 i44) {
+    int2x1 i21 = i44;
+    // expected-warning@-1{{implicit conversion truncates matrix: 'int4x4' (aka 'matrix<int, 4, 4>') to 'matrix<int, 2, 1>'}}
+    return i21;
+}
+
+export int trunc_scalar_cast6(int4x4 i44) {
+    int i1 = i44;
+    // expected-warning@-1{{implicit conversion turns matrix to scalar: 'int4x4' (aka 'matrix<int, 4, 4>') to 'int'}}
+    return i1;
+}
+
diff --git a/clang/test/SemaHLSL/static_resources.hlsl b/clang/test/SemaHLSL/static_resources.hlsl
new file mode 100644
index 0000000000000..f71e9ea98e0d9
--- /dev/null
+++ b/clang/test/SemaHLSL/static_resources.hlsl
@@ -0,0 +1,138 @@
+// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.6-compute -emit-llvm -disable-llvm-passes -o - %s | llvm-cxxfilt | FileCheck %s
+
+// CHECK-DAG: [[ONE_STR:@.*]] = private unnamed_addr constant [4 x i8] c"One\00"
+// CHECK-DAG: [[ARRAY_STR:@.*]] = private unnamed_addr constant [6 x i8] c"Array\00"
+// CHECK-DAG: [[ONEWITHCOUNTER_STR:@.*]] = private unnamed_addr constant [15 x i8] c"OneWithCounter\00"
+// CHECK-DAG: [[ARRAYWITHCOUNTER_STR:@.*]] = private unnamed_addr constant [17 x i8] c"ArrayWithCounter\00"
+// CHECK-NOT: private unnamed_addr constant [{{[0-9]+}} x i8] c"Static
+
+RWBuffer<float> One : register(u1, space5);
+RWBuffer<float> Array[2] : register(u10, space6);
+RWStructuredBuffer<int> OneWithCounter : register(u2, space4);
+RWStructuredBuffer<int> ArrayWithCounter[2] : register(u7, space4);
+
+// Check that the non-static resource One is initialized from binding on
+// startup (register 1, space 5).
+// CHECK: define internal void @__cxx_global_var_init{{.*}}
+// CHECK-NEXT: entry:
+// CHECK-NEXT: call void @hlsl::RWBuffer<float>::__createFromBinding(unsigned int, unsigned int, int, unsigned int, char const*)
+// CHECK-SAME: (ptr {{.*}} @One, i32 noundef 1, i32 noundef 5, i32 noundef 1, i32 noundef 0, ptr noundef [[ONE_STR]])
+
+// Check that the non-static resource OneWithCounter is initialized from binding on
+// startup (register 2, space 4).
+// CHECK: define internal void @__cxx_global_var_init{{.*}}
+// CHECK-NEXT: entry:
+// CHECK-NEXT: call void @hlsl::RWStructuredBuffer<int>::__createFromBindingWithImplicitCounter(unsigned int, unsigned int, int, unsigned int, char const*, unsigned int)
+// CHECK-SAME: (ptr {{.*}} @OneWithCounter, i32 noundef 2, i32 noundef 4, i32 noundef 1, i32 noundef 0, ptr noundef [[ONEWITHCOUNTER_STR]], i32 noundef 0)
+
+// Note that non-static resource arrays are not initialized on startup.
+// The individual resources from the array are initialized on access.
+
+static RWBuffer<float> StaticOne;
+static RWBuffer<float> StaticArray[2];
+
+// Check that StaticOne resource is initialized on startup with the default
+// constructor and not from binding. It will initalize the handle to poison.
+// CHECK: define internal void @__cxx_global_var_init{{.*}}
+// CHECK-NEXT: entry:
+// CHECK-NEXT: call void @hlsl::RWBuffer<float>::RWBuffer()(ptr {{.*}} @StaticOne)
+
+// Check that StaticArray elements are initialized on startup with the default
+// constructor and not from binding. The initializer will loop over the array
+// elements and call the default constructor for each one, setting the handle to poison.
+// CHECK: define internal void @__cxx_global_var_init{{.*}}
+// CHECK-NEXT: entry:
+// CHECK-NEXT: br label %arrayctor.loop
+// CHECK: arrayctor.loop:                                   ; preds = %arrayctor.loop, %entry
+// CHECK-NEXT:   %arrayctor.cur = phi ptr [ @StaticArray, %entry ], [ %arrayctor.next, %arrayctor.loop ]
+// CHECK-NEXT: call void @hlsl::RWBuffer<float>::RWBuffer()(ptr {{.*}} %arrayctor.cur)
+// CHECK-NEXT: %arrayctor.next = getelementptr inbounds %"class.hlsl::RWBuffer", ptr %arrayctor.cur, i32 1
+// CHECK-NEXT: %arrayctor.done = icmp eq ptr %arrayctor.next, getelementptr inbounds (%"class.hlsl::RWBuffer", ptr @StaticArray, i32 2)
+// CHECK-NEXT: br i1 %arrayctor.done, label %arrayctor.cont, label %arrayctor.loop
+// CHECK: arrayctor.cont:                                   ; preds = %arrayctor.loop
+// CHECK-NEXT: ret void
+
+static RWStructuredBuffer<int> StaticOneWithCounter;
+
+// Check that StaticOneWithCounter resource is initialized on startup with the default
+// constructor and not from binding. It will initalize the handle to poison.
+// CHECK: define internal void @__cxx_global_var_init{{.*}}
+// CHECK-NEXT: entry:
+// CHECK-NEXT: call void @hlsl::RWStructuredBuffer<int>::RWStructuredBuffer()(ptr {{.*}} @StaticOneWithCounter)
+
+// No other global initialization routines should be present.
+// CHECK-NOT: define internal void @__cxx_global_var_init{{.*}}
+
+[numthreads(4,1,1)]
+void main() {
+// CHECK: define internal void @main()()
+// CHECK-NEXT: entry:
+// CHECK-NEXT: %[[TMP0:.*]] = alloca %"class.hlsl::RWBuffer"
+
+  static RWBuffer<float> StaticLocal;
+// Check that StaticLocal is initialized by default constructor (handle set to poison)
+// and not from binding.
+// call void @hlsl::RWBuffer<float>::RWBuffer()(ptr {{.*}} @main()::StaticLocal)
+
+  StaticLocal = Array[1];
+// A[2][0] is accessed here, so it should be initialized from binding (register 10, space 6, index 1),
+// and then assigned to StaticLocal using = operator.
+// CHECK: call void @hlsl::RWBuffer<float>::__createFromBinding(unsigned int, unsigned int, int, unsigned int, char const*)
+// CHECK-SAME: (ptr {{.*}} %[[TMP0]], i32 noundef 10, i32 noundef 6, i32 noundef 2, i32 noundef 1, ptr noundef [[ARRAY_STR]])
+// CHECK-NEXT: call {{.*}} ptr @hlsl::RWBuffer<float>::operator=({{.*}})(ptr {{.*}} @main()::StaticLocal, ptr {{.*}} %[[TMP0]])
+
+  StaticOne = One;
+// Operator = call to assign non-static One handle to static StaticOne.
+// CHECK-NEXT: call {{.*}} ptr @hlsl::RWBuffer<float>::operator=({{.*}})(ptr {{.*}} @StaticOne, ptr {{.*}} @One)
+
+  StaticArray = Array;
+// Check that each elements of StaticArray is initialized from binding (register 10, space 6, indices 0 and 1).
+// CHECK: call void @hlsl::RWBuffer<float>::__createFromBinding(unsigned int, unsigned int, int, unsigned int, char const*)
+// CHECK-SAME: (ptr {{.*}} sret(%"class.hlsl::RWBuffer") align 4 @StaticArray, i32 noundef 10, i32 noundef 6, i32 noundef 2, i32 noundef 0, ptr noundef [[ARRAY_STR]])
+// CHECK-NEXT: call void @hlsl::RWBuffer<float>::__createFromBinding(unsigned int, unsigned int, int, unsigned int, char const*)
+// CHECK-SAME: (ptr {{.*}} sret(%"class.hlsl::RWBuffer") align 4 getelementptr ([2 x %"class.hlsl::RWBuffer"], ptr @StaticArray, i32 0, i32 1),
+// CHECK-SAME: i32 noundef 10, i32 noundef 6, i32 noundef 2, i32 noundef 1, ptr noundef [[ARRAY_STR]]
+
+  StaticArray[1] = One;
+// Operator = call to assign non-static One handle to StaticArray element.
+// CHECK-NEXT: call {{.*}} ptr @hlsl::RWBuffer<float>::operator=(hlsl::RWBuffer<float> const&)
+// CHECK-SAME: (ptr {{.*}} getelementptr inbounds ([2 x %"class.hlsl::RWBuffer"], ptr @StaticArray, i32 0, i32 1), ptr {{.*}} @One)
+
+  StaticLocal[0] = 123;
+// CHECK-NEXT: %[[PTR0:.*]] = call {{.*}} ptr @hlsl::RWBuffer<float>::operator[](unsigned int)(ptr {{.*}} @main()::StaticLocal, i32 noundef 0)
+// CHECK-NEXT: store float 1.230000e+02, ptr %[[PTR0]]
+
+  StaticOne[1] = 456;
+// CHECK-NEXT: %[[PTR1:.*]] = call {{.*}} ptr @hlsl::RWBuffer<float>::operator[](unsigned int)(ptr {{.*}}) @StaticOne, i32 noundef 1)
+// CHECK-NEXT: store float 4.560000e+02, ptr %[[PTR1]], align 4
+
+  StaticArray[1][2] = 789;
+// CHECK-NEXT: %[[PTR2:.*]] = call {{.*}} ptr @hlsl::RWBuffer<float>::operator[](unsigned int)
+// CHECK-SAME: (ptr {{.*}} getelementptr inbounds ([2 x %"class.hlsl::RWBuffer"], ptr @StaticArray, i32 0, i32 1), i32 noundef 2)
+// CHECK-NEXT: store float 7.890000e+02, ptr %[[PTR2]], align 4
+
+  static RWStructuredBuffer<int> StaticLocalWithCounter;
+// Check that StaticLocalWithCounter is initialized by default constructor (handle set to poison)
+// and not from binding.
+// call void @hlsl::RWStructuredBuffer<int>::RWStructuredBuffer()(ptr {{.*}} @main()::StaticLocalWithCounter)
+
+  static RWStructuredBuffer<int> StaticLocalArrayWithCounter[2];
+
+  StaticLocalWithCounter = OneWithCounter;
+// Operator = call to assign non-static OneWithCounter handles to StaticLocalWithCounter handles.
+// CHECK: call {{.*}} ptr @hlsl::RWStructuredBuffer<int>::operator=(hlsl::RWStructuredBuffer<int> const&)(ptr {{.*}} @main()::StaticLocalWithCounter, ptr {{.*}} @OneWithCounter)
+
+  StaticLocalArrayWithCounter = ArrayWithCounter;
+// Check that each elements of StaticLocalArrayWithCounter is initialized from binding
+// of ArrayWithCounter (register 7, space 4, indices 0 and 1).
+// CHECK: call void @hlsl::RWStructuredBuffer<int>::__createFromBindingWithImplicitCounter(unsigned int, unsigned int, int, unsigned int, char const*, unsigned int)
+// CHECK-SAME: (ptr {{.*}} sret(%"class.hlsl::RWStructuredBuffer") align 4 @main()::StaticLocalArrayWithCounter,
+// CHECK-SAME: i32 noundef 7, i32 noundef 4, i32 noundef 2, i32 noundef 0, ptr noundef [[ARRAYWITHCOUNTER_STR]], i32 noundef 1)
+
+// CHECK-NEXT: call void @hlsl::RWStructuredBuffer<int>::__createFromBindingWithImplicitCounter(unsigned int, unsigned int, int, unsigned int, char const*, unsigned int)
+// CHECK-SAME: (ptr {{.*}} sret(%"class.hlsl::RWStructuredBuffer") align 4 getelementptr ([2 x %"class.hlsl::RWStructuredBuffer"], ptr @main()::StaticLocalArrayWithCounter, i32 0, i32 1),
+// CHECK-SAME: i32 noundef 7, i32 noundef 4, i32 noundef 2, i32 noundef 1, ptr noundef [[ARRAYWITHCOUNTER_STR]], i32 noundef 1)
+}
+
+// No other binding initialization calls should be present.
+// CHECK-NOT: call void @hlsl::RWBuffer<float>::__createFrom{{.*}}Binding{{.*}}
diff --git a/clang/tools/clang-scan-deps/ClangScanDeps.cpp b/clang/tools/clang-scan-deps/ClangScanDeps.cpp
index 5f5bf42df5e6b..3efa28b0469c1 100644
--- a/clang/tools/clang-scan-deps/ClangScanDeps.cpp
+++ b/clang/tools/clang-scan-deps/ClangScanDeps.cpp
@@ -284,11 +284,9 @@ class ResourceDirectoryCache {
     if (CachedResourceDir != Cache.end())
       return CachedResourceDir->second;
 
-    std::vector<StringRef> PrintResourceDirArgs{ClangBinaryName};
-    if (ClangCLMode)
-      PrintResourceDirArgs.push_back("/clang:-print-resource-dir");
-    else
-      PrintResourceDirArgs.push_back("-print-resource-dir");
+    const std::array<StringRef, 2> PrintResourceDirArgs{
+        ClangBinaryName,
+        ClangCLMode ? "/clang:-print-resource-dir" : "-print-resource-dir"};
 
     llvm::SmallString<64> OutputFile, ErrorFile;
     llvm::sys::fs::createTemporaryFile("print-resource-dir-output",
diff --git a/flang-rt/include/flang-rt/runtime/descriptor.h b/flang-rt/include/flang-rt/runtime/descriptor.h
index 92d2210cbc640..8c848fcab24ee 100644
--- a/flang-rt/include/flang-rt/runtime/descriptor.h
+++ b/flang-rt/include/flang-rt/runtime/descriptor.h
@@ -512,9 +512,7 @@ class Descriptor {
 
   RT_API_ATTRS void Check() const;
 
-  // When dumpRawType, dumps stringified CFI_type_*, otherwise
-  // try to canonicalize and print as a Fortran type.
-  void Dump(FILE * = stdout, bool dumpRawType = true) const;
+  void Dump(FILE * = stdout) const;
 
   RT_API_ATTRS inline bool HasAddendum() const {
     return raw_.extra & _CFI_ADDENDUM_FLAG;
diff --git a/flang-rt/lib/runtime/descriptor.cpp b/flang-rt/lib/runtime/descriptor.cpp
index 8c547107a47de..c95da0a5371e5 100644
--- a/flang-rt/lib/runtime/descriptor.cpp
+++ b/flang-rt/lib/runtime/descriptor.cpp
@@ -292,168 +292,14 @@ RT_API_ATTRS void Descriptor::Check() const {
   // TODO
 }
 
-static const char *GetTypeStr(ISO::CFI_type_t type, bool dumpRawType) {
-  if (dumpRawType) {
-#define CASE(x) \
-  case (x): \
-    return #x;
-    switch (type) {
-      CASE(CFI_type_signed_char)
-      CASE(CFI_type_short)
-      CASE(CFI_type_int)
-      CASE(CFI_type_long)
-      CASE(CFI_type_long_long)
-      CASE(CFI_type_size_t)
-      CASE(CFI_type_int8_t)
-      CASE(CFI_type_int16_t)
-      CASE(CFI_type_int32_t)
-      CASE(CFI_type_int64_t)
-      CASE(CFI_type_int128_t)
-      CASE(CFI_type_int_least8_t)
-      CASE(CFI_type_int_least16_t)
-      CASE(CFI_type_int_least32_t)
-      CASE(CFI_type_int_least64_t)
-      CASE(CFI_type_int_least128_t)
-      CASE(CFI_type_int_fast8_t)
-      CASE(CFI_type_int_fast16_t)
-      CASE(CFI_type_int_fast32_t)
-      CASE(CFI_type_int_fast64_t)
-      CASE(CFI_type_int_fast128_t)
-      CASE(CFI_type_intmax_t)
-      CASE(CFI_type_intptr_t)
-      CASE(CFI_type_ptrdiff_t)
-      CASE(CFI_type_half_float)
-      CASE(CFI_type_bfloat)
-      CASE(CFI_type_float)
-      CASE(CFI_type_double)
-      CASE(CFI_type_extended_double)
-      CASE(CFI_type_long_double)
-      CASE(CFI_type_float128)
-      CASE(CFI_type_half_float_Complex)
-      CASE(CFI_type_bfloat_Complex)
-      CASE(CFI_type_float_Complex)
-      CASE(CFI_type_double_Complex)
-      CASE(CFI_type_extended_double_Complex)
-      CASE(CFI_type_long_double_Complex)
-      CASE(CFI_type_float128_Complex)
-      CASE(CFI_type_Bool)
-      CASE(CFI_type_char)
-      CASE(CFI_type_cptr)
-      CASE(CFI_type_struct)
-      CASE(CFI_type_char16_t)
-      CASE(CFI_type_char32_t)
-      CASE(CFI_type_uint8_t)
-      CASE(CFI_type_uint16_t)
-      CASE(CFI_type_uint32_t)
-      CASE(CFI_type_uint64_t)
-      CASE(CFI_type_uint128_t)
-    }
-#undef CASE
-    return nullptr;
-  }
-  TypeCode code{type};
-
-  if (!code.IsValid())
-    return "invalid";
-
-  common::optional<std::pair<TypeCategory, int>> categoryAndKind =
-      code.GetCategoryAndKind();
-  if (!categoryAndKind)
-    return nullptr;
-
-  TypeCategory tcat;
-  int kind;
-  std::tie(tcat, kind) = *categoryAndKind;
-
-#define CASE(cat, k) \
-  case (k): \
-    return #cat "(kind=" #k ")";
-  switch (tcat) {
-  case TypeCategory::Integer:
-    switch (kind) {
-      CASE(INTEGER, 1)
-      CASE(INTEGER, 2)
-      CASE(INTEGER, 4)
-      CASE(INTEGER, 8)
-      CASE(INTEGER, 16)
-    }
-    break;
-  case TypeCategory::Unsigned:
-    switch (kind) {
-      CASE(UNSIGNED, 1)
-      CASE(UNSIGNED, 2)
-      CASE(UNSIGNED, 4)
-      CASE(UNSIGNED, 8)
-      CASE(UNSIGNED, 16)
-    }
-    break;
-  case TypeCategory::Real:
-    switch (kind) {
-      CASE(REAL, 2)
-      CASE(REAL, 3)
-      CASE(REAL, 4)
-      CASE(REAL, 8)
-      CASE(REAL, 10)
-      CASE(REAL, 16)
-    }
-    break;
-  case TypeCategory::Complex:
-    switch (kind) {
-      CASE(COMPLEX, 2)
-      CASE(COMPLEX, 3)
-      CASE(COMPLEX, 4)
-      CASE(COMPLEX, 8)
-      CASE(COMPLEX, 10)
-      CASE(COMPLEX, 16)
-    }
-    break;
-  case TypeCategory::Character:
-    switch (kind) {
-      CASE(CHARACTER, 1)
-      CASE(CHARACTER, 2)
-      CASE(CHARACTER, 4)
-    }
-    break;
-  case TypeCategory::Logical:
-    switch (kind) {
-      CASE(LOGICAL, 1)
-      CASE(LOGICAL, 2)
-      CASE(LOGICAL, 4)
-      CASE(LOGICAL, 8)
-    }
-    break;
-  case TypeCategory::Derived:
-    return "DERIVED";
-  }
-#undef CASE
-  return nullptr;
-}
-
-void Descriptor::Dump(FILE *f, bool dumpRawType) const {
+void Descriptor::Dump(FILE *f) const {
   std::fprintf(f, "Descriptor @ %p:\n", reinterpret_cast<const void *>(this));
   std::fprintf(f, "  base_addr %p\n", raw_.base_addr);
-  std::fprintf(f, "  elem_len  %zd\n", ElementBytes());
+  std::fprintf(f, "  elem_len  %zd\n", static_cast<std::size_t>(raw_.elem_len));
   std::fprintf(f, "  version   %d\n", static_cast<int>(raw_.version));
-  if (rank() > 0) {
-    std::fprintf(f, "  rank      %d\n", rank());
-  } else {
-    std::fprintf(f, "  scalar\n");
-  }
-  int ty = static_cast<int>(raw_.type);
-  if (const char *tyStr = GetTypeStr(raw_.type, dumpRawType)) {
-    std::fprintf(f, "  type      %d \"%s\"\n", ty, tyStr);
-  } else {
-    std::fprintf(f, "  type      %d\n", ty);
-  }
-  int attr = static_cast<int>(raw_.attribute);
-  if (IsPointer()) {
-    std::fprintf(f, "  attribute %d (pointer) \n", attr);
-  } else if (IsAllocatable()) {
-    std::fprintf(f, "  attribute %d (allocatable)\n", attr);
-  } else {
-    std::fprintf(f, "  attribute %d\n", attr);
-  }
-
+  std::fprintf(f, "  rank      %d\n", static_cast<int>(raw_.rank));
+  std::fprintf(f, "  type      %d\n", static_cast<int>(raw_.type));
+  std::fprintf(f, "  attribute %d\n", static_cast<int>(raw_.attribute));
   std::fprintf(f, "  extra     %d\n", static_cast<int>(raw_.extra));
   std::fprintf(f, "    addendum  %d\n", static_cast<int>(HasAddendum()));
   std::fprintf(f, "    alloc_idx %d\n", static_cast<int>(GetAllocIdx()));
diff --git a/flang-rt/lib/runtime/extensions.cpp b/flang-rt/lib/runtime/extensions.cpp
index d39e429eb88b9..c110b0381890c 100644
--- a/flang-rt/lib/runtime/extensions.cpp
+++ b/flang-rt/lib/runtime/extensions.cpp
@@ -416,14 +416,6 @@ std::int64_t RTNAME(time)() { return time(nullptr); }
 // MCLOCK: returns accumulated CPU time in ticks
 std::int32_t FORTRAN_PROCEDURE_NAME(mclock)() { return std::clock(); }
 
-void RTNAME(ShowDescriptor)(const Fortran::runtime::Descriptor *descr) {
-  if (descr) {
-    descr->Dump(stderr, /*dumpRawType=*/false);
-  } else {
-    std::fprintf(stderr, "NULL\n");
-  }
-}
-
 static void _internal_srand(int seed) { rand_seed = seed ? seed : 123459876; }
 
 // IRAND(I)
diff --git a/flang-rt/unittests/Runtime/Descriptor.cpp b/flang-rt/unittests/Runtime/Descriptor.cpp
index f86ff4cf16a55..4a7bb43a492af 100644
--- a/flang-rt/unittests/Runtime/Descriptor.cpp
+++ b/flang-rt/unittests/Runtime/Descriptor.cpp
@@ -9,7 +9,6 @@
 #include "flang-rt/runtime/descriptor.h"
 #include "tools.h"
 #include "gtest/gtest.h"
-#include <regex>
 
 using namespace Fortran::runtime;
 
@@ -159,115 +158,3 @@ TEST(Descriptor, FixedStride) {
   EXPECT_TRUE(descriptor.IsContiguous());
   EXPECT_EQ(descriptor.FixedStride().value_or(-666), 0);
 }
-
-static std::string getAddrFilteredContent(FILE *fin) {
-  rewind(fin);
-  std::ostringstream content;
-  char buffer[1024];
-  size_t bytes_read;
-  while ((bytes_read = fread(buffer, 1, sizeof(buffer), fin)) > 0) {
-    content.write(buffer, bytes_read);
-  }
-  return std::regex_replace(
-      content.str(), std::regex("(0x[0-9a-fA-F]*)"), "[address]");
-}
-
-TEST(Descriptor, Dump) {
-  StaticDescriptor<4> staticDesc[2];
-  Descriptor &descriptor{staticDesc[0].descriptor()};
-  using Type = std::int32_t;
-  Type data[8][8][8];
-  constexpr int four{static_cast<int>(sizeof data[0][0][0])};
-  TypeCode integer{TypeCategory::Integer, four};
-  // Scalar
-  descriptor.Establish(integer, four, data, 0);
-  FILE *tmpf = tmpfile();
-  ASSERT_TRUE(tmpf) << "tmpfile returned NULL";
-  auto resetTmpFile = [tmpf]() {
-    rewind(tmpf);
-    ftruncate(fileno(tmpf), 0);
-  };
-
-  descriptor.Dump(tmpf, /*dumpRawType=*/false);
-  // also dump as CFI type
-  descriptor.Dump(tmpf, /*dumpRawType=*/true);
-  std::string output = getAddrFilteredContent(tmpf);
-  ASSERT_STREQ(output.c_str(),
-      "Descriptor @ [address]:\n"
-      "  base_addr [address]\n"
-      "  elem_len  4\n"
-      "  version   20240719\n"
-      "  scalar\n"
-      "  type      9 \"INTEGER(kind=4)\"\n"
-      "  attribute 0\n"
-      "  extra     0\n"
-      "    addendum  0\n"
-      "    alloc_idx 0\n"
-      "Descriptor @ [address]:\n"
-      "  base_addr [address]\n"
-      "  elem_len  4\n"
-      "  version   20240719\n"
-      "  scalar\n"
-      "  type      9 \"CFI_type_int32_t\"\n"
-      "  attribute 0\n"
-      "  extra     0\n"
-      "    addendum  0\n"
-      "    alloc_idx 0\n");
-
-  // Contiguous matrix (0:7, 0:7)
-  SubscriptValue extent[3]{8, 8, 8};
-  descriptor.Establish(integer, four, data, 2, extent);
-  resetTmpFile();
-  descriptor.Dump(tmpf, /*dumpRawType=*/false);
-  output = getAddrFilteredContent(tmpf);
-  ASSERT_STREQ(output.c_str(),
-      "Descriptor @ [address]:\n"
-      "  base_addr [address]\n"
-      "  elem_len  4\n"
-      "  version   20240719\n"
-      "  rank      2\n"
-      "  type      9 \"INTEGER(kind=4)\"\n"
-      "  attribute 0\n"
-      "  extra     0\n"
-      "    addendum  0\n"
-      "    alloc_idx 0\n"
-      "  dim[0] lower_bound 0\n"
-      "         extent      8\n"
-      "         sm          4\n"
-      "  dim[1] lower_bound 0\n"
-      "         extent      8\n"
-      "         sm          32\n");
-
-  TypeCode real{TypeCategory::Real, four};
-  // Discontiguous real 3-D array (0:7, 0:6:2, 0:6:2)
-  descriptor.Establish(real, four, data, 3, extent);
-  descriptor.GetDimension(1).SetExtent(4);
-  descriptor.GetDimension(1).SetByteStride(8 * 2 * four);
-  descriptor.GetDimension(2).SetExtent(4);
-  descriptor.GetDimension(2).SetByteStride(8 * 8 * 2 * four);
-
-  resetTmpFile();
-  descriptor.Dump(tmpf, /*dumpRawType=*/false);
-  output = getAddrFilteredContent(tmpf);
-  ASSERT_STREQ(output.c_str(),
-      "Descriptor @ [address]:\n"
-      "  base_addr [address]\n"
-      "  elem_len  4\n"
-      "  version   20240719\n"
-      "  rank      3\n"
-      "  type      27 \"REAL(kind=4)\"\n"
-      "  attribute 0\n"
-      "  extra     0\n"
-      "    addendum  0\n"
-      "    alloc_idx 0\n"
-      "  dim[0] lower_bound 0\n"
-      "         extent      8\n"
-      "         sm          4\n"
-      "  dim[1] lower_bound 0\n"
-      "         extent      4\n"
-      "         sm          64\n"
-      "  dim[2] lower_bound 0\n"
-      "         extent      4\n"
-      "         sm          512\n");
-  fclose(tmpf);
-}
diff --git a/flang/docs/Intrinsics.md b/flang/docs/Intrinsics.md
index ecc9143af69fd..31bead9f8bfdc 100644
--- a/flang/docs/Intrinsics.md
+++ b/flang/docs/Intrinsics.md
@@ -1414,48 +1414,6 @@ This is prefixed by `STRING`, a colon and a space.
 - **Class:** subroutine
 - **Syntax:** `CALL PERROR(STRING)`
 
-### Non-Standard Intrinsics: SHOW_DESCRIPTOR
-
-#### Description
-`SHOW_DESCRIPTOR(VAR)` prints (on the C stderr stream) a contents of a descriptor for the variable VAR,
-which can be of any type and rank, including scalars.
-Requires use of flang_debug module.
-
-Here is an example of its output:
-```
-Descriptor @ 0x7ffe506fc368:
-  base_addr 0x55944caef0f0
-  elem_len  4
-  version   20240719
-  rank      1
-  type      9 "INTEGER(kind=4)"
-  attribute 2 (allocatable)
-  extra     0
-    addendum  0
-    alloc_idx 0
-  dim[0] lower_bound 1
-         extent      5
-         sm          4
-```
-
-#### Usage and Info
-- **Standard:** flang extension
-- **Class:** subroutine
-- **Syntax:** `CALL show_descriptor(VAR)`
-
-#### Example
-```Fortran
-subroutine test
-  use flang_debug
-  implicit none
-  character(len=9) :: c = 'Hey buddy'
-  integer :: a(5)
-  call show_descriptor(c)
-  call show_descriptor(c(1:3))
-  call show_descriptor(a)
-end subroutine test
-```
-
 ### Non-Standard Intrinsics: SRAND
 
 #### Description
diff --git a/flang/include/flang/Optimizer/Builder/IntrinsicCall.h b/flang/include/flang/Optimizer/Builder/IntrinsicCall.h
index b248106b51101..0ae9177f98fd8 100644
--- a/flang/include/flang/Optimizer/Builder/IntrinsicCall.h
+++ b/flang/include/flang/Optimizer/Builder/IntrinsicCall.h
@@ -413,7 +413,6 @@ struct IntrinsicLibrary {
   template <typename Shift>
   mlir::Value genShift(mlir::Type resultType, llvm::ArrayRef<mlir::Value>);
   mlir::Value genShiftA(mlir::Type resultType, llvm::ArrayRef<mlir::Value>);
-  void genShowDescriptor(llvm::ArrayRef<fir::ExtendedValue>);
   mlir::Value genSign(mlir::Type, llvm::ArrayRef<mlir::Value>);
   mlir::Value genSind(mlir::Type, llvm::ArrayRef<mlir::Value>);
   mlir::Value genSinpi(mlir::Type, llvm::ArrayRef<mlir::Value>);
diff --git a/flang/include/flang/Optimizer/Builder/Runtime/Intrinsics.h b/flang/include/flang/Optimizer/Builder/Runtime/Intrinsics.h
index 2f52ffce5eb43..30c3189366cec 100644
--- a/flang/include/flang/Optimizer/Builder/Runtime/Intrinsics.h
+++ b/flang/include/flang/Optimizer/Builder/Runtime/Intrinsics.h
@@ -111,9 +111,6 @@ void genSleep(fir::FirOpBuilder &builder, mlir::Location loc,
 mlir::Value genChdir(fir::FirOpBuilder &builder, mlir::Location loc,
                      mlir::Value name);
 
-/// generate dump of a descriptor
-void genShowDescriptor(fir::FirOpBuilder &builder, mlir::Location loc,
-                       mlir::Value descriptor);
 
 mlir::Value genIrand(fir::FirOpBuilder &builder, mlir::Location loc,
                      mlir::Value i);
diff --git a/flang/include/flang/Optimizer/Dialect/CUF/CUFOps.td b/flang/include/flang/Optimizer/Dialect/CUF/CUFOps.td
index 07bb47e26b968..3fda523acb382 100644
--- a/flang/include/flang/Optimizer/Dialect/CUF/CUFOps.td
+++ b/flang/include/flang/Optimizer/Dialect/CUF/CUFOps.td
@@ -350,15 +350,14 @@ def cuf_SharedMemoryOp
   let arguments = (ins TypeAttr:$in_type, OptionalAttr<StrAttr>:$uniq_name,
       OptionalAttr<StrAttr>:$bindc_name, Variadic<AnyIntegerType>:$typeparams,
       Variadic<AnyIntegerType>:$shape,
-      Optional<AnyIntegerType>:$offset // offset in bytes from the shared memory
-                                       // base address.
-  );
+      // offset in bytes from the shared memory base address.
+      Optional<AnyIntegerType>:$offset, OptionalAttr<I64Attr>:$alignment);
 
   let results = (outs fir_ReferenceType:$ptr);
 
   let assemblyFormat = [{
       (`[` $offset^ `:` type($offset) `]`)? $in_type (`(` $typeparams^ `:` type($typeparams) `)`)?
-        (`,` $shape^ `:` type($shape) )?  attr-dict `->` qualified(type($ptr))
+        (`,` $shape^ `:` type($shape) )?  (`align` $alignment^ )? attr-dict `->` qualified(type($ptr))
   }];
 
   let builders = [OpBuilder<(ins "mlir::Type":$inType,
diff --git a/flang/include/flang/Optimizer/OpenACC/Support/FIROpenACCTypeInterfaces.h b/flang/include/flang/Optimizer/OpenACC/Support/FIROpenACCTypeInterfaces.h
index 3167c554abbdd..0f133623475f8 100644
--- a/flang/include/flang/Optimizer/OpenACC/Support/FIROpenACCTypeInterfaces.h
+++ b/flang/include/flang/Optimizer/OpenACC/Support/FIROpenACCTypeInterfaces.h
@@ -43,6 +43,15 @@ struct OpenACCPointerLikeModel
                mlir::TypedValue<mlir::acc::PointerLikeType> destination,
                mlir::TypedValue<mlir::acc::PointerLikeType> source,
                mlir::Type varType) const;
+
+  mlir::Value genLoad(mlir::Type pointer, mlir::OpBuilder &builder,
+                      mlir::Location loc,
+                      mlir::TypedValue<mlir::acc::PointerLikeType> srcPtr,
+                      mlir::Type valueType) const;
+
+  bool genStore(mlir::Type pointer, mlir::OpBuilder &builder,
+                mlir::Location loc, mlir::Value valueToStore,
+                mlir::TypedValue<mlir::acc::PointerLikeType> destPtr) const;
 };
 
 template <typename T>
diff --git a/flang/include/flang/Runtime/extensions.h b/flang/include/flang/Runtime/extensions.h
index f3ed15b86f1be..f2765a5987ea1 100644
--- a/flang/include/flang/Runtime/extensions.h
+++ b/flang/include/flang/Runtime/extensions.h
@@ -25,11 +25,6 @@ typedef std::uint32_t gid_t;
 #else
 #include "sys/types.h" //pid_t
 #endif
-namespace Fortran {
-namespace runtime {
-class Descriptor;
-}
-} // namespace Fortran
 
 extern "C" {
 
@@ -107,9 +102,6 @@ int FORTRAN_PROCEDURE_NAME(mclock)();
 float FORTRAN_PROCEDURE_NAME(secnds)(float *refTime);
 float RTNAME(Secnds)(float *refTime, const char *sourceFile, int line);
 
-// Extension subroutine SHOW_DESCRIPTOR(D)
-void RTNAME(ShowDescriptor)(const Fortran::runtime::Descriptor *descr);
-
 // GNU extension function IRAND(I)
 int RTNAME(Irand)(int *i);
 
diff --git a/flang/lib/Evaluate/intrinsics.cpp b/flang/lib/Evaluate/intrinsics.cpp
index d69400e0ec188..bbcb766274e7f 100644
--- a/flang/lib/Evaluate/intrinsics.cpp
+++ b/flang/lib/Evaluate/intrinsics.cpp
@@ -1713,8 +1713,6 @@ static const IntrinsicInterface intrinsicSubroutine[]{
         {}, Rank::scalar, IntrinsicClass::impureSubroutine},
     {"second", {{"time", DefaultReal, Rank::scalar}}, {}, Rank::scalar,
         IntrinsicClass::impureSubroutine},
-    {"__builtin_show_descriptor", {{"d", AnyData, Rank::anyOrAssumedRank}}, {},
-        Rank::elemental, IntrinsicClass::impureSubroutine},
     {"system",
         {{"command", DefaultChar, Rank::scalar},
             {"exitstat", DefaultInt, Rank::scalar, Optionality::optional,
diff --git a/flang/lib/Optimizer/Builder/CUDAIntrinsicCall.cpp b/flang/lib/Optimizer/Builder/CUDAIntrinsicCall.cpp
index 270037f5fcb00..ae6120826f8d2 100644
--- a/flang/lib/Optimizer/Builder/CUDAIntrinsicCall.cpp
+++ b/flang/lib/Optimizer/Builder/CUDAIntrinsicCall.cpp
@@ -17,6 +17,8 @@
 #include "flang/Evaluate/common.h"
 #include "flang/Optimizer/Builder/FIRBuilder.h"
 #include "flang/Optimizer/Builder/MutableBox.h"
+#include "flang/Optimizer/Dialect/CUF/CUFOps.h"
+#include "flang/Optimizer/HLFIR/HLFIROps.h"
 #include "mlir/Dialect/Index/IR/IndexOps.h"
 #include "mlir/Dialect/SCF/IR/SCF.h"
 #include "mlir/Dialect/Vector/IR/VectorOps.h"
@@ -1489,6 +1491,13 @@ void CUDAIntrinsicLibrary::genTMABulkG2S(
       builder, loc, dst, src, barrier, fir::getBase(args[3]), {}, {});
 }
 
+static void setAlignment(mlir::Value ptr, unsigned alignment) {
+  if (auto declareOp = mlir::dyn_cast<hlfir::DeclareOp>(ptr.getDefiningOp()))
+    if (auto sharedOp = mlir::dyn_cast<cuf::SharedMemoryOp>(
+            declareOp.getMemref().getDefiningOp()))
+      sharedOp.setAlignment(alignment);
+}
+
 static void genTMABulkLoad(fir::FirOpBuilder &builder, mlir::Location loc,
                            mlir::Value barrier, mlir::Value src,
                            mlir::Value dst, mlir::Value nelem,
@@ -1496,8 +1505,11 @@ static void genTMABulkLoad(fir::FirOpBuilder &builder, mlir::Location loc,
   mlir::Value size = mlir::arith::MulIOp::create(builder, loc, nelem, eleSize);
   auto llvmPtrTy = mlir::LLVM::LLVMPointerType::get(builder.getContext());
   barrier = builder.createConvert(loc, llvmPtrTy, barrier);
-  dst = builder.createConvert(loc, llvmPtrTy, dst);
-  src = builder.createConvert(loc, llvmPtrTy, src);
+  setAlignment(dst, 16);
+  dst = convertPtrToNVVMSpace(builder, loc, dst,
+                              mlir::NVVM::NVVMMemorySpace::Shared);
+  src = convertPtrToNVVMSpace(builder, loc, src,
+                              mlir::NVVM::NVVMMemorySpace::Shared);
   mlir::NVVM::InlinePtxOp::create(
       builder, loc, mlir::TypeRange{}, {dst, src, size, barrier}, {},
       "cp.async.bulk.shared::cluster.global.mbarrier::complete_tx::bytes [%0], "
diff --git a/flang/lib/Optimizer/Builder/IntrinsicCall.cpp b/flang/lib/Optimizer/Builder/IntrinsicCall.cpp
index 75a74eeb18417..3619e5bb942db 100644
--- a/flang/lib/Optimizer/Builder/IntrinsicCall.cpp
+++ b/flang/lib/Optimizer/Builder/IntrinsicCall.cpp
@@ -727,10 +727,6 @@ static constexpr IntrinsicHandler handlers[]{
     {"shifta", &I::genShiftA},
     {"shiftl", &I::genShift<mlir::arith::ShLIOp>},
     {"shiftr", &I::genShift<mlir::arith::ShRUIOp>},
-    {"show_descriptor",
-     &I::genShowDescriptor,
-     {{{"d", asBox}}},
-     /*isElemental=*/false},
     {"sign", &I::genSign},
     {"signal",
      &I::genSignalSubroutine,
@@ -7888,16 +7884,6 @@ mlir::Value IntrinsicLibrary::genShiftA(mlir::Type resultType,
   return result;
 }
 
-void IntrinsicLibrary::genShowDescriptor(
-    llvm::ArrayRef<fir::ExtendedValue> args) {
-  assert(args.size() == 1 && "expected single argument for show_descriptor");
-  const mlir::Value descriptor = fir::getBase(args[0]);
-
-  assert(fir::isa_box_type(descriptor.getType()) &&
-         "argument must have been lowered to box type");
-  fir::runtime::genShowDescriptor(builder, loc, descriptor);
-}
-
 // SIGNAL
 void IntrinsicLibrary::genSignalSubroutine(
     llvm::ArrayRef<fir::ExtendedValue> args) {
diff --git a/flang/lib/Optimizer/Builder/Runtime/Intrinsics.cpp b/flang/lib/Optimizer/Builder/Runtime/Intrinsics.cpp
index caeed0c0f6bfe..4d366135c305f 100644
--- a/flang/lib/Optimizer/Builder/Runtime/Intrinsics.cpp
+++ b/flang/lib/Optimizer/Builder/Runtime/Intrinsics.cpp
@@ -471,13 +471,6 @@ mlir::Value fir::runtime::genChdir(fir::FirOpBuilder &builder,
   return fir::CallOp::create(builder, loc, func, args).getResult(0);
 }
 
-void fir::runtime::genShowDescriptor(fir::FirOpBuilder &builder,
-                                     mlir::Location loc, mlir::Value descAddr) {
-  mlir::func::FuncOp func{
-      fir::runtime::getRuntimeFunc<mkRTKey(ShowDescriptor)>(loc, builder)};
-  fir::CallOp::create(builder, loc, func, descAddr);
-}
-
 mlir::Value fir::runtime::genIrand(fir::FirOpBuilder &builder,
                                    mlir::Location loc, mlir::Value i) {
   auto runtimeFunc = fir::runtime::getRuntimeFunc<mkRTKey(Irand)>(loc, builder);
diff --git a/flang/lib/Optimizer/Dialect/CUF/CUFOps.cpp b/flang/lib/Optimizer/Dialect/CUF/CUFOps.cpp
index 687007d957225..671e5f9455c22 100644
--- a/flang/lib/Optimizer/Dialect/CUF/CUFOps.cpp
+++ b/flang/lib/Optimizer/Dialect/CUF/CUFOps.cpp
@@ -333,7 +333,7 @@ void cuf::SharedMemoryOp::build(
       bindcName.empty() ? mlir::StringAttr{} : builder.getStringAttr(bindcName);
   build(builder, result, wrapAllocaResultType(inType),
         mlir::TypeAttr::get(inType), nameAttr, bindcAttr, typeparams, shape,
-        /*offset=*/mlir::Value{});
+        /*offset=*/mlir::Value{}, /*alignment=*/mlir::IntegerAttr{});
   result.addAttributes(attributes);
 }
 
diff --git a/flang/lib/Optimizer/OpenACC/Support/FIROpenACCTypeInterfaces.cpp b/flang/lib/Optimizer/OpenACC/Support/FIROpenACCTypeInterfaces.cpp
index ae0f5fb8197fa..9fcc7d3681c39 100644
--- a/flang/lib/Optimizer/OpenACC/Support/FIROpenACCTypeInterfaces.cpp
+++ b/flang/lib/Optimizer/OpenACC/Support/FIROpenACCTypeInterfaces.cpp
@@ -1014,4 +1014,114 @@ template bool OpenACCPointerLikeModel<fir::LLVMPointerType>::genCopy(
     mlir::TypedValue<mlir::acc::PointerLikeType> source,
     mlir::Type varType) const;
 
+template <typename Ty>
+mlir::Value OpenACCPointerLikeModel<Ty>::genLoad(
+    mlir::Type pointer, mlir::OpBuilder &builder, mlir::Location loc,
+    mlir::TypedValue<mlir::acc::PointerLikeType> srcPtr,
+    mlir::Type valueType) const {
+
+  // Unwrap to get the pointee type.
+  mlir::Type pointeeTy = fir::dyn_cast_ptrEleTy(pointer);
+  assert(pointeeTy && "expected pointee type to be extractable");
+
+  // Box types contain both a descriptor and referenced data. The genLoad API
+  // handles simple loads and cannot properly manage both parts.
+  if (fir::isa_box_type(pointeeTy))
+    return {};
+
+  // Unlimited polymorphic (class(*)) cannot be handled because type is unknown.
+  if (fir::isUnlimitedPolymorphicType(pointeeTy))
+    return {};
+
+  // Return empty for dynamic size types because the load logic
+  // cannot be determined simply from the type.
+  if (fir::hasDynamicSize(pointeeTy))
+    return {};
+
+  mlir::Value loadedValue = fir::LoadOp::create(builder, loc, srcPtr);
+
+  // If valueType is provided and differs from the loaded type, insert a convert
+  if (valueType && loadedValue.getType() != valueType)
+    return fir::ConvertOp::create(builder, loc, valueType, loadedValue);
+
+  return loadedValue;
+}
+
+template mlir::Value OpenACCPointerLikeModel<fir::ReferenceType>::genLoad(
+    mlir::Type pointer, mlir::OpBuilder &builder, mlir::Location loc,
+    mlir::TypedValue<mlir::acc::PointerLikeType> srcPtr,
+    mlir::Type valueType) const;
+
+template mlir::Value OpenACCPointerLikeModel<fir::PointerType>::genLoad(
+    mlir::Type pointer, mlir::OpBuilder &builder, mlir::Location loc,
+    mlir::TypedValue<mlir::acc::PointerLikeType> srcPtr,
+    mlir::Type valueType) const;
+
+template mlir::Value OpenACCPointerLikeModel<fir::HeapType>::genLoad(
+    mlir::Type pointer, mlir::OpBuilder &builder, mlir::Location loc,
+    mlir::TypedValue<mlir::acc::PointerLikeType> srcPtr,
+    mlir::Type valueType) const;
+
+template mlir::Value OpenACCPointerLikeModel<fir::LLVMPointerType>::genLoad(
+    mlir::Type pointer, mlir::OpBuilder &builder, mlir::Location loc,
+    mlir::TypedValue<mlir::acc::PointerLikeType> srcPtr,
+    mlir::Type valueType) const;
+
+template <typename Ty>
+bool OpenACCPointerLikeModel<Ty>::genStore(
+    mlir::Type pointer, mlir::OpBuilder &builder, mlir::Location loc,
+    mlir::Value valueToStore,
+    mlir::TypedValue<mlir::acc::PointerLikeType> destPtr) const {
+
+  // Unwrap to get the pointee type.
+  mlir::Type pointeeTy = fir::dyn_cast_ptrEleTy(pointer);
+  assert(pointeeTy && "expected pointee type to be extractable");
+
+  // Box types contain both a descriptor and referenced data. The genStore API
+  // handles simple stores and cannot properly manage both parts.
+  if (fir::isa_box_type(pointeeTy))
+    return false;
+
+  // Unlimited polymorphic (class(*)) cannot be handled because type is unknown.
+  if (fir::isUnlimitedPolymorphicType(pointeeTy))
+    return false;
+
+  // Return false for dynamic size types because the store logic
+  // cannot be determined simply from the type.
+  if (fir::hasDynamicSize(pointeeTy))
+    return false;
+
+  // Get the type from the value being stored
+  mlir::Type valueType = valueToStore.getType();
+  mlir::Value convertedValue = valueToStore;
+
+  // If the value type differs from the pointee type, insert a convert
+  if (valueType != pointeeTy)
+    convertedValue =
+        fir::ConvertOp::create(builder, loc, pointeeTy, valueToStore);
+
+  fir::StoreOp::create(builder, loc, convertedValue, destPtr);
+  return true;
+}
+
+template bool OpenACCPointerLikeModel<fir::ReferenceType>::genStore(
+    mlir::Type pointer, mlir::OpBuilder &builder, mlir::Location loc,
+    mlir::Value valueToStore,
+    mlir::TypedValue<mlir::acc::PointerLikeType> destPtr) const;
+
+template bool OpenACCPointerLikeModel<fir::PointerType>::genStore(
+    mlir::Type pointer, mlir::OpBuilder &builder, mlir::Location loc,
+    mlir::Value valueToStore,
+    mlir::TypedValue<mlir::acc::PointerLikeType> destPtr) const;
+
+template bool OpenACCPointerLikeModel<fir::HeapType>::genStore(
+    mlir::Type pointer, mlir::OpBuilder &builder, mlir::Location loc,
+    mlir::Value valueToStore,
+    mlir::TypedValue<mlir::acc::PointerLikeType> destPtr) const;
+
+template bool OpenACCPointerLikeModel<fir::LLVMPointerType>::genStore(
+    mlir::Type pointer, mlir::OpBuilder &builder, mlir::Location loc,
+    mlir::Value valueToStore,
+    mlir::TypedValue<mlir::acc::PointerLikeType> destPtr) const;
+
 } // namespace fir::acc
diff --git a/flang/module/__fortran_builtins.f90 b/flang/module/__fortran_builtins.f90
index a9b60508785db..4d134fa4b62b1 100644
--- a/flang/module/__fortran_builtins.f90
+++ b/flang/module/__fortran_builtins.f90
@@ -28,9 +28,6 @@
   intrinsic :: __builtin_c_f_pointer
   public :: __builtin_c_f_pointer
 
-  intrinsic :: __builtin_show_descriptor
-  public :: __builtin_show_descriptor
-
   intrinsic :: sizeof ! extension
   public :: sizeof
 
diff --git a/flang/module/flang_debug.f90 b/flang/module/flang_debug.f90
deleted file mode 100644
index baab3b2477f49..0000000000000
--- a/flang/module/flang_debug.f90
+++ /dev/null
@@ -1,14 +0,0 @@
-!===-- module/flang_debug.f90 ----------------------------------------------===!
-!
-! Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-! See https://llvm.org/LICENSE.txt for license information.
-! SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-!
-!===------------------------------------------------------------------------===!
-
-module flang_debug
-
-  use __fortran_builtins, only: &
-    show_descriptor => __builtin_show_descriptor
-
-end module flang_debug
diff --git a/flang/test/Fir/OpenACC/pointer-like-interface-load.mlir b/flang/test/Fir/OpenACC/pointer-like-interface-load.mlir
new file mode 100644
index 0000000000000..170ea56b24742
--- /dev/null
+++ b/flang/test/Fir/OpenACC/pointer-like-interface-load.mlir
@@ -0,0 +1,95 @@
+// RUN: fir-opt %s --split-input-file --pass-pipeline="builtin.module(func.func(test-acc-pointer-like-interface{test-mode=load}))" 2>&1 | FileCheck %s
+
+func.func @test_load_scalar_f32() {
+  %ptr = fir.alloca f32 {test.ptr}
+  // CHECK: Successfully generated load for operation: %{{.*}} = fir.alloca f32 {test.ptr}
+  // CHECK: Loaded value type: f32
+  // CHECK: Generated: %{{.*}} = fir.load %{{.*}} : !fir.ref<f32>
+  return
+}
+
+// -----
+
+func.func @test_load_scalar_i32() {
+  %ptr = fir.alloca i32 {test.ptr}
+  // CHECK: Successfully generated load for operation: %{{.*}} = fir.alloca i32 {test.ptr}
+  // CHECK: Loaded value type: i32
+  // CHECK: Generated: %{{.*}} = fir.load %{{.*}} : !fir.ref<i32>
+  return
+}
+
+// -----
+
+func.func @test_load_scalar_i64() {
+  %ptr = fir.alloca i64 {test.ptr}
+  // CHECK: Successfully generated load for operation: %{{.*}} = fir.alloca i64 {test.ptr}
+  // CHECK: Loaded value type: i64
+  // CHECK: Generated: %{{.*}} = fir.load %{{.*}} : !fir.ref<i64>
+  return
+}
+
+// -----
+
+func.func @test_load_heap_scalar() {
+  %ptr = fir.allocmem f64 {test.ptr}
+  // CHECK: Successfully generated load for operation: %{{.*}} = fir.allocmem f64 {test.ptr}
+  // CHECK: Loaded value type: f64
+  // CHECK: Generated: %{{.*}} = fir.load %{{.*}} : !fir.heap<f64>
+  return
+}
+
+// -----
+
+func.func @test_load_logical() {
+  %ptr = fir.alloca !fir.logical<4> {test.ptr}
+  // CHECK: Successfully generated load for operation: %{{.*}} = fir.alloca !fir.logical<4> {test.ptr}
+  // CHECK: Loaded value type: !fir.logical<4>
+  // CHECK: Generated: %{{.*}} = fir.load %{{.*}} : !fir.ref<!fir.logical<4>>
+  return
+}
+
+// -----
+
+func.func @test_load_derived_type() {
+  %ptr = fir.alloca !fir.type<_QTt{i:i32}> {test.ptr}
+  // CHECK: Successfully generated load for operation: %{{.*}} = fir.alloca !fir.type<_QTt{i:i32}> {test.ptr}
+  // CHECK: Loaded value type: !fir.type<_QTt{i:i32}>
+  // CHECK: Generated: %{{.*}} = fir.load %{{.*}} : !fir.ref<!fir.type<_QTt{i:i32}>>
+  return
+}
+
+// -----
+
+func.func @test_load_constant_array() {
+  %ptr = fir.alloca !fir.array<10xf32> {test.ptr}
+  // CHECK: Successfully generated load for operation: %{{.*}} = fir.alloca !fir.array<10xf32> {test.ptr}
+  // CHECK: Loaded value type: !fir.array<10xf32>
+  // CHECK: Generated: %{{.*}} = fir.load %{{.*}} : !fir.ref<!fir.array<10xf32>>
+  return
+}
+
+// -----
+
+func.func @test_load_dynamic_array_fails() {
+  %c10 = arith.constant 10 : index
+  %ptr = fir.alloca !fir.array<?xf32>, %c10 {test.ptr}
+  // CHECK: Failed to generate load for operation: %{{.*}} = fir.alloca !fir.array<?xf32>
+  return
+}
+
+// -----
+
+func.func @test_load_box_fails() {
+  %ptr = fir.alloca !fir.box<!fir.ptr<f32>> {test.ptr}
+  // CHECK: Failed to generate load for operation: %{{.*}} = fir.alloca !fir.box<!fir.ptr<f32>>
+  return
+}
+
+// -----
+
+func.func @test_load_unlimited_polymorphic_fails() {
+  %ptr = fir.alloca !fir.class<none> {test.ptr}
+  // CHECK: Failed to generate load for operation: %{{.*}} = fir.alloca !fir.class<none>
+  return
+}
+
diff --git a/flang/test/Fir/OpenACC/pointer-like-interface-store.mlir b/flang/test/Fir/OpenACC/pointer-like-interface-store.mlir
new file mode 100644
index 0000000000000..5ea4f0e750c65
--- /dev/null
+++ b/flang/test/Fir/OpenACC/pointer-like-interface-store.mlir
@@ -0,0 +1,85 @@
+// RUN: fir-opt %s --split-input-file --pass-pipeline="builtin.module(func.func(test-acc-pointer-like-interface{test-mode=store}))" 2>&1 | FileCheck %s
+
+func.func @test_store_scalar_f32() {
+  %ptr = fir.alloca f32 {test.ptr}
+  // CHECK: Successfully generated store for operation: %{{.*}} = fir.alloca f32 {test.ptr}
+  // CHECK: Generated: %[[VAL:.*]] = arith.constant 4.200000e+01 : f32
+  // CHECK: Generated: fir.store %[[VAL]] to %{{.*}} : !fir.ref<f32>
+  return
+}
+
+// -----
+
+func.func @test_store_scalar_i32() {
+  %ptr = fir.alloca i32 {test.ptr}
+  // CHECK: Successfully generated store for operation: %{{.*}} = fir.alloca i32 {test.ptr}
+  // CHECK: Generated: %[[VAL:.*]] = arith.constant 42 : i32
+  // CHECK: Generated: fir.store %[[VAL]] to %{{.*}} : !fir.ref<i32>
+  return
+}
+
+// -----
+
+func.func @test_store_scalar_i64() {
+  %ptr = fir.alloca i64 {test.ptr}
+  // CHECK: Successfully generated store for operation: %{{.*}} = fir.alloca i64 {test.ptr}
+  // CHECK: Generated: %[[VAL:.*]] = arith.constant 42 : i64
+  // CHECK: Generated: fir.store %[[VAL]] to %{{.*}} : !fir.ref<i64>
+  return
+}
+
+// -----
+
+func.func @test_store_heap_scalar() {
+  %ptr = fir.allocmem f64 {test.ptr}
+  // CHECK: Successfully generated store for operation: %{{.*}} = fir.allocmem f64 {test.ptr}
+  // CHECK: Generated: %[[VAL:.*]] = arith.constant 4.200000e+01 : f64
+  // CHECK: Generated: fir.store %[[VAL]] to %{{.*}} : !fir.heap<f64>
+  return
+}
+
+// -----
+
+func.func @test_store_with_type_conversion() {
+  %ptr = fir.alloca i32 {test.ptr}
+  // CHECK: Successfully generated store for operation: %{{.*}} = fir.alloca i32 {test.ptr}
+  // CHECK: Generated: %[[VAL:.*]] = arith.constant 42 : i32
+  // CHECK: Generated: fir.store %[[VAL]] to %{{.*}} : !fir.ref<i32>
+  return
+}
+
+// -----
+
+func.func @test_store_constant_array() {
+  %val = fir.undefined !fir.array<10xf32> {test.value}
+  %ptr = fir.alloca !fir.array<10xf32> {test.ptr}
+  // CHECK: Successfully generated store for operation: %{{.*}} = fir.alloca !fir.array<10xf32> {test.ptr}
+  // CHECK: Generated: fir.store %{{.*}} to %{{.*}} : !fir.ref<!fir.array<10xf32>>
+  return
+}
+
+// -----
+
+func.func @test_store_dynamic_array_fails() {
+  %c10 = arith.constant 10 : index
+  %ptr = fir.alloca !fir.array<?xf32>, %c10 {test.ptr}
+  // CHECK: Failed to generate store for operation: %{{.*}} = fir.alloca !fir.array<?xf32>
+  return
+}
+
+// -----
+
+func.func @test_store_box_fails() {
+  %ptr = fir.alloca !fir.box<!fir.ptr<f32>> {test.ptr}
+  // CHECK: Failed to generate store for operation: %{{.*}} = fir.alloca !fir.box<!fir.ptr<f32>>
+  return
+}
+
+// -----
+
+func.func @test_store_unlimited_polymorphic_fails() {
+  %ptr = fir.alloca !fir.class<none> {test.ptr}
+  // CHECK: Failed to generate store for operation: %{{.*}} = fir.alloca !fir.class<none>
+  return
+}
+
diff --git a/flang/test/Lower/CUDA/cuda-device-proc.cuf b/flang/test/Lower/CUDA/cuda-device-proc.cuf
index 434322ea22265..1e3c66307c334 100644
--- a/flang/test/Lower/CUDA/cuda-device-proc.cuf
+++ b/flang/test/Lower/CUDA/cuda-device-proc.cuf
@@ -538,11 +538,12 @@ end subroutine
 ! CHECK-LABEL: func.func @_QPtest_tma_bulk_load_c4
 ! CHECK: %[[BARRIER:.*]]:2 = hlfir.declare %{{.*}} {data_attr = #cuf.cuda<shared>, uniq_name = "_QFtest_tma_bulk_load_c4Ebarrier1"} : (!fir.ref<i64>) -> (!fir.ref<i64>, !fir.ref<i64>)
 ! CHECK: %[[ELEM_COUNT:.*]]:2 = hlfir.declare %{{.*}} {data_attr = #cuf.cuda<device>, uniq_name = "_QFtest_tma_bulk_load_c4Eelem_count"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+! CHECK: cuf.shared_memory !fir.array<1024xcomplex<f32>> align 16 {bindc_name = "tmp", uniq_name = "_QFtest_tma_bulk_load_c4Etmp"} -> !fir.ref<!fir.array<1024xcomplex<f32>>>
 ! CHECK: %[[COUNT:.*]] = fir.load %[[ELEM_COUNT]]#0 : !fir.ref<i32>
 ! CHECK: %[[ELEM_SIZE:.*]] = arith.constant 8 : i32
 ! CHECK: %[[SIZE:.*]] = arith.muli %[[COUNT]], %[[ELEM_SIZE]] : i32
 ! CHECK: %[[BARRIER_PTR:.*]] = fir.convert %[[BARRIER]]#0 : (!fir.ref<i64>) -> !llvm.ptr
-! CHECK: nvvm.inline_ptx "cp.async.bulk.shared::cluster.global.mbarrier::complete_tx::bytes [%0], [%1], %2, [%3];" ro(%{{.*}}, %{{.*}}, %[[SIZE]], %[[BARRIER_PTR]] : !llvm.ptr, !llvm.ptr, i32, !llvm.ptr)
+! CHECK: nvvm.inline_ptx "cp.async.bulk.shared::cluster.global.mbarrier::complete_tx::bytes [%0], [%1], %2, [%3];" ro(%{{.*}}, %{{.*}}, %[[SIZE]], %[[BARRIER_PTR]] : !llvm.ptr<3>, !llvm.ptr<3>, i32, !llvm.ptr)
 ! CHECK: nvvm.inline_ptx "mbarrier.expect_tx.relaxed.cta.shared::cta.b64 [%0], %1;" ro(%[[BARRIER_PTR]], %[[SIZE]] : !llvm.ptr, i32)
 
 attributes(global) subroutine test_tma_bulk_load_c8(a, n)
@@ -557,11 +558,12 @@ end subroutine
 ! CHECK-LABEL: func.func @_QPtest_tma_bulk_load_c8
 ! CHECK: %[[BARRIER:.*]]:2 = hlfir.declare %{{.*}} {data_attr = #cuf.cuda<shared>, uniq_name = "_QFtest_tma_bulk_load_c8Ebarrier1"} : (!fir.ref<i64>) -> (!fir.ref<i64>, !fir.ref<i64>)
 ! CHECK: %[[ELEM_COUNT:.*]]:2 = hlfir.declare %{{.*}} {data_attr = #cuf.cuda<device>, uniq_name = "_QFtest_tma_bulk_load_c8Eelem_count"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+! CHECK: cuf.shared_memory !fir.array<1024xcomplex<f64>> align 16 {bindc_name = "tmp", uniq_name = "_QFtest_tma_bulk_load_c8Etmp"} -> !fir.ref<!fir.array<1024xcomplex<f64>>>
 ! CHECK: %[[COUNT:.*]] = fir.load %[[ELEM_COUNT]]#0 : !fir.ref<i32>
 ! CHECK: %[[ELEM_SIZE:.*]] = arith.constant 16 : i32
 ! CHECK: %[[SIZE:.*]] = arith.muli %[[COUNT]], %[[ELEM_SIZE]] : i32
 ! CHECK: %[[BARRIER_PTR:.*]] = fir.convert %[[BARRIER]]#0 : (!fir.ref<i64>) -> !llvm.ptr
-! CHECK: nvvm.inline_ptx "cp.async.bulk.shared::cluster.global.mbarrier::complete_tx::bytes [%0], [%1], %2, [%3];" ro(%{{.*}}, %{{.*}}, %[[SIZE]], %[[BARRIER_PTR]] : !llvm.ptr, !llvm.ptr, i32, !llvm.ptr)
+! CHECK: nvvm.inline_ptx "cp.async.bulk.shared::cluster.global.mbarrier::complete_tx::bytes [%0], [%1], %2, [%3];" ro(%{{.*}}, %{{.*}}, %[[SIZE]], %[[BARRIER_PTR]] : !llvm.ptr<3>, !llvm.ptr<3>, i32, !llvm.ptr)
 ! CHECK: nvvm.inline_ptx "mbarrier.expect_tx.relaxed.cta.shared::cta.b64 [%0], %1;" ro(%[[BARRIER_PTR]], %[[SIZE]] : !llvm.ptr, i32)
 
 attributes(global) subroutine test_tma_bulk_load_i4(a, n)
@@ -576,11 +578,12 @@ end subroutine
 ! CHECK-LABEL: func.func @_QPtest_tma_bulk_load_i4
 ! CHECK: %[[BARRIER:.*]]:2 = hlfir.declare %{{.*}} {data_attr = #cuf.cuda<shared>, uniq_name = "_QFtest_tma_bulk_load_i4Ebarrier1"} : (!fir.ref<i64>) -> (!fir.ref<i64>, !fir.ref<i64>)
 ! CHECK: %[[ELEM_COUNT:.*]]:2 = hlfir.declare %{{.*}} {data_attr = #cuf.cuda<device>, uniq_name = "_QFtest_tma_bulk_load_i4Eelem_count"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+! CHECK: cuf.shared_memory !fir.array<1024xi32> align 16 {bindc_name = "tmp", uniq_name = "_QFtest_tma_bulk_load_i4Etmp"} -> !fir.ref<!fir.array<1024xi32>>
 ! CHECK: %[[COUNT:.*]] = fir.load %[[ELEM_COUNT]]#0 : !fir.ref<i32>
 ! CHECK: %[[ELEM_SIZE:.*]] = arith.constant 4 : i32
 ! CHECK: %[[SIZE:.*]] = arith.muli %[[COUNT]], %[[ELEM_SIZE]] : i32
 ! CHECK: %[[BARRIER_PTR:.*]] = fir.convert %[[BARRIER]]#0 : (!fir.ref<i64>) -> !llvm.ptr
-! CHECK: nvvm.inline_ptx "cp.async.bulk.shared::cluster.global.mbarrier::complete_tx::bytes [%0], [%1], %2, [%3];" ro(%{{.*}}, %{{.*}}, %[[SIZE]], %[[BARRIER_PTR]] : !llvm.ptr, !llvm.ptr, i32, !llvm.ptr)
+! CHECK: nvvm.inline_ptx "cp.async.bulk.shared::cluster.global.mbarrier::complete_tx::bytes [%0], [%1], %2, [%3];" ro(%{{.*}}, %{{.*}}, %[[SIZE]], %[[BARRIER_PTR]] : !llvm.ptr<3>, !llvm.ptr<3>, i32, !llvm.ptr)
 ! CHECK: nvvm.inline_ptx "mbarrier.expect_tx.relaxed.cta.shared::cta.b64 [%0], %1;" ro(%[[BARRIER_PTR]], %[[SIZE]] : !llvm.ptr, i32)
 
 attributes(global) subroutine test_tma_bulk_load_i8(a, n)
@@ -595,11 +598,12 @@ end subroutine
 ! CHECK-LABEL: func.func @_QPtest_tma_bulk_load_i8
 ! CHECK: %[[BARRIER:.*]]:2 = hlfir.declare %{{.*}} {data_attr = #cuf.cuda<shared>, uniq_name = "_QFtest_tma_bulk_load_i8Ebarrier1"} : (!fir.ref<i64>) -> (!fir.ref<i64>, !fir.ref<i64>)
 ! CHECK: %[[ELEM_COUNT:.*]]:2 = hlfir.declare %{{.*}} {data_attr = #cuf.cuda<device>, uniq_name = "_QFtest_tma_bulk_load_i8Eelem_count"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+! CHECK: cuf.shared_memory !fir.array<1024xi64> align 16 {bindc_name = "tmp", uniq_name = "_QFtest_tma_bulk_load_i8Etmp"} -> !fir.ref<!fir.array<1024xi64>>
 ! CHECK: %[[COUNT:.*]] = fir.load %[[ELEM_COUNT]]#0 : !fir.ref<i32>
 ! CHECK: %[[ELEM_SIZE:.*]] = arith.constant 8 : i32
 ! CHECK: %[[SIZE:.*]] = arith.muli %[[COUNT]], %[[ELEM_SIZE]] : i32
 ! CHECK: %[[BARRIER_PTR:.*]] = fir.convert %[[BARRIER]]#0 : (!fir.ref<i64>) -> !llvm.ptr
-! CHECK: nvvm.inline_ptx "cp.async.bulk.shared::cluster.global.mbarrier::complete_tx::bytes [%0], [%1], %2, [%3];" ro(%{{.*}}, %{{.*}}, %[[SIZE]], %[[BARRIER_PTR]] : !llvm.ptr, !llvm.ptr, i32, !llvm.ptr)
+! CHECK: nvvm.inline_ptx "cp.async.bulk.shared::cluster.global.mbarrier::complete_tx::bytes [%0], [%1], %2, [%3];" ro(%{{.*}}, %{{.*}}, %[[SIZE]], %[[BARRIER_PTR]] : !llvm.ptr<3>, !llvm.ptr<3>, i32, !llvm.ptr)
 ! CHECK: nvvm.inline_ptx "mbarrier.expect_tx.relaxed.cta.shared::cta.b64 [%0], %1;" ro(%[[BARRIER_PTR]], %[[SIZE]] : !llvm.ptr, i32)
 
 attributes(global) subroutine test_tma_bulk_load_r2(a, n)
@@ -614,11 +618,12 @@ end subroutine
 ! CHECK-LABEL: func.func @_QPtest_tma_bulk_load_r2
 ! CHECK: %[[BARRIER:.*]]:2 = hlfir.declare %{{.*}} {data_attr = #cuf.cuda<shared>, uniq_name = "_QFtest_tma_bulk_load_r2Ebarrier1"} : (!fir.ref<i64>) -> (!fir.ref<i64>, !fir.ref<i64>)
 ! CHECK: %[[ELEM_COUNT:.*]]:2 = hlfir.declare %{{.*}} {data_attr = #cuf.cuda<device>, uniq_name = "_QFtest_tma_bulk_load_r2Eelem_count"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+! CHECK: cuf.shared_memory !fir.array<1024xf16> align 16 {bindc_name = "tmp", uniq_name = "_QFtest_tma_bulk_load_r2Etmp"} -> !fir.ref<!fir.array<1024xf16>>
 ! CHECK: %[[COUNT:.*]] = fir.load %[[ELEM_COUNT]]#0 : !fir.ref<i32>
 ! CHECK: %[[ELEM_SIZE:.*]] = arith.constant 2 : i32
 ! CHECK: %[[SIZE:.*]] = arith.muli %[[COUNT]], %[[ELEM_SIZE]] : i32
 ! CHECK: %[[BARRIER_PTR:.*]] = fir.convert %[[BARRIER]]#0 : (!fir.ref<i64>) -> !llvm.ptr
-! CHECK: nvvm.inline_ptx "cp.async.bulk.shared::cluster.global.mbarrier::complete_tx::bytes [%0], [%1], %2, [%3];" ro(%{{.*}}, %{{.*}}, %[[SIZE]], %[[BARRIER_PTR]] : !llvm.ptr, !llvm.ptr, i32, !llvm.ptr)
+! CHECK: nvvm.inline_ptx "cp.async.bulk.shared::cluster.global.mbarrier::complete_tx::bytes [%0], [%1], %2, [%3];" ro(%{{.*}}, %{{.*}}, %[[SIZE]], %[[BARRIER_PTR]] : !llvm.ptr<3>, !llvm.ptr<3>, i32, !llvm.ptr)
 ! CHECK: nvvm.inline_ptx "mbarrier.expect_tx.relaxed.cta.shared::cta.b64 [%0], %1;" ro(%[[BARRIER_PTR]], %[[SIZE]] : !llvm.ptr, i32)
 
 attributes(global) subroutine test_tma_bulk_load_r4(a, n)
@@ -633,11 +638,12 @@ end subroutine
 ! CHECK-LABEL: func.func @_QPtest_tma_bulk_load_r4
 ! CHECK: %[[BARRIER:.*]]:2 = hlfir.declare %{{.*}} {data_attr = #cuf.cuda<shared>, uniq_name = "_QFtest_tma_bulk_load_r4Ebarrier1"} : (!fir.ref<i64>) -> (!fir.ref<i64>, !fir.ref<i64>)
 ! CHECK: %[[ELEM_COUNT:.*]]:2 = hlfir.declare %{{.*}} {data_attr = #cuf.cuda<device>, uniq_name = "_QFtest_tma_bulk_load_r4Eelem_count"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+! CHECK: cuf.shared_memory !fir.array<1024xf32> align 16 {bindc_name = "tmp", uniq_name = "_QFtest_tma_bulk_load_r4Etmp"} -> !fir.ref<!fir.array<1024xf32>>
 ! CHECK: %[[COUNT:.*]] = fir.load %[[ELEM_COUNT]]#0 : !fir.ref<i32>
 ! CHECK: %[[ELEM_SIZE:.*]] = arith.constant 4 : i32
 ! CHECK: %[[SIZE:.*]] = arith.muli %[[COUNT]], %[[ELEM_SIZE]] : i32
 ! CHECK: %[[BARRIER_PTR:.*]] = fir.convert %[[BARRIER]]#0 : (!fir.ref<i64>) -> !llvm.ptr
-! CHECK: nvvm.inline_ptx "cp.async.bulk.shared::cluster.global.mbarrier::complete_tx::bytes [%0], [%1], %2, [%3];" ro(%{{.*}}, %{{.*}}, %[[SIZE]], %[[BARRIER_PTR]] : !llvm.ptr, !llvm.ptr, i32, !llvm.ptr)
+! CHECK: nvvm.inline_ptx "cp.async.bulk.shared::cluster.global.mbarrier::complete_tx::bytes [%0], [%1], %2, [%3];" ro(%{{.*}}, %{{.*}}, %[[SIZE]], %[[BARRIER_PTR]] : !llvm.ptr<3>, !llvm.ptr<3>, i32, !llvm.ptr)
 ! CHECK: nvvm.inline_ptx "mbarrier.expect_tx.relaxed.cta.shared::cta.b64 [%0], %1;" ro(%[[BARRIER_PTR]], %[[SIZE]] : !llvm.ptr, i32)
 
 attributes(global) subroutine test_tma_bulk_load_r8(a, n)
@@ -652,11 +658,12 @@ end subroutine
 ! CHECK-LABEL: func.func @_QPtest_tma_bulk_load_r8
 ! CHECK: %[[BARRIER:.*]]:2 = hlfir.declare %{{.*}} {data_attr = #cuf.cuda<shared>, uniq_name = "_QFtest_tma_bulk_load_r8Ebarrier1"} : (!fir.ref<i64>) -> (!fir.ref<i64>, !fir.ref<i64>)
 ! CHECK: %[[ELEM_COUNT:.*]]:2 = hlfir.declare %{{.*}} {data_attr = #cuf.cuda<device>, uniq_name = "_QFtest_tma_bulk_load_r8Eelem_count"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+! CHECK: cuf.shared_memory !fir.array<1024xf64> align 16 {bindc_name = "tmp", uniq_name = "_QFtest_tma_bulk_load_r8Etmp"} -> !fir.ref<!fir.array<1024xf64>>
 ! CHECK: %[[COUNT:.*]] = fir.load %[[ELEM_COUNT]]#0 : !fir.ref<i32>
 ! CHECK: %[[ELEM_SIZE:.*]] = arith.constant 8 : i32
 ! CHECK: %[[SIZE:.*]] = arith.muli %[[COUNT]], %[[ELEM_SIZE]] : i32
 ! CHECK: %[[BARRIER_PTR:.*]] = fir.convert %[[BARRIER]]#0 : (!fir.ref<i64>) -> !llvm.ptr
-! CHECK: nvvm.inline_ptx "cp.async.bulk.shared::cluster.global.mbarrier::complete_tx::bytes [%0], [%1], %2, [%3];" ro(%{{.*}}, %{{.*}}, %[[SIZE]], %[[BARRIER_PTR]] : !llvm.ptr, !llvm.ptr, i32, !llvm.ptr)
+! CHECK: nvvm.inline_ptx "cp.async.bulk.shared::cluster.global.mbarrier::complete_tx::bytes [%0], [%1], %2, [%3];" ro(%{{.*}}, %{{.*}}, %[[SIZE]], %[[BARRIER_PTR]] : !llvm.ptr<3>, !llvm.ptr<3>, i32, !llvm.ptr)
 ! CHECK: nvvm.inline_ptx "mbarrier.expect_tx.relaxed.cta.shared::cta.b64 [%0], %1;" ro(%[[BARRIER_PTR]], %[[SIZE]] : !llvm.ptr, i32)
 
 attributes(global) subroutine test_tma_bulk_store_c4(c, n)
diff --git a/flang/test/Lower/Intrinsics/show_descriptor.f90 b/flang/test/Lower/Intrinsics/show_descriptor.f90
deleted file mode 100644
index a0b8d3eb4348f..0000000000000
--- a/flang/test/Lower/Intrinsics/show_descriptor.f90
+++ /dev/null
@@ -1,241 +0,0 @@
-! RUN: bbc -emit-fir %s -o - | FileCheck %s
-
-module test_show_descriptor
-use flang_debug
-contains
-subroutine test_int
-! CHECK-LABEL:   func.func @_QMtest_show_descriptorPtest_int() {
-  implicit none
-  integer :: n
-  integer,allocatable :: a(:)
-  n = 5
-  allocate(a(n))
-! CHECK:           %[[C3:.*]] = arith.constant 3 : index
-! CHECK:           %[[C1:.*]] = arith.constant 1 : index
-! CHECK:           %[[C5:.*]] = arith.constant 5 : i32
-! CHECK:           %[[C0:.*]] = arith.constant 0 : index
-! CHECK:           %[[DUMMY_SCOPE_0:.*]] = fir.dummy_scope : !fir.dscope
-! CHECK:           %[[ALLOCA_0:.*]] = fir.alloca !fir.box<!fir.heap<!fir.array<?xi32>>> {bindc_name = "a", uniq_name = "_QMtest_show_descriptorFtest_intEa"}
-! CHECK:           %[[ZERO_BITS_0:.*]] = fir.zero_bits !fir.heap<!fir.array<?xi32>>
-! CHECK:           %[[SHAPE_0:.*]] = fir.shape %[[C0]] : (index) -> !fir.shape<1>
-! CHECK:           %[[EMBOX_0:.*]] = fir.embox %[[ZERO_BITS_0]](%[[SHAPE_0]]) : (!fir.heap<!fir.array<?xi32>>, !fir.shape<1>) -> !fir.box<!fir.heap<!fir.array<?xi32>>>
-! CHECK:           fir.store %[[EMBOX_0]] to %[[ALLOCA_0]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
-! CHECK:           %[[DECLARE_0:.*]] = fir.declare %[[ALLOCA_0]] {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QMtest_show_descriptorFtest_intEa"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>) -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
-! CHECK:           %[[ALLOCA_1:.*]] = fir.alloca i32 {bindc_name = "n", uniq_name = "_QMtest_show_descriptorFtest_intEn"}
-! CHECK:           %[[DECLARE_1:.*]] = fir.declare %[[ALLOCA_1]] {uniq_name = "_QMtest_show_descriptorFtest_intEn"} : (!fir.ref<i32>) -> !fir.ref<i32>
-! CHECK:           fir.store %[[C5]] to %[[DECLARE_1]] : !fir.ref<i32>
-! CHECK:           %[[LOAD_0:.*]] = fir.load %[[DECLARE_1]] : !fir.ref<i32>
-! CHECK:           %[[CONVERT_0:.*]] = fir.convert %[[LOAD_0]] : (i32) -> index
-! CHECK:           %[[CMPI_0:.*]] = arith.cmpi sgt, %[[CONVERT_0]], %[[C0]] : index
-! CHECK:           %[[SELECT_0:.*]] = arith.select %[[CMPI_0]], %[[CONVERT_0]], %[[C0]] : index
-! CHECK:           %[[ALLOCMEM_0:.*]] = fir.allocmem !fir.array<?xi32>, %[[SELECT_0]] {fir.must_be_heap = true, uniq_name = "_QMtest_show_descriptorFtest_intEa.alloc"}
-
-  call show_descriptor(a)
-! CHECK:           %[[SHAPE_1:.*]] = fir.shape %[[SELECT_0]] : (index) -> !fir.shape<1>
-! CHECK:           %[[EMBOX_1:.*]] = fir.embox %[[ALLOCMEM_0]](%[[SHAPE_1]]) : (!fir.heap<!fir.array<?xi32>>, !fir.shape<1>) -> !fir.box<!fir.heap<!fir.array<?xi32>>>
-! CHECK:           fir.store %[[EMBOX_1]] to %[[DECLARE_0]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
-! CHECK:           %[[LOAD_1:.*]] = fir.load %[[DECLARE_0]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
-! CHECK:           fir.call @_FortranAShowDescriptor(%[[LOAD_1]]) fastmath<contract> : (!fir.box<!fir.heap<!fir.array<?xi32>>>) -> ()
-
-  call show_descriptor(a(1:3))
-! CHECK:           %[[LOAD_2:.*]] = fir.load %[[DECLARE_0]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
-! CHECK:           %[[SHAPE_2:.*]] = fir.shape %[[C3]] : (index) -> !fir.shape<1>
-! CHECK:           %[[BOX_ADDR_0:.*]] = fir.box_addr %[[LOAD_2]] : (!fir.box<!fir.heap<!fir.array<?xi32>>>) -> !fir.heap<!fir.array<?xi32>>
-! CHECK:           %[[CONSTANT_4:.*]] = arith.constant 0 : index
-! CHECK:           %[[BOX_DIMS_0:.*]]:3 = fir.box_dims %[[LOAD_2]], %[[CONSTANT_4]] : (!fir.box<!fir.heap<!fir.array<?xi32>>>, index) -> (index, index, index)
-! CHECK:           %[[SHAPE_SHIFT_0:.*]] = fir.shape_shift %[[BOX_DIMS_0]]#0, %[[BOX_DIMS_0]]#1 : (index, index) -> !fir.shapeshift<1>
-! CHECK:           %[[ARRAY_COOR_0:.*]] = fir.array_coor %[[BOX_ADDR_0]](%[[SHAPE_SHIFT_0]]) %[[C1]] : (!fir.heap<!fir.array<?xi32>>, !fir.shapeshift<1>, index) -> !fir.ref<i32>
-! CHECK:           %[[CONVERT_1:.*]] = fir.convert %[[ARRAY_COOR_0]] : (!fir.ref<i32>) -> !fir.ref<!fir.array<3xi32>>
-! CHECK:           %[[EMBOX_2:.*]] = fir.embox %[[CONVERT_1]](%[[SHAPE_2]]) : (!fir.ref<!fir.array<3xi32>>, !fir.shape<1>) -> !fir.box<!fir.array<3xi32>>
-! CHECK:           fir.call @_FortranAShowDescriptor(%[[EMBOX_2]]) fastmath<contract> : (!fir.box<!fir.array<3xi32>>) -> ()
-  deallocate(a)
-end subroutine test_int
-
-subroutine test_char
-! CHECK-LABEL:   func.func @_QMtest_show_descriptorPtest_char() {
-  implicit none
-  character(len=9) :: c = 'Hey buddy'
-  call show_descriptor(c)
-! CHECK:           %[[C3:.*]] = arith.constant 3 : index
-! CHECK:           %[[C1:.*]] = arith.constant 1 : index
-! CHECK:           %[[C9:.*]] = arith.constant 9 : index
-! CHECK:           %[[DUMMY_SCOPE_0:.*]] = fir.dummy_scope : !fir.dscope
-! CHECK:           %[[ADDRESS_OF_0:.*]] = fir.address_of(@_QMtest_show_descriptorFtest_charEc) : !fir.ref<!fir.char<1,9>>
-! CHECK:           %[[DECLARE_0:.*]] = fir.declare %[[ADDRESS_OF_0]] typeparams %[[C9]] {uniq_name = "_QMtest_show_descriptorFtest_charEc"} : (!fir.ref<!fir.char<1,9>>, index) -> !fir.ref<!fir.char<1,9>>
-! CHECK:           %[[EMBOX_0:.*]] = fir.embox %[[DECLARE_0]] : (!fir.ref<!fir.char<1,9>>) -> !fir.box<!fir.char<1,9>>
-! CHECK:           fir.call @_FortranAShowDescriptor(%[[EMBOX_0]]) fastmath<contract> : (!fir.box<!fir.char<1,9>>) -> ()
-
-  call show_descriptor(c(1:3))
-! CHECK:           %[[C1_0:.*]] = arith.constant 1 : index
-! CHECK:           %[[SUBI_0:.*]] = arith.subi %[[C1]], %[[C1_0]] : index
-! CHECK:           %[[CONVERT_0:.*]] = fir.convert %[[DECLARE_0]] : (!fir.ref<!fir.char<1,9>>) -> !fir.ref<!fir.array<9x!fir.char<1>>>
-! CHECK:           %[[COORDINATE_OF_0:.*]] = fir.coordinate_of %[[CONVERT_0]], %[[SUBI_0]] : (!fir.ref<!fir.array<9x!fir.char<1>>>, index) -> !fir.ref<!fir.char<1>>
-! CHECK:           %[[CONVERT_1:.*]] = fir.convert %[[COORDINATE_OF_0]] : (!fir.ref<!fir.char<1>>) -> !fir.ref<!fir.char<1,3>>
-! CHECK:           %[[EMBOX_1:.*]] = fir.embox %[[CONVERT_1]] : (!fir.ref<!fir.char<1,3>>) -> !fir.box<!fir.char<1,3>>
-! CHECK:           fir.call @_FortranAShowDescriptor(%[[EMBOX_1]]) fastmath<contract> : (!fir.box<!fir.char<1,3>>) -> ()
-! CHECK:           return
-end subroutine test_char
-
-subroutine test_logical
-! CHECK-LABEL:   func.func @_QMtest_show_descriptorPtest_logical() {
-  implicit none
-  logical(kind=1) :: l1 = .false.
-  logical(kind=2) :: l2 = .true.
-  logical(kind=2), dimension(2), target :: la2 = (/ .true., .false. /)
-  logical(kind=2), dimension(:), pointer :: pla2
-! CHECK:           %[[C0:.*]] = arith.constant 0 : index
-! CHECK:           %[[C2:.*]] = arith.constant 2 : index
-! CHECK:           %[[DUMMY_SCOPE_0:.*]] = fir.dummy_scope : !fir.dscope
-! CHECK:           %[[ADDRESS_OF_0:.*]] = fir.address_of(@_QMtest_show_descriptorFtest_logicalEl1) : !fir.ref<!fir.logical<1>>
-! CHECK:           %[[DECLARE_0:.*]] = fir.declare %[[ADDRESS_OF_0]] {uniq_name = "_QMtest_show_descriptorFtest_logicalEl1"} : (!fir.ref<!fir.logical<1>>) -> !fir.ref<!fir.logical<1>>
-! CHECK:           %[[ADDRESS_OF_1:.*]] = fir.address_of(@_QMtest_show_descriptorFtest_logicalEl2) : !fir.ref<!fir.logical<2>>
-! CHECK:           %[[DECLARE_1:.*]] = fir.declare %[[ADDRESS_OF_1]] {uniq_name = "_QMtest_show_descriptorFtest_logicalEl2"} : (!fir.ref<!fir.logical<2>>) -> !fir.ref<!fir.logical<2>>
-! CHECK:           %[[ADDRESS_OF_2:.*]] = fir.address_of(@_QMtest_show_descriptorFtest_logicalEla2) : !fir.ref<!fir.array<2x!fir.logical<2>>>
-! CHECK:           %[[SHAPE_0:.*]] = fir.shape %[[C2]] : (index) -> !fir.shape<1>
-! CHECK:           %[[DECLARE_2:.*]] = fir.declare %[[ADDRESS_OF_2]](%[[SHAPE_0]]) {fortran_attrs = #fir.var_attrs<target>, uniq_name = "_QMtest_show_descriptorFtest_logicalEla2"} : (!fir.ref<!fir.array<2x!fir.logical<2>>>, !fir.shape<1>) -> !fir.ref<!fir.array<2x!fir.logical<2>>>
-! CHECK:           %[[ALLOCA_0:.*]] = fir.alloca !fir.box<!fir.ptr<!fir.array<?x!fir.logical<2>>>> {bindc_name = "pla2", uniq_name = "_QMtest_show_descriptorFtest_logicalEpla2"}
-! CHECK:           %[[ZERO_BITS_0:.*]] = fir.zero_bits !fir.ptr<!fir.array<?x!fir.logical<2>>>
-! CHECK:           %[[SHAPE_1:.*]] = fir.shape %[[C0]] : (index) -> !fir.shape<1>
-! CHECK:           %[[EMBOX_0:.*]] = fir.embox %[[ZERO_BITS_0]](%[[SHAPE_1]]) : (!fir.ptr<!fir.array<?x!fir.logical<2>>>, !fir.shape<1>) -> !fir.box<!fir.ptr<!fir.array<?x!fir.logical<2>>>>
-! CHECK:           fir.store %[[EMBOX_0]] to %[[ALLOCA_0]] : !fir.ref<!fir.box<!fir.ptr<!fir.array<?x!fir.logical<2>>>>>
-
-  call show_descriptor(l1)
-  call show_descriptor(l2)
-  pla2 => la2
-! CHECK:           %[[DECLARE_3:.*]] = fir.declare %[[ALLOCA_0]] {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QMtest_show_descriptorFtest_logicalEpla2"} : (!fir.ref<!fir.box<!fir.ptr<!fir.array<?x!fir.logical<2>>>>>) -> !fir.ref<!fir.box<!fir.ptr<!fir.array<?x!fir.logical<2>>>>>
-! CHECK:           %[[EMBOX_1:.*]] = fir.embox %[[DECLARE_0]] : (!fir.ref<!fir.logical<1>>) -> !fir.box<!fir.logical<1>>
-! CHECK:           fir.call @_FortranAShowDescriptor(%[[EMBOX_1]]) fastmath<contract> : (!fir.box<!fir.logical<1>>) -> ()
-! CHECK:           %[[EMBOX_2:.*]] = fir.embox %[[DECLARE_1]] : (!fir.ref<!fir.logical<2>>) -> !fir.box<!fir.logical<2>>
-! CHECK:           fir.call @_FortranAShowDescriptor(%[[EMBOX_2]]) fastmath<contract> : (!fir.box<!fir.logical<2>>) -> ()
-
-  call show_descriptor(la2)
-  call show_descriptor(pla2)
-! CHECK:           %[[CONVERT_0:.*]] = fir.convert %[[DECLARE_2]] : (!fir.ref<!fir.array<2x!fir.logical<2>>>) -> !fir.ref<!fir.array<?x!fir.logical<2>>>
-! CHECK:           %[[EMBOX_3:.*]] = fir.embox %[[CONVERT_0]](%[[SHAPE_0]]) : (!fir.ref<!fir.array<?x!fir.logical<2>>>, !fir.shape<1>) -> !fir.box<!fir.ptr<!fir.array<?x!fir.logical<2>>>>
-! CHECK:           fir.store %[[EMBOX_3]] to %[[DECLARE_3]] : !fir.ref<!fir.box<!fir.ptr<!fir.array<?x!fir.logical<2>>>>>
-! CHECK:           %[[EMBOX_4:.*]] = fir.embox %[[DECLARE_2]](%[[SHAPE_0]]) : (!fir.ref<!fir.array<2x!fir.logical<2>>>, !fir.shape<1>) -> !fir.box<!fir.array<2x!fir.logical<2>>>
-! CHECK:           fir.call @_FortranAShowDescriptor(%[[EMBOX_4]]) fastmath<contract> : (!fir.box<!fir.array<2x!fir.logical<2>>>) -> ()
-! CHECK:           %[[LOAD_0:.*]] = fir.load %[[DECLARE_3]] : !fir.ref<!fir.box<!fir.ptr<!fir.array<?x!fir.logical<2>>>>>
-! CHECK:           fir.call @_FortranAShowDescriptor(%[[LOAD_0]]) fastmath<contract> : (!fir.box<!fir.ptr<!fir.array<?x!fir.logical<2>>>>) -> ()
-! CHECK:           return
-end subroutine test_logical
-
-subroutine test_real
-! CHECK-LABEL:   func.func @_QMtest_show_descriptorPtest_real() {
-  implicit none
-  real :: half = 0.5
-  real :: row(3) = (/ 1 , 2, 3 /)
-  real(kind=8) :: w(4) = (/ .00011_8 , .00012_8, .00013_8, .00014_8 /)
-! CHECK:           %[[C2:.*]] = arith.constant 2 : index
-! CHECK:           %[[C1:.*]] = arith.constant 1 : index
-! CHECK:           %[[C4:.*]] = arith.constant 4 : index
-! CHECK:           %[[C3:.*]] = arith.constant 3 : index
-! CHECK:           %[[DUMMY_SCOPE_2:.*]] = fir.dummy_scope : !fir.dscope
-! CHECK:           %[[ADDRESS_OF_4:.*]] = fir.address_of(@_QMtest_show_descriptorFtest_realEhalf) : !fir.ref<f32>
-! CHECK:           %[[DECLARE_5:.*]] = fir.declare %[[ADDRESS_OF_4]] {uniq_name = "_QMtest_show_descriptorFtest_realEhalf"} : (!fir.ref<f32>) -> !fir.ref<f32>
-! CHECK:           %[[ADDRESS_OF_5:.*]] = fir.address_of(@_QMtest_show_descriptorFtest_realErow) : !fir.ref<!fir.array<3xf32>>
-! CHECK:           %[[SHAPE_2:.*]] = fir.shape %[[C3]] : (index) -> !fir.shape<1>
-! CHECK:           %[[DECLARE_6:.*]] = fir.declare %[[ADDRESS_OF_5]](%[[SHAPE_2]]) {uniq_name = "_QMtest_show_descriptorFtest_realErow"} : (!fir.ref<!fir.array<3xf32>>, !fir.shape<1>) -> !fir.ref<!fir.array<3xf32>>
-! CHECK:           %[[ADDRESS_OF_6:.*]] = fir.address_of(@_QMtest_show_descriptorFtest_realEw) : !fir.ref<!fir.array<4xf64>>
-! CHECK:           %[[SHAPE_3:.*]] = fir.shape %[[C4]] : (index) -> !fir.shape<1>
-! CHECK:           %[[DECLARE_7:.*]] = fir.declare %[[ADDRESS_OF_6]](%[[SHAPE_3]]) {uniq_name = "_QMtest_show_descriptorFtest_realEw"} : (!fir.ref<!fir.array<4xf64>>, !fir.shape<1>) -> !fir.ref<!fir.array<4xf64>>
-
-  call show_descriptor(half)
-  call show_descriptor(row)
-  call show_descriptor(w)
-  call show_descriptor(w(1:4:2))
-! CHECK:           %[[EMBOX_7:.*]] = fir.embox %[[DECLARE_5]] : (!fir.ref<f32>) -> !fir.box<f32>
-! CHECK:           fir.call @_FortranAShowDescriptor(%[[EMBOX_7]]) fastmath<contract> : (!fir.box<f32>) -> ()
-! CHECK:           %[[EMBOX_8:.*]] = fir.embox %[[DECLARE_6]](%[[SHAPE_2]]) : (!fir.ref<!fir.array<3xf32>>, !fir.shape<1>) -> !fir.box<!fir.array<3xf32>>
-! CHECK:           fir.call @_FortranAShowDescriptor(%[[EMBOX_8]]) fastmath<contract> : (!fir.box<!fir.array<3xf32>>) -> ()
-! CHECK:           %[[EMBOX_9:.*]] = fir.embox %[[DECLARE_7]](%[[SHAPE_3]]) : (!fir.ref<!fir.array<4xf64>>, !fir.shape<1>) -> !fir.box<!fir.array<4xf64>>
-! CHECK:           fir.call @_FortranAShowDescriptor(%[[EMBOX_9]]) fastmath<contract> : (!fir.box<!fir.array<4xf64>>) -> ()
-! CHECK:           %[[SHAPE_4:.*]] = fir.shape %[[C2]] : (index) -> !fir.shape<1>
-! CHECK:           %[[UNDEFINED_0:.*]] = fir.undefined index
-! CHECK:           %[[SLICE_0:.*]] = fir.slice %[[C1]], %[[C4]], %[[C2]] : (index, index, index) -> !fir.slice<1>
-! CHECK:           %[[EMBOX_10:.*]] = fir.embox %[[DECLARE_7]](%[[SHAPE_3]]) {{\[}}%[[SLICE_0]]] : (!fir.ref<!fir.array<4xf64>>, !fir.shape<1>, !fir.slice<1>) -> !fir.box<!fir.array<2xf64>>
-! CHECK:           fir.call @_FortranAShowDescriptor(%[[EMBOX_10]]) fastmath<contract> : (!fir.box<!fir.array<2xf64>>) -> ()
-! CHECK:           return
-end subroutine test_real
-
-subroutine test_complex
-! CHECK-LABEL:   func.func @_QMtest_show_descriptorPtest_complex() {
-  implicit none
-  complex, parameter :: hr = 0.5
-  complex, parameter :: hi = (0, 0.5)
-  complex :: c1 = hr
-  complex :: c2 = hi
-  complex :: a2(2) = (/ hr, hi /)
-! CHECK:           %[[CST_0:.*]] = arith.constant 0.000000e+00 : f32
-! CHECK:           %[[CST_1:.*]] = arith.constant 5.000000e-01 : f32
-! CHECK:           %[[C2:.*]] = arith.constant 2 : index
-! CHECK:           %[[ALLOCA_1:.*]] = fir.alloca complex<f32>
-! CHECK:           %[[ALLOCA_2:.*]] = fir.alloca complex<f32>
-! CHECK:           %[[DUMMY_SCOPE_3:.*]] = fir.dummy_scope : !fir.dscope
-! CHECK:           %[[ADDRESS_OF_7:.*]] = fir.address_of(@_QMtest_show_descriptorFtest_complexEa2) : !fir.ref<!fir.array<2xcomplex<f32>>>
-! CHECK:           %[[SHAPE_5:.*]] = fir.shape %[[C2]] : (index) -> !fir.shape<1>
-! CHECK:           %[[DECLARE_8:.*]] = fir.declare %[[ADDRESS_OF_7]](%[[SHAPE_5]]) {uniq_name = "_QMtest_show_descriptorFtest_complexEa2"} : (!fir.ref<!fir.array<2xcomplex<f32>>>, !fir.shape<1>) -> !fir.ref<!fir.array<2xcomplex<f32>>>
-! CHECK:           %[[ADDRESS_OF_8:.*]] = fir.address_of(@_QMtest_show_descriptorFtest_complexEc1) : !fir.ref<complex<f32>>
-! CHECK:           %[[DECLARE_9:.*]] = fir.declare %[[ADDRESS_OF_8]] {uniq_name = "_QMtest_show_descriptorFtest_complexEc1"} : (!fir.ref<complex<f32>>) -> !fir.ref<complex<f32>>
-! CHECK:           %[[ADDRESS_OF_9:.*]] = fir.address_of(@_QMtest_show_descriptorFtest_complexEc2) : !fir.ref<complex<f32>>
-! CHECK:           %[[DECLARE_10:.*]] = fir.declare %[[ADDRESS_OF_9]] {uniq_name = "_QMtest_show_descriptorFtest_complexEc2"} : (!fir.ref<complex<f32>>) -> !fir.ref<complex<f32>>
-! CHECK:           %[[ADDRESS_OF_10:.*]] = fir.address_of(@_QMtest_show_descriptorFtest_complexEChi) : !fir.ref<complex<f32>>
-! CHECK:           %[[DECLARE_11:.*]] = fir.declare %[[ADDRESS_OF_10]] {fortran_attrs = #fir.var_attrs<parameter>, uniq_name = "_QMtest_show_descriptorFtest_complexEChi"} : (!fir.ref<complex<f32>>) -> !fir.ref<complex<f32>>
-! CHECK:           %[[ADDRESS_OF_11:.*]] = fir.address_of(@_QMtest_show_descriptorFtest_complexEChr) : !fir.ref<complex<f32>>
-! CHECK:           %[[DECLARE_12:.*]] = fir.declare %[[ADDRESS_OF_11]] {fortran_attrs = #fir.var_attrs<parameter>, uniq_name = "_QMtest_show_descriptorFtest_complexEChr"} : (!fir.ref<complex<f32>>) -> !fir.ref<complex<f32>>
-! CHECK:           %[[UNDEFINED_1:.*]] = fir.undefined complex<f32>
-! CHECK:           %[[INSERT_VALUE_0:.*]] = fir.insert_value %[[UNDEFINED_1]], %[[CST_1]], [0 : index] : (complex<f32>, f32) -> complex<f32>
-! CHECK:           %[[INSERT_VALUE_1:.*]] = fir.insert_value %[[INSERT_VALUE_0]], %[[CST_0]], [1 : index] : (complex<f32>, f32) -> complex<f32>
-! CHECK:           fir.store %[[INSERT_VALUE_1]] to %[[ALLOCA_2]] : !fir.ref<complex<f32>>
-
-  call show_descriptor(hr)
-! CHECK:           %[[EMBOX_11:.*]] = fir.embox %[[ALLOCA_2]] : (!fir.ref<complex<f32>>) -> !fir.box<complex<f32>>
-! CHECK:           fir.call @_FortranAShowDescriptor(%[[EMBOX_11]]) fastmath<contract> : (!fir.box<complex<f32>>) -> ()
-
-  call show_descriptor(hi)
-! CHECK:           %[[INSERT_VALUE_2:.*]] = fir.insert_value %[[UNDEFINED_1]], %[[CST_0]], [0 : index] : (complex<f32>, f32) -> complex<f32>
-! CHECK:           %[[INSERT_VALUE_3:.*]] = fir.insert_value %[[INSERT_VALUE_2]], %[[CST_1]], [1 : index] : (complex<f32>, f32) -> complex<f32>
-! CHECK:           fir.store %[[INSERT_VALUE_3]] to %[[ALLOCA_1]] : !fir.ref<complex<f32>>
-! CHECK:           %[[EMBOX_12:.*]] = fir.embox %[[ALLOCA_1]] : (!fir.ref<complex<f32>>) -> !fir.box<complex<f32>>
-! CHECK:           fir.call @_FortranAShowDescriptor(%[[EMBOX_12]]) fastmath<contract> : (!fir.box<complex<f32>>) -> ()
-
-  call show_descriptor(a2)
-! CHECK:           %[[EMBOX_13:.*]] = fir.embox %[[DECLARE_8]](%[[SHAPE_5]]) : (!fir.ref<!fir.array<2xcomplex<f32>>>, !fir.shape<1>) -> !fir.box<!fir.array<2xcomplex<f32>>>
-! CHECK:           fir.call @_FortranAShowDescriptor(%[[EMBOX_13]]) fastmath<contract> : (!fir.box<!fir.array<2xcomplex<f32>>>) -> ()
-! CHECK:           return
-end subroutine test_complex
-
-subroutine test_derived
-! CHECK-LABEL:   func.func @_QMtest_show_descriptorPtest_derived() {
-  implicit none
-  type :: t1
-     integer :: a
-     integer :: b
-  end type t1
-  type, extends (t1) :: t2
-     integer :: c
-  end type t2
-  type(t2) :: vt2 = t2(7,5,3)
-! CHECK:           %[[C0:.*]] = arith.constant 0 : index
-! CHECK:           %[[C2:.*]] = arith.constant 2 : index
-! CHECK:           %[[C1:.*]] = arith.constant 1 : index
-! CHECK:           %[[DUMMY_SCOPE_4:.*]] = fir.dummy_scope : !fir.dscope
-! CHECK:           %[[ADDRESS_OF_12:.*]] = fir.address_of(@_QMtest_show_descriptorFtest_derivedE.n.a) : !fir.ref<!fir.char<1>>
-! CHECK:           %[[DECLARE_13:.*]] = fir.declare %[[ADDRESS_OF_12]] typeparams %[[C1]] {fortran_attrs = #fir.var_attrs<target>, uniq_name = "_QMtest_show_descriptorFtest_derivedE.n.a"} : (!fir.ref<!fir.char<1>>, index) -> !fir.ref<!fir.char<1>>
-! CHECK:           %[[ADDRESS_OF_13:.*]] = fir.address_of(@_QMtest_show_descriptorFtest_derivedE.n.b) : !fir.ref<!fir.char<1>>
-! CHECK:           %[[DECLARE_14:.*]] = fir.declare %[[ADDRESS_OF_13]] typeparams %[[C1]] {fortran_attrs = #fir.var_attrs<target>, uniq_name = "_QMtest_show_descriptorFtest_derivedE.n.b"} : (!fir.ref<!fir.char<1>>, index) -> !fir.ref<!fir.char<1>>
-! CHECK:           %[[ADDRESS_OF_14:.*]] = fir.address_of(@_QMtest_show_descriptorFtest_derivedE.n.t1) : !fir.ref<!fir.char<1,2>>
-! CHECK:           %[[DECLARE_15:.*]] = fir.declare %[[ADDRESS_OF_14]] typeparams %[[C2]] {fortran_attrs = #fir.var_attrs<target>, uniq_name = "_QMtest_show_descriptorFtest_derivedE.n.t1"} : (!fir.ref<!fir.char<1,2>>, index) -> !fir.ref<!fir.char<1,2>>
-! CHECK:           %[[ADDRESS_OF_15:.*]] = fir.address_of(@_QMtest_show_descriptorFtest_derivedE.n.c) : !fir.ref<!fir.char<1>>
-! CHECK:           %[[DECLARE_16:.*]] = fir.declare %[[ADDRESS_OF_15]] typeparams %[[C1]] {fortran_attrs = #fir.var_attrs<target>, uniq_name = "_QMtest_show_descriptorFtest_derivedE.n.c"} : (!fir.ref<!fir.char<1>>, index) -> !fir.ref<!fir.char<1>>
-! CHECK:           %[[ADDRESS_OF_16:.*]] = fir.address_of(@_QMtest_show_descriptorFtest_derivedE.n.t2) : !fir.ref<!fir.char<1,2>>
-! CHECK:           %[[DECLARE_17:.*]] = fir.declare %[[ADDRESS_OF_16]] typeparams %[[C2]] {fortran_attrs = #fir.var_attrs<target>, uniq_name = "_QMtest_show_descriptorFtest_derivedE.n.t2"} : (!fir.ref<!fir.char<1,2>>, index) -> !fir.ref<!fir.char<1,2>>
-! CHECK:           %[[ADDRESS_OF_17:.*]] = fir.address_of(@_QMtest_show_descriptorFtest_derivedEvt2) : !fir.ref<!fir.type<_QMtest_show_descriptorFtest_derivedTt2{t1:!fir.type<_QMtest_show_descriptorFtest_derivedTt1{a:i32,b:i32}>,c:i32}>>
-! CHECK:           %[[DECLARE_18:.*]] = fir.declare %[[ADDRESS_OF_17]] {uniq_name = "_QMtest_show_descriptorFtest_derivedEvt2"} : (!fir.ref<!fir.type<_QMtest_show_descriptorFtest_derivedTt2{t1:!fir.type<_QMtest_show_descriptorFtest_derivedTt1{a:i32,b:i32}>,c:i32}>>) -> !fir.ref<!fir.type<_QMtest_show_descriptorFtest_derivedTt2{t1:!fir.type<_QMtest_show_descriptorFtest_derivedTt1{a:i32,b:i32}>,c:i32}>>
-
-  call show_descriptor(vt2)
-! CHECK:           %[[EMBOX_16:.*]] = fir.embox %[[DECLARE_18]] : (!fir.ref<!fir.type<_QMtest_show_descriptorFtest_derivedTt2{t1:!fir.type<_QMtest_show_descriptorFtest_derivedTt1{a:i32,b:i32}>,c:i32}>>) -> !fir.box<!fir.type<_QMtest_show_descriptorFtest_derivedTt2{t1:!fir.type<_QMtest_show_descriptorFtest_derivedTt1{a:i32,b:i32}>,c:i32}>>
-! CHECK:           fir.call @_FortranAShowDescriptor(%[[EMBOX_16]]) fastmath<contract> : (!fir.box<!fir.type<_QMtest_show_descriptorFtest_derivedTt2{t1:!fir.type<_QMtest_show_descriptorFtest_derivedTt1{a:i32,b:i32}>,c:i32}>>) -> ()
-! CHECK:           return
-end subroutine test_derived
-end module test_show_descriptor
diff --git a/flang/tools/f18/CMakeLists.txt b/flang/tools/f18/CMakeLists.txt
index 3baffbd0d310d..a2b4d73f48384 100644
--- a/flang/tools/f18/CMakeLists.txt
+++ b/flang/tools/f18/CMakeLists.txt
@@ -31,7 +31,6 @@ set(MODULES_WITHOUT_IMPLEMENTATION
   "iso_fortran_env"
   "f90deviceio"
   "iso_fortran_env_impl"
-  "flang_debug"
 )
 
 set(MODULES ${MODULES_WITH_IMPLEMENTATION} ${MODULES_WITHOUT_IMPLEMENTATION})
diff --git a/libclc/opencl/lib/generic/atomic/atomic_def.inc b/libclc/opencl/lib/generic/atomic/atomic_def.inc
index a4ccab5990888..e6b7c831e10d3 100644
--- a/libclc/opencl/lib/generic/atomic/atomic_def.inc
+++ b/libclc/opencl/lib/generic/atomic/atomic_def.inc
@@ -12,7 +12,8 @@
                                  defined(cl_khr_int64_extended_atomics))
 #define __CLC_HAVE_64_ATOMIC
 #endif
-#if defined(__CLC_FPSIZE) && (__CLC_FPSIZE < 64 || defined(__CLC_HAVE_64_ATOMIC)
+#if defined(__CLC_FPSIZE) &&                                                   \
+    (__CLC_FPSIZE < 64 || defined(__CLC_HAVE_64_ATOMIC))
 #define __CLC_HAVE_FP_ATOMIC
 #endif
 #if defined(__CLC_GENSIZE) &&                                                  \
diff --git a/libcxx/include/__type_traits/common_reference.h b/libcxx/include/__type_traits/common_reference.h
index 297b6adbe1ca0..59badb64267de 100644
--- a/libcxx/include/__type_traits/common_reference.h
+++ b/libcxx/include/__type_traits/common_reference.h
@@ -18,37 +18,16 @@
 #include <__type_traits/is_reference.h>
 #include <__type_traits/remove_cvref.h>
 #include <__type_traits/remove_reference.h>
-#include <__type_traits/type_identity.h>
 #include <__utility/declval.h>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
 #  pragma GCC system_header
 #endif
 
-#if _LIBCPP_STD_VER >= 20
-
 _LIBCPP_BEGIN_NAMESPACE_STD
 
-template <class...>
-struct _LIBCPP_NO_SPECIALIZATIONS common_reference;
-
-template <class... _Types>
-using common_reference_t = typename common_reference<_Types...>::type;
-
-template <class, class, template <class> class, template <class> class>
-struct basic_common_reference {};
-
-#  if __has_builtin(__builtin_common_reference)
-
-template <class _Tp, class _Up, template <class> class _Tx, template <class> class _Ux>
-using __basic_common_reference_t = basic_common_reference<_Tp, _Up, _Tx, _Ux>::type;
-
-template <class... _Args>
-struct _LIBCPP_NO_SPECIALIZATIONS common_reference
-    : __builtin_common_reference<__basic_common_reference_t, common_type_t, type_identity, __empty, _Args...> {};
-
-#  else
-
+// common_reference
+#if _LIBCPP_STD_VER >= 20
 // Let COND_RES(X, Y) be:
 template <class _Xp, class _Yp>
 using __cond_res _LIBCPP_NODEBUG = decltype(false ? std::declval<_Xp (&)()>()() : std::declval<_Yp (&)()>()());
@@ -130,10 +109,19 @@ struct __common_ref {};
 
 // Note C: For the common_reference trait applied to a parameter pack [...]
 
+template <class...>
+struct _LIBCPP_NO_SPECIALIZATIONS common_reference;
+
+template <class... _Types>
+using common_reference_t = typename common_reference<_Types...>::type;
+
+template <class, class, template <class> class, template <class> class>
+struct basic_common_reference {};
+
 _LIBCPP_DIAGNOSTIC_PUSH
-#    if __has_warning("-Winvalid-specialization")
+#  if __has_warning("-Winvalid-specialization")
 _LIBCPP_CLANG_DIAGNOSTIC_IGNORED("-Winvalid-specialization")
-#    endif
+#  endif
 // bullet 1 - sizeof...(T) == 0
 template <>
 struct common_reference<> {};
@@ -207,10 +195,8 @@ _LIBCPP_DIAGNOSTIC_POP
 template <class...>
 struct _LIBCPP_NO_SPECIALIZATIONS common_reference {};
 
-#  endif // __has_builtin(__builtin_common_reference)
+#endif // _LIBCPP_STD_VER >= 20
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP_STD_VER >= 20
-
 #endif // _LIBCPP___TYPE_TRAITS_COMMON_REFERENCE_H
diff --git a/libcxx/include/ext/hash_map b/libcxx/include/ext/hash_map
index 01ca7498f0cc1..09c981131ff96 100644
--- a/libcxx/include/ext/hash_map
+++ b/libcxx/include/ext/hash_map
@@ -570,10 +570,7 @@ hash_map<_Key, _Tp, _Hash, _Pred, _Alloc>::hash_map(
 }
 
 template <class _Key, class _Tp, class _Hash, class _Pred, class _Alloc>
-hash_map<_Key, _Tp, _Hash, _Pred, _Alloc>::hash_map(const hash_map& __u) : __table_(__u.__table_) {
-  __table_.__rehash_unique(__u.bucket_count());
-  insert(__u.begin(), __u.end());
-}
+hash_map<_Key, _Tp, _Hash, _Pred, _Alloc>::hash_map(const hash_map& __u) : __table_(__u.__table_) {}
 
 template <class _Key, class _Tp, class _Hash, class _Pred, class _Alloc>
 typename hash_map<_Key, _Tp, _Hash, _Pred, _Alloc>::__node_holder
diff --git a/libcxx/include/ext/hash_set b/libcxx/include/ext/hash_set
index 2796774fee24a..56aa4d8a47eeb 100644
--- a/libcxx/include/ext/hash_set
+++ b/libcxx/include/ext/hash_set
@@ -356,10 +356,7 @@ hash_set<_Value, _Hash, _Pred, _Alloc>::hash_set(
 }
 
 template <class _Value, class _Hash, class _Pred, class _Alloc>
-hash_set<_Value, _Hash, _Pred, _Alloc>::hash_set(const hash_set& __u) : __table_(__u.__table_) {
-  __table_.__rehash_unique(__u.bucket_count());
-  insert(__u.begin(), __u.end());
-}
+hash_set<_Value, _Hash, _Pred, _Alloc>::hash_set(const hash_set& __u) : __table_(__u.__table_) {}
 
 template <class _Value, class _Hash, class _Pred, class _Alloc>
 template <class _InputIterator>
diff --git a/libcxx/include/module.modulemap.in b/libcxx/include/module.modulemap.in
index 5d2a92bf7905c..0ac5a1ade817f 100644
--- a/libcxx/include/module.modulemap.in
+++ b/libcxx/include/module.modulemap.in
@@ -1517,7 +1517,6 @@ module std [system] {
       header "__iterator/iterator_traits.h"
       export std_core.type_traits.integral_constant
       export std_core.type_traits.is_convertible
-      export std_core.type_traits.nat
     }
     module iterator_with_data         { header "__iterator/iterator_with_data.h" }
     module iterator                   { header "__iterator/iterator.h" }
diff --git a/libcxx/test/extensions/gnu/hash_map/copy.pass.cpp b/libcxx/test/extensions/gnu/hash_map/copy.pass.cpp
new file mode 100644
index 0000000000000..65b8debda0676
--- /dev/null
+++ b/libcxx/test/extensions/gnu/hash_map/copy.pass.cpp
@@ -0,0 +1,27 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// ADDITIONAL_COMPILE_FLAGS: -Wno-deprecated
+
+// hash_map::hash_map(const hash_map&)
+
+#include <cassert>
+#include <ext/hash_map>
+
+int main(int, char**) {
+  __gnu_cxx::hash_map<int, int> map;
+
+  map.insert(std::make_pair(1, 1));
+  map.insert(std::make_pair(2, 1));
+
+  auto map2 = map;
+
+  assert(map2.size() == 2);
+
+  return 0;
+}
diff --git a/libcxx/test/extensions/gnu/hash_set/copy.pass.cpp b/libcxx/test/extensions/gnu/hash_set/copy.pass.cpp
new file mode 100644
index 0000000000000..95a3579194923
--- /dev/null
+++ b/libcxx/test/extensions/gnu/hash_set/copy.pass.cpp
@@ -0,0 +1,27 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// ADDITIONAL_COMPILE_FLAGS: -Wno-deprecated
+
+// hash_set::hash_set(const hash_set&)
+
+#include <cassert>
+#include <ext/hash_set>
+
+int main(int, char**) {
+  __gnu_cxx::hash_set<int> set;
+
+  set.insert(1);
+  set.insert(2);
+
+  auto set2 = set;
+
+  assert(set2.size() == 2);
+
+  return 0;
+}
diff --git a/lld/ELF/Options.td b/lld/ELF/Options.td
index fb9e5df648c75..eb299099a9b71 100644
--- a/lld/ELF/Options.td
+++ b/lld/ELF/Options.td
@@ -154,7 +154,7 @@ def bp_startup_sort: JJ<"bp-startup-sort=">, MetaVarName<"[none,function]">,
 
 // Auxiliary options related to balanced partition
 defm bp_compression_sort_startup_functions: BB<"bp-compression-sort-startup-functions",
-  "When --irpgo-profile is pecified, prioritize function similarity for compression in addition to startup time", "">;
+  "When --irpgo-profile is specified, prioritize function similarity for compression in addition to startup time", "">;
 def verbose_bp_section_orderer: FF<"verbose-bp-section-orderer">,
   HelpText<"Print information on balanced partitioning">;
 
diff --git a/lldb/bindings/python/python-wrapper.swig b/lldb/bindings/python/python-wrapper.swig
index ef501fbafc947..0ba152166522b 100644
--- a/lldb/bindings/python/python-wrapper.swig
+++ b/lldb/bindings/python/python-wrapper.swig
@@ -425,6 +425,18 @@ void *lldb_private::python::LLDBSWIGPython_CastPyObjectToSBBreakpoint(PyObject *
   return sb_ptr;
 }
 
+void *lldb_private::python::LLDBSWIGPython_CastPyObjectToSBThread(PyObject * data) {
+  lldb::SBThread *sb_ptr = nullptr;
+
+  int valid_cast =
+      SWIG_ConvertPtr(data, (void **)&sb_ptr, SWIGTYPE_p_lldb__SBThread, 0);
+
+  if (valid_cast == -1)
+    return NULL;
+
+  return sb_ptr;
+}
+
 void *lldb_private::python::LLDBSWIGPython_CastPyObjectToSBFrame(PyObject * data) {
   lldb::SBFrame *sb_ptr = nullptr;
 
diff --git a/lldb/docs/resources/lldbgdbremote.md b/lldb/docs/resources/lldbgdbremote.md
index 032edb66690e4..fdd9b057f0b4a 100644
--- a/lldb/docs/resources/lldbgdbremote.md
+++ b/lldb/docs/resources/lldbgdbremote.md
@@ -2508,7 +2508,7 @@ stack traces.
 
 Get the value of a Wasm global variable for the given frame index at the given
 variable index. The indexes are encoded as base 10. The result is a hex-encoded
-address from where to read the value.
+little-endian value of the global.
 
 ```
 send packet: $qWasmGlobal:0;2#cb
@@ -2523,7 +2523,7 @@ variables.
 
 Get the value of a Wasm function argument or local variable for the given frame
 index at the given variable index. The indexes are encoded as base 10. The
-result is a hex-encoded address from where to read the value.
+result is a hex-encoded little-endian value of the local.
 
 
 ```
@@ -2539,7 +2539,7 @@ variables.
 
 Get the value of a Wasm local variable from the Wasm operand stack, for the
 given frame index at the given variable index. The indexes are encoded as base
-10. The result is a hex-encoded address from where to read value.
+10. The result is a hex-encoded little-endian value from the stack at the given index.
 
 ```
 send packet: $qWasmStackValue:0;2#cb
diff --git a/lldb/examples/python/templates/scripted_frame_provider.py b/lldb/examples/python/templates/scripted_frame_provider.py
index 20f4d76d188c2..7a72f1a24c9da 100644
--- a/lldb/examples/python/templates/scripted_frame_provider.py
+++ b/lldb/examples/python/templates/scripted_frame_provider.py
@@ -31,7 +31,54 @@ class ScriptedFrameProvider(metaclass=ABCMeta):
         )
     """
 
+    @staticmethod
+    def applies_to_thread(thread):
+        """Determine if this frame provider should be used for a given thread.
+
+        This static method is called before creating an instance of the frame
+        provider to determine if it should be applied to a specific thread.
+        Override this method to provide custom filtering logic.
+
+        Args:
+            thread (lldb.SBThread): The thread to check.
+
+        Returns:
+            bool: True if this frame provider should be used for the thread,
+                False otherwise. The default implementation returns True for
+                all threads.
+
+        Example:
+
+        .. code-block:: python
+
+            @staticmethod
+            def applies_to_thread(thread):
+                # Only apply to thread 1
+                return thread.GetIndexID() == 1
+        """
+        return True
+
+    @staticmethod
     @abstractmethod
+    def get_description():
+        """Get a description of this frame provider.
+
+        This method should return a human-readable string describing what
+        this frame provider does. The description is used for debugging
+        and display purposes.
+
+        Returns:
+            str: A description of the frame provider.
+
+        Example:
+
+        .. code-block:: python
+
+            def get_description(self):
+                return "Crash log frame provider for thread 1"
+        """
+        pass
+
     def __init__(self, input_frames, args):
         """Construct a scripted frame provider.
 
diff --git a/lldb/examples/python/templates/scripted_process.py b/lldb/examples/python/templates/scripted_process.py
index b4232f632a30a..24aa9818bb989 100644
--- a/lldb/examples/python/templates/scripted_process.py
+++ b/lldb/examples/python/templates/scripted_process.py
@@ -243,6 +243,7 @@ def __init__(self, process, args):
                 key/value pairs used by the scripted thread.
         """
         self.target = None
+        self.arch = None
         self.originating_process = None
         self.process = None
         self.args = None
@@ -264,6 +265,9 @@ def __init__(self, process, args):
             and process.IsValid()
         ):
             self.target = process.target
+            triple = self.target.triple
+            if triple:
+                self.arch = triple.split("-")[0]
             self.originating_process = process
             self.process = self.target.GetProcess()
             self.get_register_info()
@@ -350,17 +354,14 @@ def get_stackframes(self):
     def get_register_info(self):
         if self.register_info is None:
             self.register_info = dict()
-            if "x86_64" in self.originating_process.arch:
+            if "x86_64" in self.arch:
                 self.register_info["sets"] = ["General Purpose Registers"]
                 self.register_info["registers"] = INTEL64_GPR
-            elif (
-                "arm64" in self.originating_process.arch
-                or self.originating_process.arch == "aarch64"
-            ):
+            elif "arm64" in self.arch or self.arch == "aarch64":
                 self.register_info["sets"] = ["General Purpose Registers"]
                 self.register_info["registers"] = ARM64_GPR
             else:
-                raise ValueError("Unknown architecture", self.originating_process.arch)
+                raise ValueError("Unknown architecture", self.arch)
         return self.register_info
 
     @abstractmethod
@@ -403,11 +404,12 @@ def __init__(self, thread, args):
         """Construct a scripted frame.
 
         Args:
-            thread (ScriptedThread): The thread owning this frame.
+            thread (ScriptedThread/lldb.SBThread): The thread owning this frame.
             args (lldb.SBStructuredData): A Dictionary holding arbitrary
                 key/value pairs used by the scripted frame.
         """
         self.target = None
+        self.arch = None
         self.originating_thread = None
         self.thread = None
         self.args = None
@@ -417,15 +419,17 @@ def __init__(self, thread, args):
         self.register_ctx = {}
         self.variables = []
 
-        if (
-            isinstance(thread, ScriptedThread)
-            or isinstance(thread, lldb.SBThread)
-            and thread.IsValid()
+        if isinstance(thread, ScriptedThread) or (
+            isinstance(thread, lldb.SBThread) and thread.IsValid()
         ):
-            self.target = thread.target
             self.process = thread.process
+            self.target = self.process.target
+            triple = self.target.triple
+            if triple:
+                self.arch = triple.split("-")[0]
+            tid = thread.tid if isinstance(thread, ScriptedThread) else thread.id
             self.originating_thread = thread
-            self.thread = self.process.GetThreadByIndexID(thread.tid)
+            self.thread = self.process.GetThreadByIndexID(tid)
             self.get_register_info()
 
     @abstractmethod
@@ -506,7 +510,18 @@ def get_variables(self, filters):
 
     def get_register_info(self):
         if self.register_info is None:
-            self.register_info = self.originating_thread.get_register_info()
+            if isinstance(self.originating_thread, ScriptedThread):
+                self.register_info = self.originating_thread.get_register_info()
+            elif isinstance(self.originating_thread, lldb.SBThread):
+                self.register_info = dict()
+                if "x86_64" in self.arch:
+                    self.register_info["sets"] = ["General Purpose Registers"]
+                    self.register_info["registers"] = INTEL64_GPR
+                elif "arm64" in self.arch or self.arch == "aarch64":
+                    self.register_info["sets"] = ["General Purpose Registers"]
+                    self.register_info["registers"] = ARM64_GPR
+                else:
+                    raise ValueError("Unknown architecture", self.arch)
         return self.register_info
 
     @abstractmethod
@@ -640,12 +655,12 @@ def get_stop_reason(self):
 
             # TODO: Passthrough stop reason from driving process
             if self.driving_thread.GetStopReason() != lldb.eStopReasonNone:
-                if "arm64" in self.originating_process.arch:
+                if "arm64" in self.arch:
                     stop_reason["type"] = lldb.eStopReasonException
                     stop_reason["data"]["desc"] = (
                         self.driving_thread.GetStopDescription(100)
                     )
-                elif self.originating_process.arch == "x86_64":
+                elif self.arch == "x86_64":
                     stop_reason["type"] = lldb.eStopReasonSignal
                     stop_reason["data"]["signal"] = signal.SIGTRAP
                 else:
diff --git a/lldb/include/lldb/API/SBTarget.h b/lldb/include/lldb/API/SBTarget.h
index ce81ae46a0905..0318492f1054c 100644
--- a/lldb/include/lldb/API/SBTarget.h
+++ b/lldb/include/lldb/API/SBTarget.h
@@ -19,6 +19,7 @@
 #include "lldb/API/SBLaunchInfo.h"
 #include "lldb/API/SBStatisticsOptions.h"
 #include "lldb/API/SBSymbolContextList.h"
+#include "lldb/API/SBThreadCollection.h"
 #include "lldb/API/SBType.h"
 #include "lldb/API/SBValue.h"
 #include "lldb/API/SBWatchpoint.h"
@@ -1003,6 +1004,35 @@ class LLDB_API SBTarget {
 
   lldb::SBMutex GetAPIMutex() const;
 
+  /// Register a scripted frame provider for this target.
+  /// If a scripted frame provider with the same name and same argument
+  /// dictionary is already registered on this target, it will be overwritten.
+  ///
+  /// \param[in] class_name
+  ///     The name of the Python class that implements the frame provider.
+  ///
+  /// \param[in] args_dict
+  ///     A dictionary of arguments to pass to the frame provider class.
+  ///
+  /// \param[out] error
+  ///     An error object indicating success or failure.
+  ///
+  /// \return
+  ///     A unique identifier for the frame provider descriptor that was
+  ///     registered. 0 if the registration failed.
+  uint32_t RegisterScriptedFrameProvider(const char *class_name,
+                                         lldb::SBStructuredData args_dict,
+                                         lldb::SBError &error);
+
+  /// Remove a scripted frame provider from this target by name.
+  ///
+  /// \param[in] provider_id
+  ///     The id of the frame provider class to remove.
+  ///
+  /// \return
+  ///     An error object indicating success or failure.
+  lldb::SBError RemoveScriptedFrameProvider(uint32_t provider_id);
+
 protected:
   friend class SBAddress;
   friend class SBAddressRange;
diff --git a/lldb/include/lldb/API/SBThread.h b/lldb/include/lldb/API/SBThread.h
index f6a6d19935b83..639e7a0a1a5c0 100644
--- a/lldb/include/lldb/API/SBThread.h
+++ b/lldb/include/lldb/API/SBThread.h
@@ -256,6 +256,7 @@ class LLDB_API SBThread {
   friend class SBThreadPlan;
   friend class SBTrace;
 
+  friend class lldb_private::ScriptInterpreter;
   friend class lldb_private::python::SWIGBridge;
 
   SBThread(const lldb::ThreadSP &lldb_object_sp);
diff --git a/lldb/include/lldb/API/SBThreadCollection.h b/lldb/include/lldb/API/SBThreadCollection.h
index 5a052e6246026..d13dea0f11cd2 100644
--- a/lldb/include/lldb/API/SBThreadCollection.h
+++ b/lldb/include/lldb/API/SBThreadCollection.h
@@ -46,6 +46,7 @@ class LLDB_API SBThreadCollection {
   void SetOpaque(const lldb::ThreadCollectionSP &threads);
 
 private:
+  friend class SBTarget;
   friend class SBProcess;
   friend class SBThread;
   friend class SBSaveCoreOptions;
diff --git a/lldb/include/lldb/API/SBTrace.h b/lldb/include/lldb/API/SBTrace.h
index ce95595a423c9..d5368b234dd34 100644
--- a/lldb/include/lldb/API/SBTrace.h
+++ b/lldb/include/lldb/API/SBTrace.h
@@ -39,7 +39,7 @@ class LLDB_API SBTrace {
   SBTraceCursor CreateNewCursor(SBError &error, SBThread &thread);
 
   /// Save the trace to the specified directory, which will be created if
-  /// needed. This will also create a file \a <directory>/trace.json with the
+  /// needed. This will also create a file <directory>/trace.json with the
   /// main properties of the trace session, along with others files which
   /// contain the actual trace data. The trace.json file can be used later as
   /// input for the "trace load" command to load the trace in LLDB, or for the
diff --git a/lldb/include/lldb/Core/FormatEntity.h b/lldb/include/lldb/Core/FormatEntity.h
index 40916dc48a70b..107c30a000979 100644
--- a/lldb/include/lldb/Core/FormatEntity.h
+++ b/lldb/include/lldb/Core/FormatEntity.h
@@ -81,6 +81,7 @@ struct Entry {
     FrameRegisterByName,
     FrameIsArtificial,
     FrameKind,
+    FrameBorrowedInfo,
     ScriptFrame,
     FunctionID,
     FunctionDidChange,
diff --git a/lldb/include/lldb/Interpreter/Interfaces/ScriptedFrameProviderInterface.h b/lldb/include/lldb/Interpreter/Interfaces/ScriptedFrameProviderInterface.h
index 2d9f713676f90..49b60131399d5 100644
--- a/lldb/include/lldb/Interpreter/Interfaces/ScriptedFrameProviderInterface.h
+++ b/lldb/include/lldb/Interpreter/Interfaces/ScriptedFrameProviderInterface.h
@@ -16,11 +16,29 @@
 namespace lldb_private {
 class ScriptedFrameProviderInterface : public ScriptedInterface {
 public:
+  virtual bool AppliesToThread(llvm::StringRef class_name,
+                               lldb::ThreadSP thread_sp) {
+    return true;
+  }
+
   virtual llvm::Expected<StructuredData::GenericSP>
   CreatePluginObject(llvm::StringRef class_name,
                      lldb::StackFrameListSP input_frames,
                      StructuredData::DictionarySP args_sp) = 0;
 
+  /// Get a description string for the frame provider.
+  ///
+  /// This is called by the descriptor to fetch a description from the
+  /// scripted implementation. Implementations should call a static method
+  /// on the scripting class to retrieve the description.
+  ///
+  /// \param class_name The name of the scripting class implementing the
+  /// provider.
+  ///
+  /// \return A string describing what this frame provider does, or an
+  ///         empty string if no description is available.
+  virtual std::string GetDescription(llvm::StringRef class_name) { return {}; }
+
   virtual StructuredData::ObjectSP GetFrameAtIndex(uint32_t index) {
     return {};
   }
diff --git a/lldb/include/lldb/Interpreter/ScriptInterpreter.h b/lldb/include/lldb/Interpreter/ScriptInterpreter.h
index 7fed4940b85bf..0b91d6756552d 100644
--- a/lldb/include/lldb/Interpreter/ScriptInterpreter.h
+++ b/lldb/include/lldb/Interpreter/ScriptInterpreter.h
@@ -21,6 +21,7 @@
 #include "lldb/API/SBMemoryRegionInfo.h"
 #include "lldb/API/SBStream.h"
 #include "lldb/API/SBSymbolContext.h"
+#include "lldb/API/SBThread.h"
 #include "lldb/Breakpoint/BreakpointOptions.h"
 #include "lldb/Core/PluginInterface.h"
 #include "lldb/Core/SearchFilter.h"
@@ -580,6 +581,8 @@ class ScriptInterpreter : public PluginInterface {
 
   lldb::StreamSP GetOpaqueTypeFromSBStream(const lldb::SBStream &stream) const;
 
+  lldb::ThreadSP GetOpaqueTypeFromSBThread(const lldb::SBThread &exe_ctx) const;
+
   lldb::StackFrameSP GetOpaqueTypeFromSBFrame(const lldb::SBFrame &frame) const;
 
   SymbolContext
diff --git a/lldb/include/lldb/Target/BorrowedStackFrame.h b/lldb/include/lldb/Target/BorrowedStackFrame.h
new file mode 100644
index 0000000000000..72e7777961da7
--- /dev/null
+++ b/lldb/include/lldb/Target/BorrowedStackFrame.h
@@ -0,0 +1,146 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLDB_TARGET_BORROWEDSTACKFRAME_H
+#define LLDB_TARGET_BORROWEDSTACKFRAME_H
+
+#include "lldb/Target/StackFrame.h"
+
+namespace lldb_private {
+
+/// \class BorrowedStackFrame BorrowedStackFrame.h
+/// "lldb/Target/BorrowedStackFrame.h"
+///
+/// A wrapper around an existing StackFrame that supersedes its frame indices.
+///
+/// This class is useful when you need to present an existing stack frame
+/// with a different index, such as when creating synthetic frame views or
+/// renumbering frames without copying all the underlying data.
+///
+/// All methods delegate to the borrowed frame except for GetFrameIndex()
+/// & GetConcreteFrameIndex() which uses the overridden indices.
+class BorrowedStackFrame : public StackFrame {
+public:
+  /// Construct a BorrowedStackFrame that wraps an existing frame.
+  ///
+  /// \param [in] borrowed_frame_sp
+  ///   The existing StackFrame to borrow from. This frame's data will be
+  ///   used for all operations except frame index queries.
+  ///
+  /// \param [in] new_frame_index
+  ///   The frame index to report instead of the borrowed frame's index.
+  ///
+  /// \param [in] new_concrete_frame_index
+  ///   Optional concrete frame index. If not provided, defaults to
+  ///   new_frame_index.
+  BorrowedStackFrame(
+      lldb::StackFrameSP borrowed_frame_sp, uint32_t new_frame_index,
+      std::optional<uint32_t> new_concrete_frame_index = std::nullopt);
+
+  ~BorrowedStackFrame() override = default;
+
+  uint32_t GetFrameIndex() const override;
+  void SetFrameIndex(uint32_t index);
+
+  /// Get the concrete frame index for this borrowed frame.
+  ///
+  /// Returns the overridden concrete frame index provided at construction,
+  /// or LLDB_INVALID_FRAME_ID if the borrowed frame represents an inlined
+  /// function, since this would require some computation if we chain inlined
+  /// borrowed stack frames.
+  ///
+  /// \return
+  ///   The concrete frame index, or LLDB_INVALID_FRAME_ID for inline frames.
+  uint32_t GetConcreteFrameIndex() override;
+
+  StackID &GetStackID() override;
+
+  const Address &GetFrameCodeAddress() override;
+
+  Address GetFrameCodeAddressForSymbolication() override;
+
+  bool ChangePC(lldb::addr_t pc) override;
+
+  const SymbolContext &
+  GetSymbolContext(lldb::SymbolContextItem resolve_scope) override;
+
+  llvm::Error GetFrameBaseValue(Scalar &value) override;
+
+  DWARFExpressionList *GetFrameBaseExpression(Status *error_ptr) override;
+
+  Block *GetFrameBlock() override;
+
+  lldb::RegisterContextSP GetRegisterContext() override;
+
+  VariableList *GetVariableList(bool get_file_globals,
+                                Status *error_ptr) override;
+
+  lldb::VariableListSP
+  GetInScopeVariableList(bool get_file_globals,
+                         bool must_have_valid_location = false) override;
+
+  lldb::ValueObjectSP GetValueForVariableExpressionPath(
+      llvm::StringRef var_expr, lldb::DynamicValueType use_dynamic,
+      uint32_t options, lldb::VariableSP &var_sp, Status &error) override;
+
+  bool HasDebugInformation() override;
+
+  const char *Disassemble() override;
+
+  lldb::ValueObjectSP
+  GetValueObjectForFrameVariable(const lldb::VariableSP &variable_sp,
+                                 lldb::DynamicValueType use_dynamic) override;
+
+  bool IsInlined() override;
+
+  bool IsSynthetic() const override;
+
+  bool IsHistorical() const override;
+
+  bool IsArtificial() const override;
+
+  bool IsHidden() override;
+
+  const char *GetFunctionName() override;
+
+  const char *GetDisplayFunctionName() override;
+
+  lldb::ValueObjectSP FindVariable(ConstString name) override;
+
+  SourceLanguage GetLanguage() override;
+
+  SourceLanguage GuessLanguage() override;
+
+  lldb::ValueObjectSP GuessValueForAddress(lldb::addr_t addr) override;
+
+  lldb::ValueObjectSP GuessValueForRegisterAndOffset(ConstString reg,
+                                                     int64_t offset) override;
+
+  StructuredData::ObjectSP GetLanguageSpecificData() override;
+
+  lldb::RecognizedStackFrameSP GetRecognizedFrame() override;
+
+  /// Get the underlying borrowed frame.
+  lldb::StackFrameSP GetBorrowedFrame() const;
+
+  bool isA(const void *ClassID) const override;
+  static bool classof(const StackFrame *obj);
+
+private:
+  lldb::StackFrameSP m_borrowed_frame_sp;
+  uint32_t m_new_frame_index;
+  uint32_t m_new_concrete_frame_index;
+  static char ID;
+
+  BorrowedStackFrame(const BorrowedStackFrame &) = delete;
+  const BorrowedStackFrame &operator=(const BorrowedStackFrame &) = delete;
+};
+
+} // namespace lldb_private
+
+#endif // LLDB_TARGET_BORROWEDSTACKFRAME_H
diff --git a/lldb/include/lldb/Target/StackFrame.h b/lldb/include/lldb/Target/StackFrame.h
index 135bd81e4e8d4..46922448d6e59 100644
--- a/lldb/include/lldb/Target/StackFrame.h
+++ b/lldb/include/lldb/Target/StackFrame.h
@@ -43,6 +43,13 @@ namespace lldb_private {
 class StackFrame : public ExecutionContextScope,
                    public std::enable_shared_from_this<StackFrame> {
 public:
+  /// LLVM RTTI support.
+  /// \{
+  static char ID;
+  virtual bool isA(const void *ClassID) const { return ClassID == &ID; }
+  static bool classof(const StackFrame *obj) { return obj->isA(&ID); }
+  /// \}
+
   enum ExpressionPathOption {
     eExpressionPathOptionCheckPtrVsMember = (1u << 0),
     eExpressionPathOptionsNoFragileObjcIvar = (1u << 1),
@@ -127,7 +134,7 @@ class StackFrame : public ExecutionContextScope,
 
   lldb::ThreadSP GetThread() const { return m_thread_wp.lock(); }
 
-  StackID &GetStackID();
+  virtual StackID &GetStackID();
 
   /// Get an Address for the current pc value in this StackFrame.
   ///
@@ -135,7 +142,7 @@ class StackFrame : public ExecutionContextScope,
   ///
   /// \return
   ///   The Address object set to the current PC value.
-  const Address &GetFrameCodeAddress();
+  virtual const Address &GetFrameCodeAddress();
 
   /// Get the current code Address suitable for symbolication,
   /// may not be the same as GetFrameCodeAddress().
@@ -153,7 +160,7 @@ class StackFrame : public ExecutionContextScope,
   ///
   /// \return
   ///   The Address object set to the current PC value.
-  Address GetFrameCodeAddressForSymbolication();
+  virtual Address GetFrameCodeAddressForSymbolication();
 
   /// Change the pc value for a given thread.
   ///
@@ -165,7 +172,7 @@ class StackFrame : public ExecutionContextScope,
   /// \return
   ///     true if the pc was changed.  false if this failed -- possibly
   ///     because this frame is not a live StackFrame.
-  bool ChangePC(lldb::addr_t pc);
+  virtual bool ChangePC(lldb::addr_t pc);
 
   /// Provide a SymbolContext for this StackFrame's current pc value.
   ///
@@ -181,7 +188,8 @@ class StackFrame : public ExecutionContextScope,
   /// \return
   ///   A SymbolContext reference which includes the types of information
   ///   requested by resolve_scope, if they are available.
-  const SymbolContext &GetSymbolContext(lldb::SymbolContextItem resolve_scope);
+  virtual const SymbolContext &
+  GetSymbolContext(lldb::SymbolContextItem resolve_scope);
 
   /// Return the Canonical Frame Address (DWARF term) for this frame.
   ///
@@ -199,7 +207,7 @@ class StackFrame : public ExecutionContextScope,
   /// \return
   ///   If there is an error determining the CFA address, return an error
   ///   explaining the failure. Success otherwise.
-  llvm::Error GetFrameBaseValue(Scalar &value);
+  virtual llvm::Error GetFrameBaseValue(Scalar &value);
 
   /// Get the DWARFExpressionList corresponding to the Canonical Frame Address.
   ///
@@ -211,7 +219,7 @@ class StackFrame : public ExecutionContextScope,
   ///
   /// \return
   ///   Returns the corresponding DWARF expression, or NULL.
-  DWARFExpressionList *GetFrameBaseExpression(Status *error_ptr);
+  virtual DWARFExpressionList *GetFrameBaseExpression(Status *error_ptr);
 
   /// Get the current lexical scope block for this StackFrame, if possible.
   ///
@@ -221,7 +229,7 @@ class StackFrame : public ExecutionContextScope,
   /// \return
   ///   A pointer to the current Block.  nullptr is returned if this can
   ///   not be provided.
-  Block *GetFrameBlock();
+  virtual Block *GetFrameBlock();
 
   /// Get the RegisterContext for this frame, if possible.
   ///
@@ -235,7 +243,7 @@ class StackFrame : public ExecutionContextScope,
   ///
   /// \return
   ///   The RegisterContext shared point for this frame.
-  lldb::RegisterContextSP GetRegisterContext();
+  virtual lldb::RegisterContextSP GetRegisterContext();
 
   const lldb::RegisterContextSP &GetRegisterContextSP() const {
     return m_reg_context_sp;
@@ -261,7 +269,8 @@ class StackFrame : public ExecutionContextScope,
   ///
   /// \return
   ///     A pointer to a list of variables.
-  VariableList *GetVariableList(bool get_file_globals, Status *error_ptr);
+  virtual VariableList *GetVariableList(bool get_file_globals,
+                                        Status *error_ptr);
 
   /// Retrieve the list of variables that are in scope at this StackFrame's
   /// pc.
@@ -280,7 +289,7 @@ class StackFrame : public ExecutionContextScope,
   ///     StackFrame's pc.
   /// \return
   ///     A pointer to a list of variables.
-  lldb::VariableListSP
+  virtual lldb::VariableListSP
   GetInScopeVariableList(bool get_file_globals,
                          bool must_have_valid_location = false);
 
@@ -309,7 +318,7 @@ class StackFrame : public ExecutionContextScope,
   ///
   /// \return
   ///     A shared pointer to the ValueObject described by var_expr.
-  lldb::ValueObjectSP GetValueForVariableExpressionPath(
+  virtual lldb::ValueObjectSP GetValueForVariableExpressionPath(
       llvm::StringRef var_expr, lldb::DynamicValueType use_dynamic,
       uint32_t options, lldb::VariableSP &var_sp, Status &error);
 
@@ -318,14 +327,14 @@ class StackFrame : public ExecutionContextScope,
   /// \return
   ///    true if debug information is available for this frame (function,
   ///    compilation unit, block, etc.)
-  bool HasDebugInformation();
+  virtual bool HasDebugInformation();
 
   /// Return the disassembly for the instructions of this StackFrame's
   /// function as a single C string.
   ///
   /// \return
   ///    C string with the assembly instructions for this function.
-  const char *Disassemble();
+  virtual const char *Disassemble();
 
   /// Print a description of this frame using the provided frame format.
   ///
@@ -337,9 +346,9 @@ class StackFrame : public ExecutionContextScope,
   ///
   /// \return
   ///   \b true if and only if dumping with the given \p format worked.
-  bool DumpUsingFormat(Stream &strm,
-                       const lldb_private::FormatEntity::Entry *format,
-                       llvm::StringRef frame_marker = {});
+  virtual bool DumpUsingFormat(Stream &strm,
+                               const lldb_private::FormatEntity::Entry *format,
+                               llvm::StringRef frame_marker = {});
 
   /// Print a description for this frame using the frame-format formatter
   /// settings. If the current frame-format settings are invalid, then the
@@ -353,8 +362,8 @@ class StackFrame : public ExecutionContextScope,
   ///
   /// \param [in] frame_marker
   ///   Optional string that will be prepended to the frame output description.
-  void DumpUsingSettingsFormat(Stream *strm, bool show_unique = false,
-                               const char *frame_marker = nullptr);
+  virtual void DumpUsingSettingsFormat(Stream *strm, bool show_unique = false,
+                                       const char *frame_marker = nullptr);
 
   /// Print a description for this frame using a default format.
   ///
@@ -366,7 +375,7 @@ class StackFrame : public ExecutionContextScope,
   ///
   /// \param [in] show_fullpaths
   ///   Whether to print the full source paths or just the file base name.
-  void Dump(Stream *strm, bool show_frame_index, bool show_fullpaths);
+  virtual void Dump(Stream *strm, bool show_frame_index, bool show_fullpaths);
 
   /// Print a description of this stack frame and/or the source
   /// context/assembly for this stack frame.
@@ -389,8 +398,9 @@ class StackFrame : public ExecutionContextScope,
   ///
   /// \return
   ///   Returns true if successful.
-  bool GetStatus(Stream &strm, bool show_frame_info, bool show_source,
-                 bool show_unique = false, const char *frame_marker = nullptr);
+  virtual bool GetStatus(Stream &strm, bool show_frame_info, bool show_source,
+                         bool show_unique = false,
+                         const char *frame_marker = nullptr);
 
   /// Query whether this frame is a concrete frame on the call stack, or if it
   /// is an inlined frame derived from the debug information and presented by
@@ -401,10 +411,10 @@ class StackFrame : public ExecutionContextScope,
   virtual bool IsInlined();
 
   /// Query whether this frame is synthetic.
-  bool IsSynthetic() const;
+  virtual bool IsSynthetic() const;
 
   /// Query whether this frame is part of a historical backtrace.
-  bool IsHistorical() const;
+  virtual bool IsHistorical() const;
 
   /// Query whether this frame is artificial (e.g a synthesized result of
   /// inferring missing tail call frames from a backtrace). Artificial frames
@@ -419,7 +429,7 @@ class StackFrame : public ExecutionContextScope,
   /// Language plugins can use this API to report language-specific
   /// runtime information about this compile unit, such as additional
   /// language version details or feature flags.
-  StructuredData::ObjectSP GetLanguageSpecificData();
+  virtual StructuredData::ObjectSP GetLanguageSpecificData();
 
   /// Get the frame's demangled name.
   ///
@@ -439,9 +449,9 @@ class StackFrame : public ExecutionContextScope,
   /// \return
   ///   StackFrame index 0 indicates the currently-executing function.  Inline
   ///   frames are included in this frame index count.
-  uint32_t GetFrameIndex() const;
+  virtual uint32_t GetFrameIndex() const;
 
-  /// Set this frame's synthetic frame index.
+  /// Set this frame's frame index.
   void SetFrameIndex(uint32_t index) { m_frame_index = index; }
 
   /// Query this frame to find what frame it is in this Thread's
@@ -452,7 +462,7 @@ class StackFrame : public ExecutionContextScope,
   ///   frames are not included in this frame index count; their concrete
   ///   frame index will be the same as the concrete frame that they are
   ///   derived from.
-  uint32_t GetConcreteFrameIndex() const { return m_concrete_frame_index; }
+  virtual uint32_t GetConcreteFrameIndex() { return m_concrete_frame_index; }
 
   /// Create a ValueObject for a given Variable in this StackFrame.
   ///
@@ -466,7 +476,7 @@ class StackFrame : public ExecutionContextScope,
   ///
   /// \return
   ///     A ValueObject for this variable.
-  lldb::ValueObjectSP
+  virtual lldb::ValueObjectSP
   GetValueObjectForFrameVariable(const lldb::VariableSP &variable_sp,
                                  lldb::DynamicValueType use_dynamic);
 
@@ -474,11 +484,11 @@ class StackFrame : public ExecutionContextScope,
   /// parsing expressions given the execution context.
   ///
   /// \return   The language of the frame if known.
-  SourceLanguage GetLanguage();
+  virtual SourceLanguage GetLanguage();
 
   /// Similar to GetLanguage(), but is allowed to take a potentially incorrect
   /// guess if exact information is not available.
-  SourceLanguage GuessLanguage();
+  virtual SourceLanguage GuessLanguage();
 
   /// Attempt to econstruct the ValueObject for a given raw address touched by
   /// the current instruction.  The ExpressionPath should indicate how to get
@@ -489,7 +499,7 @@ class StackFrame : public ExecutionContextScope,
   ///
   /// \return
   ///   The ValueObject if found.  If valid, it has a valid ExpressionPath.
-  lldb::ValueObjectSP GuessValueForAddress(lldb::addr_t addr);
+  virtual lldb::ValueObjectSP GuessValueForAddress(lldb::addr_t addr);
 
   /// Attempt to reconstruct the ValueObject for the address contained in a
   /// given register plus an offset.  The ExpressionPath should indicate how
@@ -503,8 +513,8 @@ class StackFrame : public ExecutionContextScope,
   ///
   /// \return
   ///   The ValueObject if found.  If valid, it has a valid ExpressionPath.
-  lldb::ValueObjectSP GuessValueForRegisterAndOffset(ConstString reg,
-                                                     int64_t offset);
+  virtual lldb::ValueObjectSP GuessValueForRegisterAndOffset(ConstString reg,
+                                                             int64_t offset);
 
   /// Attempt to reconstruct the ValueObject for a variable with a given \a name
   /// from within the current StackFrame, within the current block. The search
@@ -517,7 +527,7 @@ class StackFrame : public ExecutionContextScope,
   ///
   /// \return
   ///   The ValueObject if found.
-  lldb::ValueObjectSP FindVariable(ConstString name);
+  virtual lldb::ValueObjectSP FindVariable(ConstString name);
 
   // lldb::ExecutionContextScope pure virtual functions
   lldb::TargetSP CalculateTarget() override;
@@ -530,7 +540,7 @@ class StackFrame : public ExecutionContextScope,
 
   void CalculateExecutionContext(ExecutionContext &exe_ctx) override;
 
-  lldb::RecognizedStackFrameSP GetRecognizedFrame();
+  virtual lldb::RecognizedStackFrameSP GetRecognizedFrame();
 
   /// Get the StackFrameList that contains this frame.
   ///
@@ -546,7 +556,9 @@ class StackFrame : public ExecutionContextScope,
   }
 
 protected:
+  friend class BorrowedStackFrame;
   friend class StackFrameList;
+  friend class SyntheticStackFrameList;
 
   void SetSymbolContextScope(SymbolContextScope *symbol_scope);
 
diff --git a/lldb/include/lldb/Target/StackFrameList.h b/lldb/include/lldb/Target/StackFrameList.h
index 8c14e92a41a4e..539c070ff0f4b 100644
--- a/lldb/include/lldb/Target/StackFrameList.h
+++ b/lldb/include/lldb/Target/StackFrameList.h
@@ -26,7 +26,7 @@ class StackFrameList : public std::enable_shared_from_this<StackFrameList> {
   StackFrameList(Thread &thread, const lldb::StackFrameListSP &prev_frames_sp,
                  bool show_inline_frames);
 
-  ~StackFrameList();
+  virtual ~StackFrameList();
 
   /// Get the number of visible frames. Frames may be created if \p can_create
   /// is true. Synthetic (inline) frames expanded from the concrete frame #0
@@ -106,6 +106,7 @@ class StackFrameList : public std::enable_shared_from_this<StackFrameList> {
 
 protected:
   friend class Thread;
+  friend class ScriptedFrameProvider;
   friend class ScriptedThread;
 
   /// Use this API to build a stack frame list (used for scripted threads, for
@@ -211,19 +212,23 @@ class StackFrameList : public std::enable_shared_from_this<StackFrameList> {
   /// Whether or not to show synthetic (inline) frames. Immutable.
   const bool m_show_inlined_frames;
 
+  /// Returns true if fetching frames was interrupted, false otherwise.
+  virtual bool FetchFramesUpTo(uint32_t end_idx,
+                               InterruptionControl allow_interrupt);
+
 private:
   uint32_t SetSelectedFrameNoLock(lldb_private::StackFrame *frame);
   lldb::StackFrameSP
   GetFrameAtIndexNoLock(uint32_t idx,
                         std::shared_lock<std::shared_mutex> &guard);
 
+  /// @{
   /// These two Fetch frames APIs and SynthesizeTailCallFrames are called in
   /// GetFramesUpTo, they are the ones that actually add frames.  They must be
   /// called with the writer end of the list mutex held.
-
-  /// Returns true if fetching frames was interrupted, false otherwise.
-  bool FetchFramesUpTo(uint32_t end_idx, InterruptionControl allow_interrupt);
+  ///
   /// Not currently interruptible so returns void.
+  /// }@
   void FetchOnlyConcreteFramesUpTo(uint32_t end_idx);
   void SynthesizeTailCallFrames(StackFrame &next_frame);
 
@@ -231,6 +236,27 @@ class StackFrameList : public std::enable_shared_from_this<StackFrameList> {
   const StackFrameList &operator=(const StackFrameList &) = delete;
 };
 
+/// A StackFrameList that wraps another StackFrameList and uses a
+/// SyntheticFrameProvider to lazily provide frames from either the provider
+/// or the underlying real stack frame list.
+class SyntheticStackFrameList : public StackFrameList {
+public:
+  SyntheticStackFrameList(Thread &thread, lldb::StackFrameListSP input_frames,
+                          const lldb::StackFrameListSP &prev_frames_sp,
+                          bool show_inline_frames);
+
+protected:
+  /// Override FetchFramesUpTo to lazily return frames from the provider
+  /// or from the actual stack frame list.
+  bool FetchFramesUpTo(uint32_t end_idx,
+                       InterruptionControl allow_interrupt) override;
+
+private:
+  /// The input stack frame list that the provider transforms.
+  /// This could be a real StackFrameList or another SyntheticStackFrameList.
+  lldb::StackFrameListSP m_input_frames;
+};
+
 } // namespace lldb_private
 
 #endif // LLDB_TARGET_STACKFRAMELIST_H
diff --git a/lldb/include/lldb/Target/SyntheticFrameProvider.h b/lldb/include/lldb/Target/SyntheticFrameProvider.h
index 61a492f356ece..2d5330cb03105 100644
--- a/lldb/include/lldb/Target/SyntheticFrameProvider.h
+++ b/lldb/include/lldb/Target/SyntheticFrameProvider.h
@@ -24,22 +24,25 @@ namespace lldb_private {
 
 /// This struct contains the metadata needed to instantiate a frame provider
 /// and optional filters to control which threads it applies to.
-struct SyntheticFrameProviderDescriptor {
+struct ScriptedFrameProviderDescriptor {
   /// Metadata for instantiating the provider (e.g. script class name and args).
   lldb::ScriptedMetadataSP scripted_metadata_sp;
 
+  /// Interface for calling static methods on the provider class.
+  lldb::ScriptedFrameProviderInterfaceSP interface_sp;
+
   /// Optional list of thread specifications to which this provider applies.
   /// If empty, the provider applies to all threads. A thread matches if it
   /// satisfies ANY of the specs in this vector (OR logic).
   std::vector<ThreadSpec> thread_specs;
 
-  SyntheticFrameProviderDescriptor() = default;
+  ScriptedFrameProviderDescriptor() = default;
 
-  SyntheticFrameProviderDescriptor(lldb::ScriptedMetadataSP metadata_sp)
+  ScriptedFrameProviderDescriptor(lldb::ScriptedMetadataSP metadata_sp)
       : scripted_metadata_sp(metadata_sp) {}
 
-  SyntheticFrameProviderDescriptor(lldb::ScriptedMetadataSP metadata_sp,
-                                   const std::vector<ThreadSpec> &specs)
+  ScriptedFrameProviderDescriptor(lldb::ScriptedMetadataSP metadata_sp,
+                                  const std::vector<ThreadSpec> &specs)
       : scripted_metadata_sp(metadata_sp), thread_specs(specs) {}
 
   /// Get the name of this descriptor (the scripted class name).
@@ -47,6 +50,12 @@ struct SyntheticFrameProviderDescriptor {
     return scripted_metadata_sp ? scripted_metadata_sp->GetClassName() : "";
   }
 
+  /// Get the description of this frame provider.
+  ///
+  /// \return A string describing what this frame provider does, or an
+  ///         empty string if no description is available.
+  std::string GetDescription() const;
+
   /// Check if this descriptor applies to the given thread.
   bool AppliesToThread(Thread &thread) const {
     // If no thread specs specified, applies to all threads.
@@ -64,6 +73,13 @@ struct SyntheticFrameProviderDescriptor {
   /// Check if this descriptor has valid metadata for script-based providers.
   bool IsValid() const { return scripted_metadata_sp != nullptr; }
 
+  /// Get a unique identifier for this descriptor based on its contents.
+  /// The ID is computed from the class name and arguments dictionary,
+  /// not from the pointer address, so two descriptors with the same
+  /// contents will have the same ID.
+  uint32_t GetID() const;
+
+  /// Dump a description of this descriptor to the given stream.
   void Dump(Stream *s) const;
 };
 
@@ -95,7 +111,7 @@ class SyntheticFrameProvider : public PluginInterface {
   ///     otherwise an \a llvm::Error.
   static llvm::Expected<lldb::SyntheticFrameProviderSP>
   CreateInstance(lldb::StackFrameListSP input_frames,
-                 const SyntheticFrameProviderDescriptor &descriptor);
+                 const ScriptedFrameProviderDescriptor &descriptor);
 
   /// Try to create a SyntheticFrameProvider instance for the given input
   /// frames using a specific C++ plugin.
@@ -125,6 +141,8 @@ class SyntheticFrameProvider : public PluginInterface {
 
   ~SyntheticFrameProvider() override;
 
+  virtual std::string GetDescription() const = 0;
+
   /// Get a single stack frame at the specified index.
   ///
   /// This method is called lazily - frames are only created when requested.
diff --git a/lldb/include/lldb/Target/Target.h b/lldb/include/lldb/Target/Target.h
index c0fcda7c0d960..812a638910b3b 100644
--- a/lldb/include/lldb/Target/Target.h
+++ b/lldb/include/lldb/Target/Target.h
@@ -32,6 +32,7 @@
 #include "lldb/Target/PathMappingList.h"
 #include "lldb/Target/SectionLoadHistory.h"
 #include "lldb/Target/Statistics.h"
+#include "lldb/Target/SyntheticFrameProvider.h"
 #include "lldb/Target/ThreadSpec.h"
 #include "lldb/Utility/ArchSpec.h"
 #include "lldb/Utility/Broadcaster.h"
@@ -745,6 +746,36 @@ class Target : public std::enable_shared_from_this<Target>,
   Status Attach(ProcessAttachInfo &attach_info,
                 Stream *stream); // Optional stream to receive first stop info
 
+  /// Add or update a scripted frame provider descriptor for this target.
+  /// All new threads in this target will check if they match any descriptors
+  /// to create their frame providers.
+  ///
+  /// \param[in] descriptor
+  ///     The descriptor to add or update.
+  ///
+  /// \return
+  ///     The descriptor identifier if the registration succeeded, otherwise an
+  ///     llvm::Error.
+  llvm::Expected<uint32_t> AddScriptedFrameProviderDescriptor(
+      const ScriptedFrameProviderDescriptor &descriptor);
+
+  /// Remove a scripted frame provider descriptor by id.
+  ///
+  /// \param[in] id
+  ///     The id of the descriptor to remove.
+  ///
+  /// \return
+  ///     True if a descriptor was removed, false if no descriptor with that
+  ///     id existed.
+  bool RemoveScriptedFrameProviderDescriptor(uint32_t id);
+
+  /// Clear all scripted frame provider descriptors for this target.
+  void ClearScriptedFrameProviderDescriptors();
+
+  /// Get all scripted frame provider descriptors for this target.
+  const llvm::DenseMap<uint32_t, ScriptedFrameProviderDescriptor> &
+  GetScriptedFrameProviderDescriptors() const;
+
   // This part handles the breakpoints.
 
   BreakpointList &GetBreakpointList(bool internal = false);
@@ -1744,6 +1775,13 @@ class Target : public std::enable_shared_from_this<Target>,
   PathMappingList m_image_search_paths;
   TypeSystemMap m_scratch_type_system_map;
 
+  /// Map of scripted frame provider descriptors for this target.
+  /// Keys are the provider descriptors ids, values are the descriptors.
+  /// Used to initialize frame providers for new threads.
+  llvm::DenseMap<uint32_t, ScriptedFrameProviderDescriptor>
+      m_frame_provider_descriptors;
+  mutable std::recursive_mutex m_frame_provider_descriptors_mutex;
+
   typedef std::map<lldb::LanguageType, lldb::REPLSP> REPLMap;
   REPLMap m_repl_map;
 
diff --git a/lldb/include/lldb/Target/Thread.h b/lldb/include/lldb/Target/Thread.h
index 841f80cd1b1eb..46ce192556756 100644
--- a/lldb/include/lldb/Target/Thread.h
+++ b/lldb/include/lldb/Target/Thread.h
@@ -1297,6 +1297,15 @@ class Thread : public std::enable_shared_from_this<Thread>,
 
   lldb::StackFrameListSP GetStackFrameList();
 
+  llvm::Error
+  LoadScriptedFrameProvider(const ScriptedFrameProviderDescriptor &descriptor);
+
+  void ClearScriptedFrameProvider();
+
+  lldb::SyntheticFrameProviderSP GetFrameProvider() const {
+    return m_frame_provider_sp;
+  }
+
 protected:
   friend class ThreadPlan;
   friend class ThreadList;
@@ -1400,6 +1409,9 @@ class Thread : public std::enable_shared_from_this<Thread>,
   /// The Thread backed by this thread, if any.
   lldb::ThreadWP m_backed_thread;
 
+  /// The Scripted Frame Provider, if any.
+  lldb::SyntheticFrameProviderSP m_frame_provider_sp;
+
 private:
   bool m_extended_info_fetched; // Have we tried to retrieve the m_extended_info
                                 // for this thread?
diff --git a/lldb/include/lldb/Target/ThreadSpec.h b/lldb/include/lldb/Target/ThreadSpec.h
index 7c7c832741196..63f8f8b5ec181 100644
--- a/lldb/include/lldb/Target/ThreadSpec.h
+++ b/lldb/include/lldb/Target/ThreadSpec.h
@@ -34,6 +34,8 @@ class ThreadSpec {
 public:
   ThreadSpec();
 
+  ThreadSpec(Thread &thread);
+
   static std::unique_ptr<ThreadSpec>
   CreateFromStructuredData(const StructuredData::Dictionary &data_dict,
                            Status &error);
diff --git a/lldb/include/lldb/Utility/ScriptedMetadata.h b/lldb/include/lldb/Utility/ScriptedMetadata.h
index 69c83edce909a..8523c95429718 100644
--- a/lldb/include/lldb/Utility/ScriptedMetadata.h
+++ b/lldb/include/lldb/Utility/ScriptedMetadata.h
@@ -10,7 +10,9 @@
 #define LLDB_INTERPRETER_SCRIPTEDMETADATA_H
 
 #include "lldb/Utility/ProcessInfo.h"
+#include "lldb/Utility/StreamString.h"
 #include "lldb/Utility/StructuredData.h"
+#include "llvm/ADT/Hashing.h"
 
 namespace lldb_private {
 class ScriptedMetadata {
@@ -27,11 +29,36 @@ class ScriptedMetadata {
     }
   }
 
+  ScriptedMetadata(const ScriptedMetadata &other)
+      : m_class_name(other.m_class_name), m_args_sp(other.m_args_sp) {}
+
   explicit operator bool() const { return !m_class_name.empty(); }
 
   llvm::StringRef GetClassName() const { return m_class_name; }
   StructuredData::DictionarySP GetArgsSP() const { return m_args_sp; }
 
+  /// Get a unique identifier for this metadata based on its contents.
+  /// The ID is computed from the class name and arguments dictionary,
+  /// not from the pointer address, so two metadata objects with the same
+  /// contents will have the same ID.
+  uint32_t GetID() const {
+    if (m_class_name.empty())
+      return 0;
+
+    // Hash the class name.
+    llvm::hash_code hash = llvm::hash_value(m_class_name);
+
+    // Hash the arguments dictionary if present.
+    if (m_args_sp) {
+      StreamString ss;
+      m_args_sp->GetDescription(ss);
+      hash = llvm::hash_combine(hash, llvm::hash_value(ss.GetData()));
+    }
+
+    // Return the lower 32 bits of the hash.
+    return static_cast<uint32_t>(hash);
+  }
+
 private:
   std::string m_class_name;
   StructuredData::DictionarySP m_args_sp;
diff --git a/lldb/include/lldb/ValueObject/ValueObjectSynthetic.h b/lldb/include/lldb/ValueObject/ValueObjectSynthetic.h
index 063d796ee4eec..1a82fd78bbba3 100644
--- a/lldb/include/lldb/ValueObject/ValueObjectSynthetic.h
+++ b/lldb/include/lldb/ValueObject/ValueObjectSynthetic.h
@@ -123,6 +123,11 @@ class ValueObjectSynthetic : public ValueObject {
 
   void SetLanguageFlags(uint64_t flags) override;
 
+  void
+  GetExpressionPath(Stream &stream,
+                    GetExpressionPathFormat epformat =
+                        eGetExpressionPathFormatDereferencePointers) override;
+
 protected:
   bool UpdateValue() override;
 
diff --git a/lldb/include/lldb/lldb-private-interfaces.h b/lldb/include/lldb/lldb-private-interfaces.h
index 5fc5c14c52f9e..52806eea190a7 100644
--- a/lldb/include/lldb/lldb-private-interfaces.h
+++ b/lldb/include/lldb/lldb-private-interfaces.h
@@ -26,7 +26,7 @@ class Value;
 
 namespace lldb_private {
 class ScriptedInterfaceUsages;
-struct SyntheticFrameProviderDescriptor;
+struct ScriptedFrameProviderDescriptor;
 typedef lldb::ABISP (*ABICreateInstance)(lldb::ProcessSP process_sp,
                                          const ArchSpec &arch);
 typedef std::unique_ptr<Architecture> (*ArchitectureCreateInstance)(
@@ -91,7 +91,7 @@ typedef lldb::ScriptInterpreterSP (*ScriptInterpreterCreateInstance)(
 typedef llvm::Expected<lldb::SyntheticFrameProviderSP> (
     *ScriptedFrameProviderCreateInstance)(
     lldb::StackFrameListSP input_frames,
-    const lldb_private::SyntheticFrameProviderDescriptor &descriptor);
+    const lldb_private::ScriptedFrameProviderDescriptor &descriptor);
 typedef llvm::Expected<lldb::SyntheticFrameProviderSP> (
     *SyntheticFrameProviderCreateInstance)(
     lldb::StackFrameListSP input_frames,
diff --git a/lldb/source/API/SBTarget.cpp b/lldb/source/API/SBTarget.cpp
index 578a7bdf7433d..78c2d49d647b5 100644
--- a/lldb/source/API/SBTarget.cpp
+++ b/lldb/source/API/SBTarget.cpp
@@ -23,6 +23,7 @@
 #include "lldb/API/SBStringList.h"
 #include "lldb/API/SBStructuredData.h"
 #include "lldb/API/SBSymbolContextList.h"
+#include "lldb/API/SBThreadCollection.h"
 #include "lldb/API/SBTrace.h"
 #include "lldb/Breakpoint/BreakpointID.h"
 #include "lldb/Breakpoint/BreakpointIDList.h"
@@ -39,6 +40,7 @@
 #include "lldb/Core/Section.h"
 #include "lldb/Core/StructuredDataImpl.h"
 #include "lldb/Host/Host.h"
+#include "lldb/Interpreter/Interfaces/ScriptedFrameProviderInterface.h"
 #include "lldb/Symbol/DeclVendor.h"
 #include "lldb/Symbol/ObjectFile.h"
 #include "lldb/Symbol/SymbolFile.h"
@@ -50,6 +52,7 @@
 #include "lldb/Target/LanguageRuntime.h"
 #include "lldb/Target/Process.h"
 #include "lldb/Target/StackFrame.h"
+#include "lldb/Target/SyntheticFrameProvider.h"
 #include "lldb/Target/Target.h"
 #include "lldb/Target/TargetList.h"
 #include "lldb/Utility/ArchSpec.h"
@@ -59,6 +62,7 @@
 #include "lldb/Utility/LLDBLog.h"
 #include "lldb/Utility/ProcessInfo.h"
 #include "lldb/Utility/RegularExpression.h"
+#include "lldb/Utility/ScriptedMetadata.h"
 #include "lldb/ValueObject/ValueObjectConstResult.h"
 #include "lldb/ValueObject/ValueObjectList.h"
 #include "lldb/ValueObject/ValueObjectVariable.h"
@@ -2435,3 +2439,81 @@ lldb::SBMutex SBTarget::GetAPIMutex() const {
     return lldb::SBMutex(target_sp);
   return lldb::SBMutex();
 }
+
+uint32_t
+SBTarget::RegisterScriptedFrameProvider(const char *class_name,
+                                        lldb::SBStructuredData args_dict,
+                                        lldb::SBError &error) {
+  LLDB_INSTRUMENT_VA(this, class_name, args_dict, error);
+
+  TargetSP target_sp = GetSP();
+  if (!target_sp) {
+    error.SetErrorString("invalid target");
+    return 0;
+  }
+
+  if (!class_name || !class_name[0]) {
+    error.SetErrorString("invalid class name");
+    return 0;
+  }
+
+  // Extract the dictionary from SBStructuredData.
+  StructuredData::DictionarySP dict_sp;
+  if (args_dict.IsValid() && args_dict.m_impl_up) {
+    StructuredData::ObjectSP obj_sp = args_dict.m_impl_up->GetObjectSP();
+    if (obj_sp && obj_sp->GetType() != lldb::eStructuredDataTypeDictionary) {
+      error.SetErrorString("SBStructuredData argument isn't a dictionary");
+      return 0;
+    }
+    dict_sp = std::make_shared<StructuredData::Dictionary>(obj_sp);
+  }
+
+  // Create the ScriptedMetadata.
+  ScriptedMetadataSP metadata_sp =
+      std::make_shared<ScriptedMetadata>(class_name, dict_sp);
+
+  // Create the interface for calling static methods.
+  ScriptedFrameProviderInterfaceSP interface_sp =
+      target_sp->GetDebugger()
+          .GetScriptInterpreter()
+          ->CreateScriptedFrameProviderInterface();
+
+  // Create a descriptor (applies to all threads by default).
+  ScriptedFrameProviderDescriptor descriptor(metadata_sp);
+  descriptor.interface_sp = interface_sp;
+
+  llvm::Expected<uint32_t> descriptor_id_or_err =
+      target_sp->AddScriptedFrameProviderDescriptor(descriptor);
+  if (!descriptor_id_or_err) {
+    error.SetErrorString(
+        llvm::toString(descriptor_id_or_err.takeError()).c_str());
+    return 0;
+  }
+
+  // Register the descriptor with the target.
+  return *descriptor_id_or_err;
+}
+
+lldb::SBError SBTarget::RemoveScriptedFrameProvider(uint32_t provider_id) {
+  LLDB_INSTRUMENT_VA(this, provider_id);
+
+  SBError error;
+  TargetSP target_sp = GetSP();
+  if (!target_sp) {
+    error.SetErrorString("invalid target");
+    return error;
+  }
+
+  if (!provider_id) {
+    error.SetErrorString("invalid provider id");
+    return error;
+  }
+
+  if (!target_sp->RemoveScriptedFrameProviderDescriptor(provider_id)) {
+    error.SetErrorStringWithFormat("no frame provider named '%u' found",
+                                   provider_id);
+    return error;
+  }
+
+  return {};
+}
diff --git a/lldb/source/Commands/CommandObjectTarget.cpp b/lldb/source/Commands/CommandObjectTarget.cpp
index 7f880d223d6c3..6e8c94fa234cd 100644
--- a/lldb/source/Commands/CommandObjectTarget.cpp
+++ b/lldb/source/Commands/CommandObjectTarget.cpp
@@ -51,6 +51,7 @@
 #include "lldb/Utility/ConstString.h"
 #include "lldb/Utility/FileSpec.h"
 #include "lldb/Utility/LLDBLog.h"
+#include "lldb/Utility/ScriptedMetadata.h"
 #include "lldb/Utility/State.h"
 #include "lldb/Utility/Stream.h"
 #include "lldb/Utility/StructuredData.h"
@@ -5402,6 +5403,202 @@ class CommandObjectTargetDump : public CommandObjectMultiword {
   ~CommandObjectTargetDump() override = default;
 };
 
+#pragma mark CommandObjectTargetFrameProvider
+
+#define LLDB_OPTIONS_target_frame_provider_register
+#include "CommandOptions.inc"
+
+class CommandObjectTargetFrameProviderRegister : public CommandObjectParsed {
+public:
+  CommandObjectTargetFrameProviderRegister(CommandInterpreter &interpreter)
+      : CommandObjectParsed(
+            interpreter, "target frame-provider register",
+            "Register frame provider for all threads in this target.", nullptr,
+            eCommandRequiresTarget),
+
+        m_class_options("target frame-provider", true, 'C', 'k', 'v', 0) {
+    m_all_options.Append(&m_class_options, LLDB_OPT_SET_1 | LLDB_OPT_SET_2,
+                         LLDB_OPT_SET_ALL);
+    m_all_options.Finalize();
+
+    AddSimpleArgumentList(eArgTypeRunArgs, eArgRepeatOptional);
+  }
+
+  ~CommandObjectTargetFrameProviderRegister() override = default;
+
+  Options *GetOptions() override { return &m_all_options; }
+
+  std::optional<std::string> GetRepeatCommand(Args &current_command_args,
+                                              uint32_t index) override {
+    return std::string("");
+  }
+
+protected:
+  void DoExecute(Args &launch_args, CommandReturnObject &result) override {
+    ScriptedMetadataSP metadata_sp = std::make_shared<ScriptedMetadata>(
+        m_class_options.GetName(), m_class_options.GetStructuredData());
+
+    Target *target = m_exe_ctx.GetTargetPtr();
+    if (!target)
+      target = &GetDebugger().GetDummyTarget();
+
+    // Create the interface for calling static methods.
+    ScriptedFrameProviderInterfaceSP interface_sp =
+        GetDebugger()
+            .GetScriptInterpreter()
+            ->CreateScriptedFrameProviderInterface();
+
+    // Create a descriptor from the metadata (applies to all threads by
+    // default).
+    ScriptedFrameProviderDescriptor descriptor(metadata_sp);
+    descriptor.interface_sp = interface_sp;
+
+    auto id_or_err = target->AddScriptedFrameProviderDescriptor(descriptor);
+    if (!id_or_err) {
+      result.SetError(id_or_err.takeError());
+      return;
+    }
+
+    result.AppendMessageWithFormat(
+        "successfully registered scripted frame provider '%s' for target\n",
+        m_class_options.GetName().c_str());
+  }
+
+  OptionGroupPythonClassWithDict m_class_options;
+  OptionGroupOptions m_all_options;
+};
+
+class CommandObjectTargetFrameProviderClear : public CommandObjectParsed {
+public:
+  CommandObjectTargetFrameProviderClear(CommandInterpreter &interpreter)
+      : CommandObjectParsed(
+            interpreter, "target frame-provider clear",
+            "Clear all registered frame providers from this target.", nullptr,
+            eCommandRequiresTarget) {}
+
+  ~CommandObjectTargetFrameProviderClear() override = default;
+
+protected:
+  void DoExecute(Args &command, CommandReturnObject &result) override {
+    Target *target = m_exe_ctx.GetTargetPtr();
+    if (!target) {
+      result.AppendError("invalid target");
+      return;
+    }
+
+    target->ClearScriptedFrameProviderDescriptors();
+
+    result.SetStatus(eReturnStatusSuccessFinishResult);
+  }
+};
+
+class CommandObjectTargetFrameProviderList : public CommandObjectParsed {
+public:
+  CommandObjectTargetFrameProviderList(CommandInterpreter &interpreter)
+      : CommandObjectParsed(
+            interpreter, "target frame-provider list",
+            "List all registered frame providers for the target.", nullptr,
+            eCommandRequiresTarget) {}
+
+  ~CommandObjectTargetFrameProviderList() override = default;
+
+protected:
+  void DoExecute(Args &command, CommandReturnObject &result) override {
+    Target *target = m_exe_ctx.GetTargetPtr();
+    if (!target)
+      target = &GetDebugger().GetDummyTarget();
+
+    const auto &descriptors = target->GetScriptedFrameProviderDescriptors();
+    if (descriptors.empty()) {
+      result.AppendMessage("no frame providers registered for this target.");
+      result.SetStatus(eReturnStatusSuccessFinishResult);
+      return;
+    }
+
+    result.AppendMessageWithFormat("%u frame provider(s) registered:\n\n",
+                                   descriptors.size());
+
+    for (const auto &entry : descriptors) {
+      const ScriptedFrameProviderDescriptor &descriptor = entry.second;
+      descriptor.Dump(&result.GetOutputStream());
+      result.GetOutputStream().PutChar('\n');
+    }
+
+    result.SetStatus(eReturnStatusSuccessFinishResult);
+  }
+};
+
+class CommandObjectTargetFrameProviderRemove : public CommandObjectParsed {
+public:
+  CommandObjectTargetFrameProviderRemove(CommandInterpreter &interpreter)
+      : CommandObjectParsed(
+            interpreter, "target frame-provider remove",
+            "Remove a registered frame provider from the target by id.",
+            "target frame-provider remove <provider-id>",
+            eCommandRequiresTarget) {
+    AddSimpleArgumentList(eArgTypeUnsignedInteger, eArgRepeatPlus);
+  }
+
+  ~CommandObjectTargetFrameProviderRemove() override = default;
+
+protected:
+  void DoExecute(Args &command, CommandReturnObject &result) override {
+    Target *target = m_exe_ctx.GetTargetPtr();
+    if (!target)
+      target = &GetDebugger().GetDummyTarget();
+
+    std::vector<uint32_t> removed_provider_ids;
+    for (size_t i = 0; i < command.GetArgumentCount(); i++) {
+      uint32_t provider_id = 0;
+      if (!llvm::to_integer(command[i].ref(), provider_id)) {
+        result.AppendError("target frame-provider remove requires integer "
+                           "provider id argument");
+        return;
+      }
+
+      if (!target->RemoveScriptedFrameProviderDescriptor(provider_id)) {
+        result.AppendErrorWithFormat(
+            "no frame provider named '%u' found in target\n", provider_id);
+        return;
+      }
+      removed_provider_ids.push_back(provider_id);
+    }
+
+    if (size_t num_removed_providers = removed_provider_ids.size()) {
+      result.AppendMessageWithFormat(
+          "Successfully removed %zu frame-providers.\n", num_removed_providers);
+      result.SetStatus(eReturnStatusSuccessFinishNoResult);
+    } else {
+      result.AppendError("0 frame providers removed.\n");
+    }
+  }
+};
+
+class CommandObjectTargetFrameProvider : public CommandObjectMultiword {
+public:
+  CommandObjectTargetFrameProvider(CommandInterpreter &interpreter)
+      : CommandObjectMultiword(
+            interpreter, "target frame-provider",
+            "Commands for registering and viewing frame providers for the "
+            "target.",
+            "target frame-provider [<sub-command-options>] ") {
+    LoadSubCommand("register",
+                   CommandObjectSP(new CommandObjectTargetFrameProviderRegister(
+                       interpreter)));
+    LoadSubCommand("clear",
+                   CommandObjectSP(
+                       new CommandObjectTargetFrameProviderClear(interpreter)));
+    LoadSubCommand(
+        "list",
+        CommandObjectSP(new CommandObjectTargetFrameProviderList(interpreter)));
+    LoadSubCommand(
+        "remove", CommandObjectSP(
+                      new CommandObjectTargetFrameProviderRemove(interpreter)));
+  }
+
+  ~CommandObjectTargetFrameProvider() override = default;
+};
+
 #pragma mark CommandObjectMultiwordTarget
 
 // CommandObjectMultiwordTarget
@@ -5417,6 +5614,9 @@ CommandObjectMultiwordTarget::CommandObjectMultiwordTarget(
                  CommandObjectSP(new CommandObjectTargetDelete(interpreter)));
   LoadSubCommand("dump",
                  CommandObjectSP(new CommandObjectTargetDump(interpreter)));
+  LoadSubCommand(
+      "frame-provider",
+      CommandObjectSP(new CommandObjectTargetFrameProvider(interpreter)));
   LoadSubCommand("list",
                  CommandObjectSP(new CommandObjectTargetList(interpreter)));
   LoadSubCommand("select",
diff --git a/lldb/source/Core/FormatEntity.cpp b/lldb/source/Core/FormatEntity.cpp
index 491f5c6320d97..c528a14fa76d0 100644
--- a/lldb/source/Core/FormatEntity.cpp
+++ b/lldb/source/Core/FormatEntity.cpp
@@ -27,6 +27,7 @@
 #include "lldb/Symbol/Symbol.h"
 #include "lldb/Symbol/SymbolContext.h"
 #include "lldb/Symbol/VariableList.h"
+#include "lldb/Target/BorrowedStackFrame.h"
 #include "lldb/Target/ExecutionContext.h"
 #include "lldb/Target/ExecutionContextScope.h"
 #include "lldb/Target/Language.h"
@@ -109,6 +110,7 @@ constexpr Definition g_frame_child_entries[] = {
                                   g_string_entry),
     Definition("is-artificial", EntryType::FrameIsArtificial),
     Definition("kind", EntryType::FrameKind),
+    Definition("borrowed-info", EntryType::FrameBorrowedInfo),
 };
 
 constexpr Definition g_function_child_entries[] = {
@@ -382,6 +384,7 @@ const char *FormatEntity::Entry::TypeToCString(Type t) {
     ENUM_TO_CSTR(FrameRegisterByName);
     ENUM_TO_CSTR(FrameIsArtificial);
     ENUM_TO_CSTR(FrameKind);
+    ENUM_TO_CSTR(FrameBorrowedInfo);
     ENUM_TO_CSTR(ScriptFrame);
     ENUM_TO_CSTR(FunctionID);
     ENUM_TO_CSTR(FunctionDidChange);
@@ -1761,6 +1764,22 @@ bool FormatEntity::Format(const Entry &entry, Stream &s,
     return false;
   }
 
+  case Entry::Type::FrameBorrowedInfo: {
+    if (exe_ctx)
+      if (StackFrame *frame = exe_ctx->GetFramePtr()) {
+        if (BorrowedStackFrame *borrowed_frame =
+                llvm::dyn_cast<BorrowedStackFrame>(frame)) {
+          if (lldb::StackFrameSP borrowed_from_sp =
+                  borrowed_frame->GetBorrowedFrame()) {
+            s.Printf(" [borrowed from frame #%u]",
+                     borrowed_from_sp->GetFrameIndex());
+            return true;
+          }
+        }
+      }
+    return false;
+  }
+
   case Entry::Type::ScriptFrame:
     if (exe_ctx) {
       StackFrame *frame = exe_ctx->GetFramePtr();
diff --git a/lldb/source/Expression/DWARFExpression.cpp b/lldb/source/Expression/DWARFExpression.cpp
index f4d1070d0a785..364b2ecadadd4 100644
--- a/lldb/source/Expression/DWARFExpression.cpp
+++ b/lldb/source/Expression/DWARFExpression.cpp
@@ -879,11 +879,11 @@ static Scalar DerefSizeExtractDataHelper(uint8_t *addr_bytes,
   return addr_data.GetAddress(&addr_data_offset);
 }
 
-static llvm::Error Evaluate_DW_OP_deref_size(DWARFExpression::Stack &stack,
-                                             ExecutionContext *exe_ctx,
-                                             lldb::ModuleSP module_sp,
-                                             Process *process, Target *target,
-                                             uint8_t size) {
+static llvm::Error Evaluate_DW_OP_deref_size(
+    DWARFExpression::Stack &stack, ExecutionContext *exe_ctx,
+    lldb::ModuleSP module_sp, Process *process, Target *target, uint8_t size,
+    size_t size_addr_bytes,
+    LocationDescriptionKind &dwarf4_location_description_kind) {
   if (stack.empty())
     return llvm::createStringError(
         "expression stack empty for DW_OP_deref_size");
@@ -892,6 +892,25 @@ static llvm::Error Evaluate_DW_OP_deref_size(DWARFExpression::Stack &stack,
     return llvm::createStringError(
         "Invalid address size for DW_OP_deref_size: %d\n", size);
 
+  // Deref a register or implicit location and truncate the value to `size`
+  // bytes. See the corresponding comment in DW_OP_deref for more details on
+  // why we deref these locations this way.
+  if (dwarf4_location_description_kind == Register ||
+      dwarf4_location_description_kind == Implicit) {
+    // Reset context to default values.
+    dwarf4_location_description_kind = Memory;
+    stack.back().ClearContext();
+
+    // Truncate the value on top of the stack to *size* bytes then
+    // extend to the size of an address (e.g. generic type).
+    Scalar scalar = stack.back().GetScalar();
+    scalar.TruncOrExtendTo(size * 8, /*sign=*/false);
+    scalar.TruncOrExtendTo(size_addr_bytes * 8,
+                           /*sign=*/false);
+    stack.back().GetScalar() = scalar;
+    return llvm::Error::success();
+  }
+
   Value::ValueType value_type = stack.back().GetValueType();
   switch (value_type) {
   case Value::ValueType::HostAddress: {
@@ -1142,8 +1161,9 @@ llvm::Expected<Value> DWARFExpression::Evaluate(
     // target machine.
     case DW_OP_deref: {
       size_t size = opcodes.GetAddressByteSize();
-      if (llvm::Error err = Evaluate_DW_OP_deref_size(stack, exe_ctx, module_sp,
-                                                      process, target, size))
+      if (llvm::Error err = Evaluate_DW_OP_deref_size(
+              stack, exe_ctx, module_sp, process, target, size, size,
+              dwarf4_location_description_kind))
         return err;
     } break;
 
@@ -1161,8 +1181,9 @@ llvm::Expected<Value> DWARFExpression::Evaluate(
     // expression stack.
     case DW_OP_deref_size: {
       size_t size = opcodes.GetU8(&offset);
-      if (llvm::Error err = Evaluate_DW_OP_deref_size(stack, exe_ctx, module_sp,
-                                                      process, target, size))
+      if (llvm::Error err = Evaluate_DW_OP_deref_size(
+              stack, exe_ctx, module_sp, process, target, size,
+              opcodes.GetAddressByteSize(), dwarf4_location_description_kind))
         return err;
     } break;
 
diff --git a/lldb/source/Interpreter/ScriptInterpreter.cpp b/lldb/source/Interpreter/ScriptInterpreter.cpp
index d2fd372bfe9e3..7bad10ff3ea61 100644
--- a/lldb/source/Interpreter/ScriptInterpreter.cpp
+++ b/lldb/source/Interpreter/ScriptInterpreter.cpp
@@ -106,6 +106,13 @@ ScriptInterpreter::GetStatusFromSBError(const lldb::SBError &error) const {
   return Status();
 }
 
+lldb::ThreadSP ScriptInterpreter::GetOpaqueTypeFromSBThread(
+    const lldb::SBThread &thread) const {
+  if (thread.m_opaque_sp)
+    return thread.m_opaque_sp->GetThreadSP();
+  return nullptr;
+}
+
 lldb::StackFrameSP
 ScriptInterpreter::GetOpaqueTypeFromSBFrame(const lldb::SBFrame &frame) const {
   if (frame.m_opaque_sp)
diff --git a/lldb/source/Plugins/CMakeLists.txt b/lldb/source/Plugins/CMakeLists.txt
index 08f444e7b15e8..b6878b21ff71a 100644
--- a/lldb/source/Plugins/CMakeLists.txt
+++ b/lldb/source/Plugins/CMakeLists.txt
@@ -22,6 +22,7 @@ add_subdirectory(SymbolFile)
 add_subdirectory(SystemRuntime)
 add_subdirectory(SymbolLocator)
 add_subdirectory(SymbolVendor)
+add_subdirectory(SyntheticFrameProvider)
 add_subdirectory(Trace)
 add_subdirectory(TraceExporter)
 add_subdirectory(TypeSystem)
diff --git a/lldb/source/Plugins/Process/scripted/ScriptedFrame.cpp b/lldb/source/Plugins/Process/scripted/ScriptedFrame.cpp
index 6519df9185df0..265bc28a8957f 100644
--- a/lldb/source/Plugins/Process/scripted/ScriptedFrame.cpp
+++ b/lldb/source/Plugins/Process/scripted/ScriptedFrame.cpp
@@ -7,42 +7,72 @@
 //===----------------------------------------------------------------------===//
 
 #include "ScriptedFrame.h"
-
+#include "Plugins/Process/Utility/RegisterContextMemory.h"
+
+#include "lldb/Core/Address.h"
+#include "lldb/Core/Debugger.h"
+#include "lldb/Interpreter/Interfaces/ScriptedFrameInterface.h"
+#include "lldb/Interpreter/Interfaces/ScriptedThreadInterface.h"
+#include "lldb/Interpreter/ScriptInterpreter.h"
+#include "lldb/Symbol/SymbolContext.h"
+#include "lldb/Target/ExecutionContext.h"
+#include "lldb/Target/Process.h"
+#include "lldb/Target/RegisterContext.h"
+#include "lldb/Target/Thread.h"
 #include "lldb/Utility/DataBufferHeap.h"
+#include "lldb/Utility/LLDBLog.h"
+#include "lldb/Utility/Log.h"
+#include "lldb/Utility/StructuredData.h"
 
 using namespace lldb;
 using namespace lldb_private;
 
+char ScriptedFrame::ID;
+
 void ScriptedFrame::CheckInterpreterAndScriptObject() const {
   lldbassert(m_script_object_sp && "Invalid Script Object.");
   lldbassert(GetInterface() && "Invalid Scripted Frame Interface.");
 }
 
 llvm::Expected<std::shared_ptr<ScriptedFrame>>
-ScriptedFrame::Create(ScriptedThread &thread,
+ScriptedFrame::Create(ThreadSP thread_sp,
+                      ScriptedThreadInterfaceSP scripted_thread_interface_sp,
                       StructuredData::DictionarySP args_sp,
                       StructuredData::Generic *script_object) {
-  if (!thread.IsValid())
-    return llvm::createStringError("Invalid scripted thread.");
+  if (!thread_sp || !thread_sp->IsValid())
+    return llvm::createStringError("invalid thread");
+
+  ProcessSP process_sp = thread_sp->GetProcess();
+  if (!process_sp || !process_sp->IsValid())
+    return llvm::createStringError("invalid process");
 
-  thread.CheckInterpreterAndScriptObject();
+  ScriptInterpreter *script_interp =
+      process_sp->GetTarget().GetDebugger().GetScriptInterpreter();
+  if (!script_interp)
+    return llvm::createStringError("no script interpreter");
 
-  auto scripted_frame_interface =
-      thread.GetInterface()->CreateScriptedFrameInterface();
+  auto scripted_frame_interface = script_interp->CreateScriptedFrameInterface();
   if (!scripted_frame_interface)
     return llvm::createStringError("failed to create scripted frame interface");
 
   llvm::StringRef frame_class_name;
   if (!script_object) {
-    std::optional<std::string> class_name =
-        thread.GetInterface()->GetScriptedFramePluginName();
-    if (!class_name || class_name->empty())
+    // If no script object is provided and we have a scripted thread interface,
+    // try to get the frame class name from it.
+    if (scripted_thread_interface_sp) {
+      std::optional<std::string> class_name =
+          scripted_thread_interface_sp->GetScriptedFramePluginName();
+      if (!class_name || class_name->empty())
+        return llvm::createStringError(
+            "failed to get scripted frame class name");
+      frame_class_name = *class_name;
+    } else {
       return llvm::createStringError(
-          "failed to get scripted thread class name");
-    frame_class_name = *class_name;
+          "no script object provided and no scripted thread interface");
+    }
   }
 
-  ExecutionContext exe_ctx(thread);
+  ExecutionContext exe_ctx(thread_sp);
   auto obj_or_err = scripted_frame_interface->CreatePluginObject(
       frame_class_name, exe_ctx, args_sp, script_object);
 
@@ -62,7 +92,7 @@ ScriptedFrame::Create(ScriptedThread &thread,
   SymbolContext sc;
   Address symbol_addr;
   if (pc != LLDB_INVALID_ADDRESS) {
-    symbol_addr.SetLoadAddress(pc, &thread.GetProcess()->GetTarget());
+    symbol_addr.SetLoadAddress(pc, &process_sp->GetTarget());
     symbol_addr.CalculateSymbolContext(&sc);
   }
 
@@ -77,11 +107,11 @@ ScriptedFrame::Create(ScriptedThread &thread,
 
   if (!reg_info)
     return llvm::createStringError(
-        "failed to get scripted thread registers info");
+        "failed to get scripted frame registers info");
 
   std::shared_ptr<DynamicRegisterInfo> register_info_sp =
-      DynamicRegisterInfo::Create(
-          *reg_info, thread.GetProcess()->GetTarget().GetArchitecture());
+      DynamicRegisterInfo::Create(*reg_info,
+                                  process_sp->GetTarget().GetArchitecture());
 
   lldb::RegisterContextSP reg_ctx_sp;
 
@@ -96,32 +126,35 @@ ScriptedFrame::Create(ScriptedThread &thread,
 
     std::shared_ptr<RegisterContextMemory> reg_ctx_memory =
         std::make_shared<RegisterContextMemory>(
-            thread, frame_id, *register_info_sp, LLDB_INVALID_ADDRESS);
+            *thread_sp, frame_id, *register_info_sp, LLDB_INVALID_ADDRESS);
     if (!reg_ctx_memory)
-      return llvm::createStringError("failed to create a register context.");
+      return llvm::createStringError("failed to create a register context");
 
     reg_ctx_memory->SetAllRegisterData(data_sp);
     reg_ctx_sp = reg_ctx_memory;
   }
 
   return std::make_shared<ScriptedFrame>(
-      thread, scripted_frame_interface, frame_id, pc, sc, reg_ctx_sp,
+      thread_sp, scripted_frame_interface, frame_id, pc, sc, reg_ctx_sp,
       register_info_sp, owned_script_object_sp);
 }
 
-ScriptedFrame::ScriptedFrame(ScriptedThread &thread,
+ScriptedFrame::ScriptedFrame(ThreadSP thread_sp,
                              ScriptedFrameInterfaceSP interface_sp,
                              lldb::user_id_t id, lldb::addr_t pc,
                              SymbolContext &sym_ctx,
                              lldb::RegisterContextSP reg_ctx_sp,
                              std::shared_ptr<DynamicRegisterInfo> reg_info_sp,
                              StructuredData::GenericSP script_object_sp)
-    : StackFrame(thread.shared_from_this(), /*frame_idx=*/id,
+    : StackFrame(thread_sp, /*frame_idx=*/id,
                  /*concrete_frame_idx=*/id, /*reg_context_sp=*/reg_ctx_sp,
                  /*cfa=*/0, /*pc=*/pc,
                  /*behaves_like_zeroth_frame=*/!id, /*symbol_ctx=*/&sym_ctx),
       m_scripted_frame_interface_sp(interface_sp),
-      m_script_object_sp(script_object_sp), m_register_info_sp(reg_info_sp) {}
+      m_script_object_sp(script_object_sp), m_register_info_sp(reg_info_sp) {
+  // FIXME: This should be part of the base class constructor.
+  m_stack_frame_kind = StackFrame::Kind::Synthetic;
+}
 
 ScriptedFrame::~ScriptedFrame() {}
 
@@ -164,7 +197,7 @@ std::shared_ptr<DynamicRegisterInfo> ScriptedFrame::GetDynamicRegisterInfo() {
     if (!reg_info)
       return ScriptedInterface::ErrorWithMessage<
           std::shared_ptr<DynamicRegisterInfo>>(
-          LLVM_PRETTY_FUNCTION, "Failed to get scripted frame registers info.",
+          LLVM_PRETTY_FUNCTION, "failed to get scripted frame registers info",
           error, LLDBLog::Thread);
 
     ThreadSP thread_sp = m_thread_wp.lock();
@@ -172,7 +205,7 @@ std::shared_ptr<DynamicRegisterInfo> ScriptedFrame::GetDynamicRegisterInfo() {
       return ScriptedInterface::ErrorWithMessage<
           std::shared_ptr<DynamicRegisterInfo>>(
           LLVM_PRETTY_FUNCTION,
-          "Failed to get scripted frame registers info: invalid thread.", error,
+          "failed to get scripted frame registers info: invalid thread", error,
           LLDBLog::Thread);
 
     ProcessSP process_sp = thread_sp->GetProcess();
@@ -180,8 +213,8 @@ std::shared_ptr<DynamicRegisterInfo> ScriptedFrame::GetDynamicRegisterInfo() {
       return ScriptedInterface::ErrorWithMessage<
           std::shared_ptr<DynamicRegisterInfo>>(
           LLVM_PRETTY_FUNCTION,
-          "Failed to get scripted frame registers info: invalid process.",
-          error, LLDBLog::Thread);
+          "failed to get scripted frame registers info: invalid process", error,
+          LLDBLog::Thread);
 
     m_register_info_sp = DynamicRegisterInfo::Create(
         *reg_info, process_sp->GetTarget().GetArchitecture());
diff --git a/lldb/source/Plugins/Process/scripted/ScriptedFrame.h b/lldb/source/Plugins/Process/scripted/ScriptedFrame.h
index b6b77c4a7d160..d1cbd429d4979 100644
--- a/lldb/source/Plugins/Process/scripted/ScriptedFrame.h
+++ b/lldb/source/Plugins/Process/scripted/ScriptedFrame.h
@@ -10,21 +10,19 @@
 #define LLDB_SOURCE_PLUGINS_SCRIPTED_FRAME_H
 
 #include "ScriptedThread.h"
-#include "lldb/Interpreter/ScriptInterpreter.h"
 #include "lldb/Target/DynamicRegisterInfo.h"
 #include "lldb/Target/StackFrame.h"
+#include "lldb/lldb-forward.h"
+#include "llvm/Support/Error.h"
+#include <memory>
 #include <string>
 
-namespace lldb_private {
-class ScriptedThread;
-}
-
 namespace lldb_private {
 
 class ScriptedFrame : public lldb_private::StackFrame {
 
 public:
-  ScriptedFrame(ScriptedThread &thread,
+  ScriptedFrame(lldb::ThreadSP thread_sp,
                 lldb::ScriptedFrameInterfaceSP interface_sp,
                 lldb::user_id_t frame_idx, lldb::addr_t pc,
                 SymbolContext &sym_ctx, lldb::RegisterContextSP reg_ctx_sp,
@@ -33,8 +31,29 @@ class ScriptedFrame : public lldb_private::StackFrame {
 
   ~ScriptedFrame() override;
 
+  /// Create a ScriptedFrame from a object instanciated in the script
+  /// interpreter.
+  ///
+  /// \param[in] thread_sp
+  ///     The thread this frame belongs to.
+  ///
+  /// \param[in] scripted_thread_interface_sp
+  ///     The scripted thread interface (needed for ScriptedThread
+  ///     compatibility). Can be nullptr for frames on real threads.
+  ///
+  /// \param[in] args_sp
+  ///     Arguments to pass to the frame creation.
+  ///
+  /// \param[in] script_object
+  ///     The optional script object representing this frame.
+  ///
+  /// \return
+  ///     An Expected containing the ScriptedFrame shared pointer if successful,
+  ///     otherwise an error.
   static llvm::Expected<std::shared_ptr<ScriptedFrame>>
-  Create(ScriptedThread &thread, StructuredData::DictionarySP args_sp,
+  Create(lldb::ThreadSP thread_sp,
+         lldb::ScriptedThreadInterfaceSP scripted_thread_interface_sp,
+         StructuredData::DictionarySP args_sp,
          StructuredData::Generic *script_object = nullptr);
 
   bool IsInlined() override;
@@ -43,6 +62,11 @@ class ScriptedFrame : public lldb_private::StackFrame {
   const char *GetFunctionName() override;
   const char *GetDisplayFunctionName() override;
 
+  bool isA(const void *ClassID) const override {
+    return ClassID == &ID || StackFrame::isA(ClassID);
+  }
+  static bool classof(const StackFrame *obj) { return obj->isA(&ID); }
+
 private:
   void CheckInterpreterAndScriptObject() const;
   lldb::ScriptedFrameInterfaceSP GetInterface() const;
@@ -55,6 +79,8 @@ class ScriptedFrame : public lldb_private::StackFrame {
   lldb::ScriptedFrameInterfaceSP m_scripted_frame_interface_sp;
   lldb_private::StructuredData::GenericSP m_script_object_sp;
   std::shared_ptr<DynamicRegisterInfo> m_register_info_sp;
+
+  static char ID;
 };
 
 } // namespace lldb_private
diff --git a/lldb/source/Plugins/Process/scripted/ScriptedThread.cpp b/lldb/source/Plugins/Process/scripted/ScriptedThread.cpp
index 491efac5aadef..1dd9c48f56a59 100644
--- a/lldb/source/Plugins/Process/scripted/ScriptedThread.cpp
+++ b/lldb/source/Plugins/Process/scripted/ScriptedThread.cpp
@@ -210,7 +210,7 @@ bool ScriptedThread::LoadArtificialStackFrames() {
     SymbolContext sc;
     symbol_addr.CalculateSymbolContext(&sc);
 
-    return std::make_shared<StackFrame>(this->shared_from_this(), idx, idx, cfa,
+    return std::make_shared<StackFrame>(shared_from_this(), idx, idx, cfa,
                                         cfa_is_valid, pc,
                                         StackFrame::Kind::Synthetic, artificial,
                                         behaves_like_zeroth_frame, &sc);
@@ -231,8 +231,8 @@ bool ScriptedThread::LoadArtificialStackFrames() {
       return error.ToError();
     }
 
-    auto frame_or_error =
-        ScriptedFrame::Create(*this, nullptr, object_sp->GetAsGeneric());
+    auto frame_or_error = ScriptedFrame::Create(
+        shared_from_this(), GetInterface(), nullptr, object_sp->GetAsGeneric());
 
     if (!frame_or_error) {
       ScriptedInterface::ErrorWithMessage<bool>(
diff --git a/lldb/source/Plugins/ScriptInterpreter/Python/Interfaces/ScriptInterpreterPythonInterfaces.cpp b/lldb/source/Plugins/ScriptInterpreter/Python/Interfaces/ScriptInterpreterPythonInterfaces.cpp
index d43036d6fe544..f6c707b2bd168 100644
--- a/lldb/source/Plugins/ScriptInterpreter/Python/Interfaces/ScriptInterpreterPythonInterfaces.cpp
+++ b/lldb/source/Plugins/ScriptInterpreter/Python/Interfaces/ScriptInterpreterPythonInterfaces.cpp
@@ -31,6 +31,7 @@ void ScriptInterpreterPythonInterfaces::Initialize() {
   ScriptedStopHookPythonInterface::Initialize();
   ScriptedBreakpointPythonInterface::Initialize();
   ScriptedThreadPlanPythonInterface::Initialize();
+  ScriptedFrameProviderPythonInterface::Initialize();
 }
 
 void ScriptInterpreterPythonInterfaces::Terminate() {
@@ -40,6 +41,7 @@ void ScriptInterpreterPythonInterfaces::Terminate() {
   ScriptedStopHookPythonInterface::Terminate();
   ScriptedBreakpointPythonInterface::Terminate();
   ScriptedThreadPlanPythonInterface::Terminate();
+  ScriptedFrameProviderPythonInterface::Terminate();
 }
 
 #endif
diff --git a/lldb/source/Plugins/ScriptInterpreter/Python/Interfaces/ScriptedFrameProviderPythonInterface.cpp b/lldb/source/Plugins/ScriptInterpreter/Python/Interfaces/ScriptedFrameProviderPythonInterface.cpp
index b866bf332b7b6..3dde5036453f4 100644
--- a/lldb/source/Plugins/ScriptInterpreter/Python/Interfaces/ScriptedFrameProviderPythonInterface.cpp
+++ b/lldb/source/Plugins/ScriptInterpreter/Python/Interfaces/ScriptedFrameProviderPythonInterface.cpp
@@ -6,6 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "lldb/Core/PluginManager.h"
 #include "lldb/Host/Config.h"
 #include "lldb/Target/Thread.h"
 #include "lldb/Utility/Log.h"
@@ -30,18 +31,45 @@ ScriptedFrameProviderPythonInterface::ScriptedFrameProviderPythonInterface(
     ScriptInterpreterPythonImpl &interpreter)
     : ScriptedFrameProviderInterface(), ScriptedPythonInterface(interpreter) {}
 
+bool ScriptedFrameProviderPythonInterface::AppliesToThread(
+    llvm::StringRef class_name, lldb::ThreadSP thread_sp) {
+  // If there is any issue with this method, we will just assume it also applies
+  // to this thread which is the default behavior.
+  constexpr bool fail_value = true;
+  Status error;
+  StructuredData::ObjectSP obj =
+      CallStaticMethod(class_name, "applies_to_thread", error, thread_sp);
+  if (!ScriptedInterface::CheckStructuredDataObject(LLVM_PRETTY_FUNCTION, obj,
+                                                    error))
+    return fail_value;
+
+  return obj->GetBooleanValue(fail_value);
+}
+
 llvm::Expected<StructuredData::GenericSP>
 ScriptedFrameProviderPythonInterface::CreatePluginObject(
     const llvm::StringRef class_name, lldb::StackFrameListSP input_frames,
     StructuredData::DictionarySP args_sp) {
   if (!input_frames)
-    return llvm::createStringError("Invalid frame list");
+    return llvm::createStringError("invalid frame list");
 
   StructuredDataImpl sd_impl(args_sp);
   return ScriptedPythonInterface::CreatePluginObject(class_name, nullptr,
                                                      input_frames, sd_impl);
 }
 
+std::string ScriptedFrameProviderPythonInterface::GetDescription(
+    llvm::StringRef class_name) {
+  Status error;
+  StructuredData::ObjectSP obj =
+      CallStaticMethod(class_name, "get_description", error);
+  if (!ScriptedInterface::CheckStructuredDataObject(LLVM_PRETTY_FUNCTION, obj,
+                                                    error))
+    return {};
+
+  return obj->GetStringValue().str();
+}
+
 StructuredData::ObjectSP
 ScriptedFrameProviderPythonInterface::GetFrameAtIndex(uint32_t index) {
   Status error;
@@ -54,4 +82,32 @@ ScriptedFrameProviderPythonInterface::GetFrameAtIndex(uint32_t index) {
   return obj;
 }
 
+bool ScriptedFrameProviderPythonInterface::CreateInstance(
+    lldb::ScriptLanguage language, ScriptedInterfaceUsages usages) {
+  if (language != eScriptLanguagePython)
+    return false;
+
+  return true;
+}
+
+void ScriptedFrameProviderPythonInterface::Initialize() {
+  const std::vector<llvm::StringRef> ci_usages = {
+      "target frame-provider register -C <script-name> [-k key -v value ...]",
+      "target frame-provider list",
+      "target frame-provider remove <provider-name>",
+      "target frame-provider clear"};
+  const std::vector<llvm::StringRef> api_usages = {
+      "SBTarget.RegisterScriptedFrameProvider",
+      "SBTarget.RemoveScriptedFrameProvider",
+      "SBTarget.ClearScriptedFrameProvider"};
+  PluginManager::RegisterPlugin(
+      GetPluginNameStatic(),
+      llvm::StringRef("Provide scripted stack frames for threads"),
+      CreateInstance, eScriptLanguagePython, {ci_usages, api_usages});
+}
+
+void ScriptedFrameProviderPythonInterface::Terminate() {
+  PluginManager::UnregisterPlugin(CreateInstance);
+}
+
 #endif
diff --git a/lldb/source/Plugins/ScriptInterpreter/Python/Interfaces/ScriptedFrameProviderPythonInterface.h b/lldb/source/Plugins/ScriptInterpreter/Python/Interfaces/ScriptedFrameProviderPythonInterface.h
index fd163984028d3..97a5cc7c669ea 100644
--- a/lldb/source/Plugins/ScriptInterpreter/Python/Interfaces/ScriptedFrameProviderPythonInterface.h
+++ b/lldb/source/Plugins/ScriptInterpreter/Python/Interfaces/ScriptedFrameProviderPythonInterface.h
@@ -14,17 +14,22 @@
 #if LLDB_ENABLE_PYTHON
 
 #include "ScriptedPythonInterface.h"
+#include "lldb/Core/PluginInterface.h"
 #include "lldb/Interpreter/Interfaces/ScriptedFrameProviderInterface.h"
 #include <optional>
 
 namespace lldb_private {
 class ScriptedFrameProviderPythonInterface
     : public ScriptedFrameProviderInterface,
-      public ScriptedPythonInterface {
+      public ScriptedPythonInterface,
+      public PluginInterface {
 public:
   ScriptedFrameProviderPythonInterface(
       ScriptInterpreterPythonImpl &interpreter);
 
+  bool AppliesToThread(llvm::StringRef class_name,
+                       lldb::ThreadSP thread_sp) override;
+
   llvm::Expected<StructuredData::GenericSP>
   CreatePluginObject(llvm::StringRef class_name,
                      lldb::StackFrameListSP input_frames,
@@ -33,10 +38,24 @@ class ScriptedFrameProviderPythonInterface
   llvm::SmallVector<AbstractMethodRequirement>
   GetAbstractMethodRequirements() const override {
     return llvm::SmallVector<AbstractMethodRequirement>(
-        {{"get_frame_at_index"}});
+        {{"get_description"}, {"get_frame_at_index"}});
   }
 
+  std::string GetDescription(llvm::StringRef class_name) override;
+
   StructuredData::ObjectSP GetFrameAtIndex(uint32_t index) override;
+
+  static void Initialize();
+  static void Terminate();
+
+  static bool CreateInstance(lldb::ScriptLanguage language,
+                             ScriptedInterfaceUsages usages);
+
+  static llvm::StringRef GetPluginNameStatic() {
+    return "ScriptedFrameProviderPythonInterface";
+  }
+
+  llvm::StringRef GetPluginName() override { return GetPluginNameStatic(); }
 };
 } // namespace lldb_private
 
diff --git a/lldb/source/Plugins/ScriptInterpreter/Python/Interfaces/ScriptedPythonInterface.cpp b/lldb/source/Plugins/ScriptInterpreter/Python/Interfaces/ScriptedPythonInterface.cpp
index af2e0b5df4d22..ba4473cf9ec4d 100644
--- a/lldb/source/Plugins/ScriptInterpreter/Python/Interfaces/ScriptedPythonInterface.cpp
+++ b/lldb/source/Plugins/ScriptInterpreter/Python/Interfaces/ScriptedPythonInterface.cpp
@@ -93,6 +93,19 @@ ScriptedPythonInterface::ExtractValueFromPythonObject<lldb::StackFrameSP>(
   return nullptr;
 }
 
+template <>
+lldb::ThreadSP
+ScriptedPythonInterface::ExtractValueFromPythonObject<lldb::ThreadSP>(
+    python::PythonObject &p, Status &error) {
+  if (lldb::SBThread *sb_thread = reinterpret_cast<lldb::SBThread *>(
+          python::LLDBSWIGPython_CastPyObjectToSBThread(p.get())))
+    return m_interpreter.GetOpaqueTypeFromSBThread(*sb_thread);
+  error = Status::FromErrorString(
+      "Couldn't cast lldb::SBThread to lldb_private::Thread.");
+
+  return nullptr;
+}
+
 template <>
 SymbolContext
 ScriptedPythonInterface::ExtractValueFromPythonObject<SymbolContext>(
diff --git a/lldb/source/Plugins/ScriptInterpreter/Python/Interfaces/ScriptedPythonInterface.h b/lldb/source/Plugins/ScriptInterpreter/Python/Interfaces/ScriptedPythonInterface.h
index 23c56610124a6..53a7ba65f64b7 100644
--- a/lldb/source/Plugins/ScriptInterpreter/Python/Interfaces/ScriptedPythonInterface.h
+++ b/lldb/source/Plugins/ScriptInterpreter/Python/Interfaces/ScriptedPythonInterface.h
@@ -387,6 +387,112 @@ class ScriptedPythonInterface : virtual public ScriptedInterface {
     return m_object_instance_sp;
   }
 
+  /// Call a static method on a Python class without creating an instance.
+  ///
+  /// This method resolves a Python class by name and calls a static method
+  /// on it, returning the result. This is useful for calling class-level
+  /// methods that don't require an instance.
+  ///
+  /// \param class_name The fully-qualified name of the Python class.
+  /// \param method_name The name of the static method to call.
+  /// \param error Output parameter to receive error information if the call
+  /// fails.
+  /// \param args Arguments to pass to the static method.
+  ///
+  /// \return The return value of the static method call, or an error value.
+  template <typename T = StructuredData::ObjectSP, typename... Args>
+  T CallStaticMethod(llvm::StringRef class_name, llvm::StringRef method_name,
+                     Status &error, Args &&...args) {
+    using namespace python;
+    using Locker = ScriptInterpreterPythonImpl::Locker;
+
+    std::string caller_signature =
+        llvm::Twine(LLVM_PRETTY_FUNCTION + llvm::Twine(" (") +
+                    llvm::Twine(class_name) + llvm::Twine(".") +
+                    llvm::Twine(method_name) + llvm::Twine(")"))
+            .str();
+
+    if (class_name.empty())
+      return ErrorWithMessage<T>(caller_signature, "missing script class name",
+                                 error);
+
+    Locker py_lock(&m_interpreter, Locker::AcquireLock | Locker::NoSTDIN,
+                   Locker::FreeLock);
+
+    // Get the interpreter dictionary.
+    auto dict =
+        PythonModule::MainModule().ResolveName<python::PythonDictionary>(
+            m_interpreter.GetDictionaryName());
+    if (!dict.IsAllocated())
+      return ErrorWithMessage<T>(
+          caller_signature,
+          llvm::formatv("could not find interpreter dictionary: {0}",
+                        m_interpreter.GetDictionaryName())
+              .str(),
+          error);
+
+    // Resolve the class.
+    auto class_obj =
+        PythonObject::ResolveNameWithDictionary<python::PythonCallable>(
+            class_name, dict);
+    if (!class_obj.IsAllocated())
+      return ErrorWithMessage<T>(
+          caller_signature,
+          llvm::formatv("could not find script class: {0}", class_name).str(),
+          error);
+
+    // Get the static method from the class.
+    if (!class_obj.HasAttribute(method_name))
+      return ErrorWithMessage<T>(
+          caller_signature,
+          llvm::formatv("class {0} does not have method {1}", class_name,
+                        method_name)
+              .str(),
+          error);
+
+    PythonCallable method =
+        class_obj.GetAttributeValue(method_name).AsType<PythonCallable>();
+    if (!method.IsAllocated())
+      return ErrorWithMessage<T>(caller_signature,
+                                 llvm::formatv("method {0}.{1} is not callable",
+                                               class_name, method_name)
+                                     .str(),
+                                 error);
+
+    // Transform the arguments.
+    std::tuple<Args...> original_args = std::forward_as_tuple(args...);
+    auto transformed_args = TransformArgs(original_args);
+
+    // Call the static method.
+    llvm::Expected<PythonObject> expected_return_object =
+        llvm::make_error<llvm::StringError>("Not initialized.",
+                                            llvm::inconvertibleErrorCode());
+    std::apply(
+        [&method, &expected_return_object](auto &&...args) {
+          llvm::consumeError(expected_return_object.takeError());
+          expected_return_object = method(args...);
+        },
+        transformed_args);
+
+    if (llvm::Error e = expected_return_object.takeError()) {
+      error = Status::FromError(std::move(e));
+      return ErrorWithMessage<T>(
+          caller_signature, "python static method could not be called", error);
+    }
+
+    PythonObject py_return = std::move(expected_return_object.get());
+
+    // Re-assign reference and pointer arguments if needed.
+    if (sizeof...(Args) > 0)
+      if (!ReassignPtrsOrRefsArgs(original_args, transformed_args))
+        return ErrorWithMessage<T>(
+            caller_signature,
+            "couldn't re-assign reference and pointer arguments", error);
+
+    // Extract value from Python object (handles unallocated case).
+    return ExtractValueFromPythonObject<T>(py_return, error);
+  }
+
 protected:
   template <typename T = StructuredData::ObjectSP>
   T ExtractValueFromPythonObject(python::PythonObject &p, Status &error) {
@@ -403,7 +509,7 @@ class ScriptedPythonInterface : virtual public ScriptedInterface {
                     llvm::Twine(method_name) + llvm::Twine(")"))
             .str();
     if (!m_object_instance_sp)
-      return ErrorWithMessage<T>(caller_signature, "Python object ill-formed",
+      return ErrorWithMessage<T>(caller_signature, "python object ill-formed",
                                  error);
 
     Locker py_lock(&m_interpreter, Locker::AcquireLock | Locker::NoSTDIN,
@@ -415,7 +521,7 @@ class ScriptedPythonInterface : virtual public ScriptedInterface {
     if (!implementor.IsAllocated())
       return llvm::is_contained(GetAbstractMethods(), method_name)
                  ? ErrorWithMessage<T>(caller_signature,
-                                       "Python implementor not allocated.",
+                                       "python implementor not allocated",
                                        error)
                  : T{};
 
@@ -436,20 +542,20 @@ class ScriptedPythonInterface : virtual public ScriptedInterface {
     if (llvm::Error e = expected_return_object.takeError()) {
       error = Status::FromError(std::move(e));
       return ErrorWithMessage<T>(caller_signature,
-                                 "Python method could not be called.", error);
+                                 "python method could not be called", error);
     }
 
     PythonObject py_return = std::move(expected_return_object.get());
 
     // Now that we called the python method with the transformed arguments,
-    // we need to interate again over both the original and transformed
+    // we need to iterate again over both the original and transformed
     // parameter pack, and transform back the parameter that were passed in
     // the original parameter pack as references or pointers.
     if (sizeof...(Args) > 0)
       if (!ReassignPtrsOrRefsArgs(original_args, transformed_args))
         return ErrorWithMessage<T>(
             caller_signature,
-            "Couldn't re-assign reference and pointer arguments.", error);
+            "couldn't re-assign reference and pointer arguments", error);
 
     if (!py_return.IsAllocated())
       return {};
@@ -655,6 +761,11 @@ lldb::StreamSP
 ScriptedPythonInterface::ExtractValueFromPythonObject<lldb::StreamSP>(
     python::PythonObject &p, Status &error);
 
+template <>
+lldb::ThreadSP
+ScriptedPythonInterface::ExtractValueFromPythonObject<lldb::ThreadSP>(
+    python::PythonObject &p, Status &error);
+
 template <>
 lldb::StackFrameSP
 ScriptedPythonInterface::ExtractValueFromPythonObject<lldb::StackFrameSP>(
diff --git a/lldb/source/Plugins/ScriptInterpreter/Python/SWIGPythonBridge.h b/lldb/source/Plugins/ScriptInterpreter/Python/SWIGPythonBridge.h
index 2c971262fc34e..32948ffd30023 100644
--- a/lldb/source/Plugins/ScriptInterpreter/Python/SWIGPythonBridge.h
+++ b/lldb/source/Plugins/ScriptInterpreter/Python/SWIGPythonBridge.h
@@ -265,6 +265,7 @@ void *LLDBSWIGPython_CastPyObjectToSBLaunchInfo(PyObject *data);
 void *LLDBSWIGPython_CastPyObjectToSBError(PyObject *data);
 void *LLDBSWIGPython_CastPyObjectToSBEvent(PyObject *data);
 void *LLDBSWIGPython_CastPyObjectToSBStream(PyObject *data);
+void *LLDBSWIGPython_CastPyObjectToSBThread(PyObject *data);
 void *LLDBSWIGPython_CastPyObjectToSBFrame(PyObject *data);
 void *LLDBSWIGPython_CastPyObjectToSBSymbolContext(PyObject *data);
 void *LLDBSWIGPython_CastPyObjectToSBValue(PyObject *data);
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.cpp b/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.cpp
index 36aa49ac3de95..7160c6eec564b 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.cpp
+++ b/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.cpp
@@ -3707,12 +3707,10 @@ bool DWARFASTParserClang::CopyUniqueClassMethodTypes(
     }
   }
 
-  DWARFASTParserClang *src_dwarf_ast_parser =
-      static_cast<DWARFASTParserClang *>(
-          SymbolFileDWARF::GetDWARFParser(*src_class_die.GetCU()));
-  DWARFASTParserClang *dst_dwarf_ast_parser =
-      static_cast<DWARFASTParserClang *>(
-          SymbolFileDWARF::GetDWARFParser(*dst_class_die.GetCU()));
+  auto *src_dwarf_ast_parser = llvm::cast<DWARFASTParserClang>(
+      SymbolFileDWARF::GetDWARFParser(*src_class_die.GetCU()));
+  auto *dst_dwarf_ast_parser = llvm::cast<DWARFASTParserClang>(
+      SymbolFileDWARF::GetDWARFParser(*dst_class_die.GetCU()));
   auto link = [&](DWARFDIE src, DWARFDIE dst) {
     auto &die_to_type = dst_class_die.GetDWARF()->GetDIEToType();
     clang::DeclContext *dst_decl_ctx =
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.h b/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.h
index f5f707129d67d..6eb2b6b48787b 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.h
+++ b/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.h
@@ -47,6 +47,11 @@ class DWARFASTParserClang : public lldb_private::plugin::dwarf::DWARFASTParser {
 
   ~DWARFASTParserClang() override;
 
+  // LLVM RTTI support
+  static bool classof(const DWARFASTParser *Parser) {
+    return Parser->GetKind() == Kind::DWARFASTParserClang;
+  }
+
   // DWARFASTParser interface.
   lldb::TypeSP
   ParseTypeFromDWARF(const lldb_private::SymbolContext &sc,
@@ -264,10 +269,6 @@ class DWARFASTParserClang : public lldb_private::plugin::dwarf::DWARFASTParser {
   lldb::ModuleSP
   GetModuleForType(const lldb_private::plugin::dwarf::DWARFDIE &die);
 
-  static bool classof(const DWARFASTParser *Parser) {
-    return Parser->GetKind() == Kind::DWARFASTParserClang;
-  }
-
 private:
   struct FieldInfo {
     /// Size in bits that this field occupies. Can but
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp b/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp
index ca8e74337733c..7ba765371c54f 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp
+++ b/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp
@@ -1560,8 +1560,8 @@ bool SymbolFileDWARF::HasForwardDeclForCompilerType(
   auto clang_type_system = compiler_type.GetTypeSystem<TypeSystemClang>();
   if (!clang_type_system)
     return false;
-  DWARFASTParserClang *ast_parser =
-      static_cast<DWARFASTParserClang *>(clang_type_system->GetDWARFParser());
+  auto *ast_parser =
+      llvm::cast<DWARFASTParserClang>(clang_type_system->GetDWARFParser());
   return ast_parser->GetClangASTImporter().CanImport(compiler_type);
 }
 
@@ -1569,8 +1569,8 @@ bool SymbolFileDWARF::CompleteType(CompilerType &compiler_type) {
   std::lock_guard<std::recursive_mutex> guard(GetModuleMutex());
   auto clang_type_system = compiler_type.GetTypeSystem<TypeSystemClang>();
   if (clang_type_system) {
-    DWARFASTParserClang *ast_parser =
-        static_cast<DWARFASTParserClang *>(clang_type_system->GetDWARFParser());
+    auto *ast_parser =
+        llvm::cast<DWARFASTParserClang>(clang_type_system->GetDWARFParser());
     if (ast_parser &&
         ast_parser->GetClangASTImporter().CanImport(compiler_type))
       return ast_parser->GetClangASTImporter().CompleteType(compiler_type);
@@ -1614,8 +1614,7 @@ bool SymbolFileDWARF::CompleteType(CompilerType &compiler_type) {
 
   if (decl_die != def_die) {
     GetDIEToType()[def_die.GetDIE()] = type;
-    DWARFASTParserClang *ast_parser =
-        static_cast<DWARFASTParserClang *>(dwarf_ast);
+    auto *ast_parser = llvm::cast<DWARFASTParserClang>(dwarf_ast);
     ast_parser->MapDeclDIEToDefDIE(decl_die, def_die);
   }
 
diff --git a/lldb/source/Plugins/SyntheticFrameProvider/CMakeLists.txt b/lldb/source/Plugins/SyntheticFrameProvider/CMakeLists.txt
new file mode 100644
index 0000000000000..85b405e648c1f
--- /dev/null
+++ b/lldb/source/Plugins/SyntheticFrameProvider/CMakeLists.txt
@@ -0,0 +1 @@
+add_subdirectory(ScriptedFrameProvider)
diff --git a/lldb/source/Plugins/SyntheticFrameProvider/ScriptedFrameProvider/CMakeLists.txt b/lldb/source/Plugins/SyntheticFrameProvider/ScriptedFrameProvider/CMakeLists.txt
new file mode 100644
index 0000000000000..fe67d39efdf11
--- /dev/null
+++ b/lldb/source/Plugins/SyntheticFrameProvider/ScriptedFrameProvider/CMakeLists.txt
@@ -0,0 +1,12 @@
+add_lldb_library(lldbPluginScriptedFrameProvider PLUGIN
+  ScriptedFrameProvider.cpp
+
+  LINK_COMPONENTS
+    Support
+
+  LINK_LIBS
+    lldbCore
+    lldbInterpreter
+    lldbTarget
+    lldbUtility
+  )
diff --git a/lldb/source/Plugins/SyntheticFrameProvider/ScriptedFrameProvider/ScriptedFrameProvider.cpp b/lldb/source/Plugins/SyntheticFrameProvider/ScriptedFrameProvider/ScriptedFrameProvider.cpp
new file mode 100644
index 0000000000000..739963e6f0c2f
--- /dev/null
+++ b/lldb/source/Plugins/SyntheticFrameProvider/ScriptedFrameProvider/ScriptedFrameProvider.cpp
@@ -0,0 +1,221 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "ScriptedFrameProvider.h"
+#include "Plugins/Process/scripted/ScriptedFrame.h"
+#include "lldb/Core/Debugger.h"
+#include "lldb/Core/PluginManager.h"
+#include "lldb/Interpreter/Interfaces/ScriptedFrameProviderInterface.h"
+#include "lldb/Interpreter/ScriptInterpreter.h"
+#include "lldb/Target/BorrowedStackFrame.h"
+#include "lldb/Target/Process.h"
+#include "lldb/Target/StackFrame.h"
+#include "lldb/Target/Thread.h"
+#include "lldb/Utility/ScriptedMetadata.h"
+#include "lldb/Utility/Status.h"
+#include "llvm/Support/Error.h"
+#include <cstdint>
+
+using namespace lldb;
+using namespace lldb_private;
+
+void ScriptedFrameProvider::Initialize() {
+  PluginManager::RegisterPlugin(GetPluginNameStatic(),
+                                "Provides synthetic frames via scripting",
+                                nullptr, ScriptedFrameProvider::CreateInstance);
+}
+
+void ScriptedFrameProvider::Terminate() {
+  PluginManager::UnregisterPlugin(ScriptedFrameProvider::CreateInstance);
+}
+
+llvm::Expected<lldb::SyntheticFrameProviderSP>
+ScriptedFrameProvider::CreateInstance(
+    lldb::StackFrameListSP input_frames,
+    const ScriptedFrameProviderDescriptor &descriptor) {
+  if (!input_frames)
+    return llvm::createStringError(
+        "failed to create scripted frame provider: invalid input frames");
+
+  Thread &thread = input_frames->GetThread();
+  ProcessSP process_sp = thread.GetProcess();
+  if (!process_sp)
+    return nullptr;
+
+  if (!descriptor.IsValid())
+    return llvm::createStringError(
+        "failed to create scripted frame provider: invalid scripted metadata");
+
+  if (!descriptor.AppliesToThread(thread))
+    return nullptr;
+
+  ScriptInterpreter *script_interp =
+      process_sp->GetTarget().GetDebugger().GetScriptInterpreter();
+  if (!script_interp)
+    return llvm::createStringError("cannot create scripted frame provider: No "
+                                   "script interpreter installed");
+
+  ScriptedFrameProviderInterfaceSP interface_sp =
+      script_interp->CreateScriptedFrameProviderInterface();
+  if (!interface_sp)
+    return llvm::createStringError(
+        "cannot create scripted frame provider: script interpreter couldn't "
+        "create Scripted Frame Provider Interface");
+
+  const ScriptedMetadataSP scripted_metadata = descriptor.scripted_metadata_sp;
+
+  // If we shouldn't attach a frame provider to this thread, just exit early.
+  if (!interface_sp->AppliesToThread(scripted_metadata->GetClassName(),
+                                     thread.shared_from_this()))
+    return nullptr;
+
+  auto obj_or_err = interface_sp->CreatePluginObject(
+      scripted_metadata->GetClassName(), input_frames,
+      scripted_metadata->GetArgsSP());
+  if (!obj_or_err)
+    return obj_or_err.takeError();
+
+  StructuredData::ObjectSP object_sp = *obj_or_err;
+  if (!object_sp || !object_sp->IsValid())
+    return llvm::createStringError(
+        "cannot create scripted frame provider: failed to create valid scripted"
+        "frame provider object");
+
+  return std::make_shared<ScriptedFrameProvider>(input_frames, interface_sp,
+                                                 descriptor);
+}
+
+ScriptedFrameProvider::ScriptedFrameProvider(
+    StackFrameListSP input_frames,
+    lldb::ScriptedFrameProviderInterfaceSP interface_sp,
+    const ScriptedFrameProviderDescriptor &descriptor)
+    : SyntheticFrameProvider(input_frames), m_interface_sp(interface_sp),
+      m_descriptor(descriptor) {}
+
+ScriptedFrameProvider::~ScriptedFrameProvider() = default;
+
+std::string ScriptedFrameProvider::GetDescription() const {
+  if (!m_interface_sp)
+    return {};
+
+  return m_interface_sp->GetDescription(m_descriptor.GetName());
+}
+
+llvm::Expected<StackFrameSP>
+ScriptedFrameProvider::GetFrameAtIndex(uint32_t idx) {
+  if (!m_interface_sp)
+    return llvm::createStringError(
+        "cannot get stack frame: scripted frame provider not initialized");
+
+  auto create_frame_from_dict =
+      [this](StructuredData::Dictionary *dict,
+             uint32_t index) -> llvm::Expected<StackFrameSP> {
+    lldb::addr_t pc;
+    if (!dict->GetValueForKeyAsInteger("pc", pc))
+      return llvm::createStringError(
+          "missing 'pc' key from scripted frame dictionary");
+
+    Address symbol_addr;
+    symbol_addr.SetLoadAddress(pc, &GetThread().GetProcess()->GetTarget());
+
+    const lldb::addr_t cfa = LLDB_INVALID_ADDRESS;
+    const bool cfa_is_valid = false;
+    const bool artificial = false;
+    const bool behaves_like_zeroth_frame = false;
+    SymbolContext sc;
+    symbol_addr.CalculateSymbolContext(&sc);
+
+    ThreadSP thread_sp = GetThread().shared_from_this();
+    return std::make_shared<StackFrame>(thread_sp, index, index, cfa,
+                                        cfa_is_valid, pc,
+                                        StackFrame::Kind::Synthetic, artificial,
+                                        behaves_like_zeroth_frame, &sc);
+  };
+
+  auto create_frame_from_script_object =
+      [this](
+          StructuredData::ObjectSP object_sp) -> llvm::Expected<StackFrameSP> {
+    Status error;
+    if (!object_sp || !object_sp->GetAsGeneric())
+      return llvm::createStringError("invalid script object");
+
+    ThreadSP thread_sp = GetThread().shared_from_this();
+    auto frame_or_error = ScriptedFrame::Create(thread_sp, nullptr, nullptr,
+                                                object_sp->GetAsGeneric());
+
+    if (!frame_or_error) {
+      ScriptedInterface::ErrorWithMessage<bool>(
+          LLVM_PRETTY_FUNCTION, toString(frame_or_error.takeError()), error);
+      return error.ToError();
+    }
+
+    return *frame_or_error;
+  };
+
+  StructuredData::ObjectSP obj_sp = m_interface_sp->GetFrameAtIndex(idx);
+
+  // None/null means no more frames or error.
+  if (!obj_sp || !obj_sp->IsValid())
+    return llvm::createStringError("invalid script object returned for frame " +
+                                   llvm::Twine(idx));
+
+  StackFrameSP synth_frame_sp = nullptr;
+  if (StructuredData::UnsignedInteger *int_obj =
+          obj_sp->GetAsUnsignedInteger()) {
+    uint32_t real_frame_index = int_obj->GetValue();
+    if (real_frame_index < m_input_frames->GetNumFrames()) {
+      StackFrameSP real_frame_sp =
+          m_input_frames->GetFrameAtIndex(real_frame_index);
+      synth_frame_sp =
+          (real_frame_index == idx)
+              ? real_frame_sp
+              : std::make_shared<BorrowedStackFrame>(real_frame_sp, idx);
+    }
+  } else if (StructuredData::Dictionary *dict = obj_sp->GetAsDictionary()) {
+    // Check if it's a dictionary describing a frame.
+    auto frame_from_dict_or_err = create_frame_from_dict(dict, idx);
+    if (!frame_from_dict_or_err) {
+      return llvm::createStringError(llvm::Twine(
+          "couldn't create frame from dictionary at index " + llvm::Twine(idx) +
+          ": " + toString(frame_from_dict_or_err.takeError())));
+    }
+    synth_frame_sp = *frame_from_dict_or_err;
+  } else if (obj_sp->GetAsGeneric()) {
+    // It's a ScriptedFrame object.
+    auto frame_from_script_obj_or_err = create_frame_from_script_object(obj_sp);
+    if (!frame_from_script_obj_or_err) {
+      return llvm::createStringError(
+          llvm::Twine("couldn't create frame from script object at index " +
+                      llvm::Twine(idx) + ": " +
+                      toString(frame_from_script_obj_or_err.takeError())));
+    }
+    synth_frame_sp = *frame_from_script_obj_or_err;
+  } else {
+    return llvm::createStringError(
+        llvm::Twine("invalid return type from get_frame_at_index at index " +
+                    llvm::Twine(idx)));
+  }
+
+  if (!synth_frame_sp)
+    return llvm::createStringError(
+        llvm::Twine("failed to create frame at index " + llvm::Twine(idx)));
+
+  synth_frame_sp->SetFrameIndex(idx);
+
+  return synth_frame_sp;
+}
+
+namespace lldb_private {
+void lldb_initialize_ScriptedFrameProvider() {
+  ScriptedFrameProvider::Initialize();
+}
+
+void lldb_terminate_ScriptedFrameProvider() {
+  ScriptedFrameProvider::Terminate();
+}
+} // namespace lldb_private
diff --git a/lldb/source/Plugins/SyntheticFrameProvider/ScriptedFrameProvider/ScriptedFrameProvider.h b/lldb/source/Plugins/SyntheticFrameProvider/ScriptedFrameProvider/ScriptedFrameProvider.h
new file mode 100644
index 0000000000000..3434bf26ade24
--- /dev/null
+++ b/lldb/source/Plugins/SyntheticFrameProvider/ScriptedFrameProvider/ScriptedFrameProvider.h
@@ -0,0 +1,53 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLDB_PLUGINS_SYNTHETICFRAMEPROVIDER_SCRIPTEDFRAMEPROVIDER_SCRIPTEDFRAMEPROVIDER_H
+#define LLDB_PLUGINS_SYNTHETICFRAMEPROVIDER_SCRIPTEDFRAMEPROVIDER_SCRIPTEDFRAMEPROVIDER_H
+
+#include "lldb/Target/SyntheticFrameProvider.h"
+#include "lldb/Utility/ScriptedMetadata.h"
+#include "lldb/Utility/Status.h"
+#include "lldb/lldb-forward.h"
+#include "llvm/Support/Error.h"
+
+namespace lldb_private {
+
+class ScriptedFrameProvider : public SyntheticFrameProvider {
+public:
+  static llvm::StringRef GetPluginNameStatic() {
+    return "ScriptedFrameProvider";
+  }
+
+  static llvm::Expected<lldb::SyntheticFrameProviderSP>
+  CreateInstance(lldb::StackFrameListSP input_frames,
+                 const ScriptedFrameProviderDescriptor &descriptor);
+
+  static void Initialize();
+
+  static void Terminate();
+
+  ScriptedFrameProvider(lldb::StackFrameListSP input_frames,
+                        lldb::ScriptedFrameProviderInterfaceSP interface_sp,
+                        const ScriptedFrameProviderDescriptor &descriptor);
+  ~ScriptedFrameProvider() override;
+
+  llvm::StringRef GetPluginName() override { return GetPluginNameStatic(); }
+
+  std::string GetDescription() const override;
+
+  /// Get a single stack frame at the specified index.
+  llvm::Expected<lldb::StackFrameSP> GetFrameAtIndex(uint32_t idx) override;
+
+private:
+  lldb::ScriptedFrameProviderInterfaceSP m_interface_sp;
+  const ScriptedFrameProviderDescriptor &m_descriptor;
+};
+
+} // namespace lldb_private
+
+#endif // LLDB_PLUGINS_SYNTHETICFRAMEPROVIDER_SCRIPTEDFRAMEPROVIDER_SCRIPTEDFRAMEPROVIDER_H
diff --git a/lldb/source/Target/BorrowedStackFrame.cpp b/lldb/source/Target/BorrowedStackFrame.cpp
new file mode 100644
index 0000000000000..5afadf21fde03
--- /dev/null
+++ b/lldb/source/Target/BorrowedStackFrame.cpp
@@ -0,0 +1,187 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "lldb/Target/BorrowedStackFrame.h"
+
+using namespace lldb;
+using namespace lldb_private;
+
+char BorrowedStackFrame::ID;
+
+BorrowedStackFrame::BorrowedStackFrame(
+    StackFrameSP borrowed_frame_sp, uint32_t new_frame_index,
+    std::optional<uint32_t> new_concrete_frame_index)
+    : StackFrame(
+          borrowed_frame_sp->GetThread(), new_frame_index,
+          borrowed_frame_sp->GetConcreteFrameIndex(),
+          borrowed_frame_sp->GetRegisterContextSP(),
+          borrowed_frame_sp->GetStackID().GetPC(),
+          borrowed_frame_sp->GetStackID().GetCallFrameAddressWithoutMetadata(),
+          borrowed_frame_sp->m_behaves_like_zeroth_frame,
+          &borrowed_frame_sp->GetSymbolContext(eSymbolContextEverything)),
+      m_borrowed_frame_sp(borrowed_frame_sp),
+      m_new_frame_index(new_frame_index) {
+  if (new_concrete_frame_index)
+    m_new_concrete_frame_index = *new_concrete_frame_index;
+  else
+    m_new_concrete_frame_index =
+        IsInlined() ? LLDB_INVALID_FRAME_ID : new_frame_index;
+}
+
+uint32_t BorrowedStackFrame::GetFrameIndex() const { return m_new_frame_index; }
+
+void BorrowedStackFrame::SetFrameIndex(uint32_t index) {
+  m_new_frame_index = index;
+}
+
+uint32_t BorrowedStackFrame::GetConcreteFrameIndex() {
+  // FIXME: We need to find where the concrete frame into which this frame was
+  // inlined landed in the new stack frame list as that is the correct concrete
+  // frame index in this
+  // stack frame.
+  return m_new_concrete_frame_index;
+}
+
+StackID &BorrowedStackFrame::GetStackID() {
+  return m_borrowed_frame_sp->GetStackID();
+}
+
+const Address &BorrowedStackFrame::GetFrameCodeAddress() {
+  return m_borrowed_frame_sp->GetFrameCodeAddress();
+}
+
+Address BorrowedStackFrame::GetFrameCodeAddressForSymbolication() {
+  return m_borrowed_frame_sp->GetFrameCodeAddressForSymbolication();
+}
+
+bool BorrowedStackFrame::ChangePC(addr_t pc) {
+  return m_borrowed_frame_sp->ChangePC(pc);
+}
+
+const SymbolContext &
+BorrowedStackFrame::GetSymbolContext(SymbolContextItem resolve_scope) {
+  return m_borrowed_frame_sp->GetSymbolContext(resolve_scope);
+}
+
+llvm::Error BorrowedStackFrame::GetFrameBaseValue(Scalar &value) {
+  return m_borrowed_frame_sp->GetFrameBaseValue(value);
+}
+
+DWARFExpressionList *
+BorrowedStackFrame::GetFrameBaseExpression(Status *error_ptr) {
+  return m_borrowed_frame_sp->GetFrameBaseExpression(error_ptr);
+}
+
+Block *BorrowedStackFrame::GetFrameBlock() {
+  return m_borrowed_frame_sp->GetFrameBlock();
+}
+
+RegisterContextSP BorrowedStackFrame::GetRegisterContext() {
+  return m_borrowed_frame_sp->GetRegisterContext();
+}
+
+VariableList *BorrowedStackFrame::GetVariableList(bool get_file_globals,
+                                                  Status *error_ptr) {
+  return m_borrowed_frame_sp->GetVariableList(get_file_globals, error_ptr);
+}
+
+VariableListSP
+BorrowedStackFrame::GetInScopeVariableList(bool get_file_globals,
+                                           bool must_have_valid_location) {
+  return m_borrowed_frame_sp->GetInScopeVariableList(get_file_globals,
+                                                     must_have_valid_location);
+}
+
+ValueObjectSP BorrowedStackFrame::GetValueForVariableExpressionPath(
+    llvm::StringRef var_expr, DynamicValueType use_dynamic, uint32_t options,
+    VariableSP &var_sp, Status &error) {
+  return m_borrowed_frame_sp->GetValueForVariableExpressionPath(
+      var_expr, use_dynamic, options, var_sp, error);
+}
+
+bool BorrowedStackFrame::HasDebugInformation() {
+  return m_borrowed_frame_sp->HasDebugInformation();
+}
+
+const char *BorrowedStackFrame::Disassemble() {
+  return m_borrowed_frame_sp->Disassemble();
+}
+
+ValueObjectSP BorrowedStackFrame::GetValueObjectForFrameVariable(
+    const VariableSP &variable_sp, DynamicValueType use_dynamic) {
+  return m_borrowed_frame_sp->GetValueObjectForFrameVariable(variable_sp,
+                                                             use_dynamic);
+}
+
+bool BorrowedStackFrame::IsInlined() {
+  return m_borrowed_frame_sp->IsInlined();
+}
+
+bool BorrowedStackFrame::IsSynthetic() const {
+  return m_borrowed_frame_sp->IsSynthetic();
+}
+
+bool BorrowedStackFrame::IsHistorical() const {
+  return m_borrowed_frame_sp->IsHistorical();
+}
+
+bool BorrowedStackFrame::IsArtificial() const {
+  return m_borrowed_frame_sp->IsArtificial();
+}
+
+bool BorrowedStackFrame::IsHidden() { return m_borrowed_frame_sp->IsHidden(); }
+
+const char *BorrowedStackFrame::GetFunctionName() {
+  return m_borrowed_frame_sp->GetFunctionName();
+}
+
+const char *BorrowedStackFrame::GetDisplayFunctionName() {
+  return m_borrowed_frame_sp->GetDisplayFunctionName();
+}
+
+ValueObjectSP BorrowedStackFrame::FindVariable(ConstString name) {
+  return m_borrowed_frame_sp->FindVariable(name);
+}
+
+SourceLanguage BorrowedStackFrame::GetLanguage() {
+  return m_borrowed_frame_sp->GetLanguage();
+}
+
+SourceLanguage BorrowedStackFrame::GuessLanguage() {
+  return m_borrowed_frame_sp->GuessLanguage();
+}
+
+ValueObjectSP BorrowedStackFrame::GuessValueForAddress(addr_t addr) {
+  return m_borrowed_frame_sp->GuessValueForAddress(addr);
+}
+
+ValueObjectSP
+BorrowedStackFrame::GuessValueForRegisterAndOffset(ConstString reg,
+                                                   int64_t offset) {
+  return m_borrowed_frame_sp->GuessValueForRegisterAndOffset(reg, offset);
+}
+
+StructuredData::ObjectSP BorrowedStackFrame::GetLanguageSpecificData() {
+  return m_borrowed_frame_sp->GetLanguageSpecificData();
+}
+
+RecognizedStackFrameSP BorrowedStackFrame::GetRecognizedFrame() {
+  return m_borrowed_frame_sp->GetRecognizedFrame();
+}
+
+StackFrameSP BorrowedStackFrame::GetBorrowedFrame() const {
+  return m_borrowed_frame_sp;
+}
+
+bool BorrowedStackFrame::isA(const void *ClassID) const {
+  return ClassID == &ID || StackFrame::isA(ClassID);
+}
+
+bool BorrowedStackFrame::classof(const StackFrame *obj) {
+  return obj->isA(&ID);
+}
diff --git a/lldb/source/Target/CMakeLists.txt b/lldb/source/Target/CMakeLists.txt
index cff59049cdce5..df2ee03860ac0 100644
--- a/lldb/source/Target/CMakeLists.txt
+++ b/lldb/source/Target/CMakeLists.txt
@@ -41,6 +41,7 @@ add_lldb_library(lldbTarget
   SyntheticFrameProvider.cpp
   SectionLoadHistory.cpp
   SectionLoadList.cpp
+  BorrowedStackFrame.cpp
   StackFrame.cpp
   StackFrameList.cpp
   StackFrameRecognizer.cpp
diff --git a/lldb/source/Target/StackFrame.cpp b/lldb/source/Target/StackFrame.cpp
index 78f67d21d6600..ca3d4a1a29b59 100644
--- a/lldb/source/Target/StackFrame.cpp
+++ b/lldb/source/Target/StackFrame.cpp
@@ -45,6 +45,9 @@
 using namespace lldb;
 using namespace lldb_private;
 
+// LLVM RTTI support.
+char StackFrame::ID;
+
 // The first bits in the flags are reserved for the SymbolContext::Scope bits
 // so we know if we have tried to look up information in our internal symbol
 // context (m_sc) already.
diff --git a/lldb/source/Target/StackFrameList.cpp b/lldb/source/Target/StackFrameList.cpp
index 8412e33aaba32..5d1a8a8370414 100644
--- a/lldb/source/Target/StackFrameList.cpp
+++ b/lldb/source/Target/StackFrameList.cpp
@@ -20,6 +20,7 @@
 #include "lldb/Target/StackFrame.h"
 #include "lldb/Target/StackFrameRecognizer.h"
 #include "lldb/Target/StopInfo.h"
+#include "lldb/Target/SyntheticFrameProvider.h"
 #include "lldb/Target/Target.h"
 #include "lldb/Target/Thread.h"
 #include "lldb/Target/Unwind.h"
@@ -55,6 +56,44 @@ StackFrameList::~StackFrameList() {
   Clear();
 }
 
+SyntheticStackFrameList::SyntheticStackFrameList(
+    Thread &thread, lldb::StackFrameListSP input_frames,
+    const lldb::StackFrameListSP &prev_frames_sp, bool show_inline_frames)
+    : StackFrameList(thread, prev_frames_sp, show_inline_frames),
+      m_input_frames(std::move(input_frames)) {}
+
+bool SyntheticStackFrameList::FetchFramesUpTo(
+    uint32_t end_idx, InterruptionControl allow_interrupt) {
+  // Check if the thread has a synthetic frame provider.
+  if (auto provider_sp = m_thread.GetFrameProvider()) {
+    // Use the synthetic frame provider to generate frames lazily.
+    // Keep fetching until we reach end_idx or the provider returns an error.
+    for (uint32_t idx = m_frames.size(); idx <= end_idx; idx++) {
+      if (allow_interrupt &&
+          m_thread.GetProcess()->GetTarget().GetDebugger().InterruptRequested())
+        return true;
+      auto frame_or_err = provider_sp->GetFrameAtIndex(idx);
+      if (!frame_or_err) {
+        // Provider returned error - we've reached the end.
+        LLDB_LOG_ERROR(GetLog(LLDBLog::Thread), frame_or_err.takeError(),
+                       "Frame provider reached end at index {0}: {1}", idx);
+        SetAllFramesFetched();
+        break;
+      }
+      StackFrameSP frame_sp = *frame_or_err;
+      // Set the frame list weak pointer so ExecutionContextRef can resolve
+      // the frame without calling Thread::GetStackFrameList().
+      frame_sp->m_frame_list_wp = shared_from_this();
+      m_frames.push_back(frame_sp);
+    }
+
+    return false; // Not interrupted.
+  }
+
+  // If no provider, fall back to the base implementation.
+  return StackFrameList::FetchFramesUpTo(end_idx, allow_interrupt);
+}
+
 void StackFrameList::CalculateCurrentInlinedDepth() {
   uint32_t cur_inlined_depth = GetCurrentInlinedDepth();
   if (cur_inlined_depth == UINT32_MAX) {
diff --git a/lldb/source/Target/SyntheticFrameProvider.cpp b/lldb/source/Target/SyntheticFrameProvider.cpp
index 241ce82c39be3..97ff42d1ed53e 100644
--- a/lldb/source/Target/SyntheticFrameProvider.cpp
+++ b/lldb/source/Target/SyntheticFrameProvider.cpp
@@ -8,10 +8,12 @@
 
 #include "lldb/Target/SyntheticFrameProvider.h"
 #include "lldb/Core/PluginManager.h"
+#include "lldb/Interpreter/Interfaces/ScriptedFrameProviderInterface.h"
 #include "lldb/Target/Thread.h"
 #include "lldb/Utility/LLDBLog.h"
 #include "lldb/Utility/Log.h"
 #include "lldb/Utility/Status.h"
+#include "lldb/Utility/Stream.h"
 
 using namespace lldb;
 using namespace lldb_private;
@@ -21,12 +23,17 @@ SyntheticFrameProvider::SyntheticFrameProvider(StackFrameListSP input_frames)
 
 SyntheticFrameProvider::~SyntheticFrameProvider() = default;
 
-void SyntheticFrameProviderDescriptor::Dump(Stream *s) const {
+void ScriptedFrameProviderDescriptor::Dump(Stream *s) const {
   if (!s)
     return;
 
+  s->Format("  ID: {0:x}\n", GetID());
   s->Printf("  Name: %s\n", GetName().str().c_str());
 
+  std::string description = GetDescription();
+  if (!description.empty())
+    s->Printf("  Description: %s\n", description.c_str());
+
   // Show thread filter information.
   if (thread_specs.empty()) {
     s->PutCString("  Thread Filter: (applies to all threads)\n");
@@ -41,9 +48,23 @@ void SyntheticFrameProviderDescriptor::Dump(Stream *s) const {
   }
 }
 
+uint32_t ScriptedFrameProviderDescriptor::GetID() const {
+  if (!scripted_metadata_sp)
+    return 0;
+
+  return scripted_metadata_sp->GetID();
+}
+
+std::string ScriptedFrameProviderDescriptor::GetDescription() const {
+  // If we have an interface, call get_description() to fetch it.
+  if (interface_sp && scripted_metadata_sp)
+    return interface_sp->GetDescription(scripted_metadata_sp->GetClassName());
+  return {};
+}
+
 llvm::Expected<SyntheticFrameProviderSP> SyntheticFrameProvider::CreateInstance(
     StackFrameListSP input_frames,
-    const SyntheticFrameProviderDescriptor &descriptor) {
+    const ScriptedFrameProviderDescriptor &descriptor) {
   if (!input_frames)
     return llvm::createStringError(
         "cannot create synthetic frame provider: invalid input frames");
diff --git a/lldb/source/Target/Target.cpp b/lldb/source/Target/Target.cpp
index 3a936b85f6339..b6a662ad3f14d 100644
--- a/lldb/source/Target/Target.cpp
+++ b/lldb/source/Target/Target.cpp
@@ -3720,6 +3720,61 @@ Status Target::Attach(ProcessAttachInfo &attach_info, Stream *stream) {
   return error;
 }
 
+llvm::Expected<uint32_t> Target::AddScriptedFrameProviderDescriptor(
+    const ScriptedFrameProviderDescriptor &descriptor) {
+  if (!descriptor.IsValid())
+    return llvm::createStringError("invalid frame provider descriptor");
+
+  llvm::StringRef name = descriptor.GetName();
+  if (name.empty())
+    return llvm::createStringError(
+        "frame provider descriptor has no class name");
+
+  std::lock_guard<std::recursive_mutex> guard(
+      m_frame_provider_descriptors_mutex);
+
+  uint32_t descriptor_id = descriptor.GetID();
+  m_frame_provider_descriptors[descriptor_id] = descriptor;
+
+  // Clear frame providers on existing threads so they reload with new config.
+  if (ProcessSP process_sp = GetProcessSP())
+    for (ThreadSP thread_sp : process_sp->Threads())
+      thread_sp->ClearScriptedFrameProvider();
+
+  return descriptor_id;
+}
+
+bool Target::RemoveScriptedFrameProviderDescriptor(uint32_t id) {
+  std::lock_guard<std::recursive_mutex> guard(
+      m_frame_provider_descriptors_mutex);
+  bool removed = m_frame_provider_descriptors.erase(id);
+
+  if (removed)
+    if (ProcessSP process_sp = GetProcessSP())
+      for (ThreadSP thread_sp : process_sp->Threads())
+        thread_sp->ClearScriptedFrameProvider();
+
+  return removed;
+}
+
+void Target::ClearScriptedFrameProviderDescriptors() {
+  std::lock_guard<std::recursive_mutex> guard(
+      m_frame_provider_descriptors_mutex);
+
+  m_frame_provider_descriptors.clear();
+
+  if (ProcessSP process_sp = GetProcessSP())
+    for (ThreadSP thread_sp : process_sp->Threads())
+      thread_sp->ClearScriptedFrameProvider();
+}
+
+const llvm::DenseMap<uint32_t, ScriptedFrameProviderDescriptor> &
+Target::GetScriptedFrameProviderDescriptors() const {
+  std::lock_guard<std::recursive_mutex> guard(
+      m_frame_provider_descriptors_mutex);
+  return m_frame_provider_descriptors;
+}
+
 void Target::FinalizeFileActions(ProcessLaunchInfo &info) {
   Log *log = GetLog(LLDBLog::Process);
 
diff --git a/lldb/source/Target/Thread.cpp b/lldb/source/Target/Thread.cpp
index 8c3e19725f8cb..b40e753aca1e9 100644
--- a/lldb/source/Target/Thread.cpp
+++ b/lldb/source/Target/Thread.cpp
@@ -13,9 +13,12 @@
 #include "lldb/Core/Module.h"
 #include "lldb/Core/StructuredDataImpl.h"
 #include "lldb/Host/Host.h"
+#include "lldb/Interpreter/Interfaces/ScriptedFrameInterface.h"
+#include "lldb/Interpreter/Interfaces/ScriptedFrameProviderInterface.h"
 #include "lldb/Interpreter/OptionValueFileSpecList.h"
 #include "lldb/Interpreter/OptionValueProperties.h"
 #include "lldb/Interpreter/Property.h"
+#include "lldb/Interpreter/ScriptInterpreter.h"
 #include "lldb/Symbol/Function.h"
 #include "lldb/Target/ABI.h"
 #include "lldb/Target/DynamicLoader.h"
@@ -26,6 +29,7 @@
 #include "lldb/Target/ScriptedThreadPlan.h"
 #include "lldb/Target/StackFrameRecognizer.h"
 #include "lldb/Target/StopInfo.h"
+#include "lldb/Target/SyntheticFrameProvider.h"
 #include "lldb/Target/SystemRuntime.h"
 #include "lldb/Target/Target.h"
 #include "lldb/Target/ThreadPlan.h"
@@ -45,6 +49,7 @@
 #include "lldb/Utility/LLDBLog.h"
 #include "lldb/Utility/Log.h"
 #include "lldb/Utility/RegularExpression.h"
+#include "lldb/Utility/ScriptedMetadata.h"
 #include "lldb/Utility/State.h"
 #include "lldb/Utility/Stream.h"
 #include "lldb/Utility/StreamString.h"
@@ -257,6 +262,7 @@ void Thread::DestroyThread() {
   std::lock_guard<std::recursive_mutex> guard(m_frame_mutex);
   m_curr_frames_sp.reset();
   m_prev_frames_sp.reset();
+  m_frame_provider_sp.reset();
   m_prev_framezero_pc.reset();
 }
 
@@ -1439,13 +1445,76 @@ void Thread::CalculateExecutionContext(ExecutionContext &exe_ctx) {
 StackFrameListSP Thread::GetStackFrameList() {
   std::lock_guard<std::recursive_mutex> guard(m_frame_mutex);
 
-  if (!m_curr_frames_sp)
+  if (m_curr_frames_sp)
+    return m_curr_frames_sp;
+
+  // First, try to load a frame provider if we don't have one yet.
+  if (!m_frame_provider_sp) {
+    ProcessSP process_sp = GetProcess();
+    if (process_sp) {
+      Target &target = process_sp->GetTarget();
+      const auto &descriptors = target.GetScriptedFrameProviderDescriptors();
+
+      // Find first descriptor that applies to this thread.
+      for (const auto &entry : descriptors) {
+        const ScriptedFrameProviderDescriptor &descriptor = entry.second;
+        if (descriptor.IsValid() && descriptor.AppliesToThread(*this)) {
+          if (llvm::Error error = LoadScriptedFrameProvider(descriptor)) {
+            LLDB_LOG_ERROR(GetLog(LLDBLog::Thread), std::move(error),
+                           "Failed to load scripted frame provider: {0}");
+          }
+          break; // Use first matching descriptor (success or failure).
+        }
+      }
+    }
+  }
+
+  // Create the frame list based on whether we have a provider.
+  if (m_frame_provider_sp) {
+    // We have a provider - create synthetic frame list.
+    StackFrameListSP input_frames = m_frame_provider_sp->GetInputFrames();
+    m_curr_frames_sp = std::make_shared<SyntheticStackFrameList>(
+        *this, input_frames, m_prev_frames_sp, true);
+  } else {
+    // No provider - use normal unwinder frames.
     m_curr_frames_sp =
         std::make_shared<StackFrameList>(*this, m_prev_frames_sp, true);
+  }
 
   return m_curr_frames_sp;
 }
 
+llvm::Error Thread::LoadScriptedFrameProvider(
+    const ScriptedFrameProviderDescriptor &descriptor) {
+  std::lock_guard<std::recursive_mutex> guard(m_frame_mutex);
+
+  // Note: We don't create input_frames here - it will be created lazily
+  // by SyntheticStackFrameList when frames are first fetched.
+  // Creating them too early can cause crashes during thread initialization.
+
+  // Create a temporary StackFrameList just to get the thread reference for the
+  // provider. The provider won't actually use this - it will get real input
+  // frames from SyntheticStackFrameList later.
+  StackFrameListSP temp_frames =
+      std::make_shared<StackFrameList>(*this, m_prev_frames_sp, true);
+
+  auto provider_or_err =
+      SyntheticFrameProvider::CreateInstance(temp_frames, descriptor);
+  if (!provider_or_err)
+    return provider_or_err.takeError();
+
+  ClearScriptedFrameProvider();
+  m_frame_provider_sp = *provider_or_err;
+  return llvm::Error::success();
+}
+
+void Thread::ClearScriptedFrameProvider() {
+  std::lock_guard<std::recursive_mutex> guard(m_frame_mutex);
+  m_frame_provider_sp.reset();
+  m_curr_frames_sp.reset();
+  m_prev_frames_sp.reset();
+}
+
 std::optional<addr_t> Thread::GetPreviousFrameZeroPC() {
   return m_prev_framezero_pc;
 }
@@ -1466,6 +1535,7 @@ void Thread::ClearStackFrames() {
     m_prev_frames_sp.swap(m_curr_frames_sp);
   m_curr_frames_sp.reset();
 
+  m_frame_provider_sp.reset();
   m_extended_info.reset();
   m_extended_info_fetched = false;
 }
diff --git a/lldb/source/Target/ThreadSpec.cpp b/lldb/source/Target/ThreadSpec.cpp
index ba4c3aa894553..624f64e3af800 100644
--- a/lldb/source/Target/ThreadSpec.cpp
+++ b/lldb/source/Target/ThreadSpec.cpp
@@ -19,6 +19,10 @@ const char *ThreadSpec::g_option_names[static_cast<uint32_t>(
 
 ThreadSpec::ThreadSpec() : m_name(), m_queue_name() {}
 
+ThreadSpec::ThreadSpec(Thread &thread)
+    : m_index(thread.GetIndexID()), m_tid(thread.GetID()),
+      m_name(thread.GetName()), m_queue_name(thread.GetQueueName()) {}
+
 std::unique_ptr<ThreadSpec> ThreadSpec::CreateFromStructuredData(
     const StructuredData::Dictionary &spec_dict, Status &error) {
   uint32_t index = UINT32_MAX;
diff --git a/lldb/source/ValueObject/ValueObjectSynthetic.cpp b/lldb/source/ValueObject/ValueObjectSynthetic.cpp
index f673c51a88412..44e53bd5fd82e 100644
--- a/lldb/source/ValueObject/ValueObjectSynthetic.cpp
+++ b/lldb/source/ValueObject/ValueObjectSynthetic.cpp
@@ -443,3 +443,18 @@ void ValueObjectSynthetic::SetLanguageFlags(uint64_t flags) {
   else
     this->ValueObject::SetLanguageFlags(flags);
 }
+
+void ValueObjectSynthetic::GetExpressionPath(Stream &stream,
+                                             GetExpressionPathFormat epformat) {
+  // A synthetic ValueObject may wrap an underlying  Register or RegisterSet
+  // ValueObject, which requires a different approach to generating the
+  // expression path. In such cases, delegate to the non-synthetic value object.
+  if (const lldb::ValueType obj_value_type = GetValueType();
+      IsSynthetic() && (obj_value_type == lldb::eValueTypeRegister ||
+                        obj_value_type == lldb::eValueTypeRegisterSet)) {
+
+    if (const lldb::ValueObjectSP raw_value = GetNonSyntheticValue())
+      return raw_value->GetExpressionPath(stream, epformat);
+  }
+  return ValueObject::GetExpressionPath(stream, epformat);
+}
diff --git a/lldb/test/API/functionalities/scripted_frame_provider/Makefile b/lldb/test/API/functionalities/scripted_frame_provider/Makefile
new file mode 100644
index 0000000000000..99998b20bcb05
--- /dev/null
+++ b/lldb/test/API/functionalities/scripted_frame_provider/Makefile
@@ -0,0 +1,3 @@
+CXX_SOURCES := main.cpp
+
+include Makefile.rules
diff --git a/lldb/test/API/functionalities/scripted_frame_provider/TestScriptedFrameProvider.py b/lldb/test/API/functionalities/scripted_frame_provider/TestScriptedFrameProvider.py
new file mode 100644
index 0000000000000..1ae0a17814e89
--- /dev/null
+++ b/lldb/test/API/functionalities/scripted_frame_provider/TestScriptedFrameProvider.py
@@ -0,0 +1,427 @@
+"""
+Test scripted frame provider functionality.
+"""
+
+import os
+
+import lldb
+import lldbsuite.test.lldbplatformutil as lldbplatformutil
+from lldbsuite.test.lldbtest import TestBase
+from lldbsuite.test import lldbutil
+
+
+class ScriptedFrameProviderTestCase(TestBase):
+    NO_DEBUG_INFO_TESTCASE = True
+
+    def setUp(self):
+        TestBase.setUp(self)
+        self.source = "main.cpp"
+
+    def test_replace_all_frames(self):
+        """Test that we can replace the entire stack."""
+        self.build()
+        target, process, thread, bkpt = lldbutil.run_to_source_breakpoint(
+            self, "Break here", lldb.SBFileSpec(self.source), only_one_thread=False
+        )
+
+        # Import the test frame provider
+        script_path = os.path.join(self.getSourceDir(), "test_frame_providers.py")
+        self.runCmd("command script import " + script_path)
+
+        # Attach the Replace provider
+        error = lldb.SBError()
+        provider_id = target.RegisterScriptedFrameProvider(
+            "test_frame_providers.ReplaceFrameProvider",
+            lldb.SBStructuredData(),
+            error,
+        )
+        self.assertTrue(error.Success(), f"Failed to register provider: {error}")
+        self.assertNotEqual(provider_id, 0, "Provider ID should be non-zero")
+
+        # Verify we have exactly 3 synthetic frames
+        self.assertEqual(thread.GetNumFrames(), 3, "Should have 3 synthetic frames")
+
+        # Verify frame indices and PCs (dictionary-based frames don't have custom function names)
+        frame0 = thread.GetFrameAtIndex(0)
+        self.assertIsNotNone(frame0)
+        self.assertEqual(frame0.GetPC(), 0x1000)
+
+        frame1 = thread.GetFrameAtIndex(1)
+        self.assertIsNotNone(frame1)
+        self.assertIn("thread_func", frame1.GetFunctionName())
+
+        frame2 = thread.GetFrameAtIndex(2)
+        self.assertIsNotNone(frame2)
+        self.assertEqual(frame2.GetPC(), 0x3000)
+
+    def test_prepend_frames(self):
+        """Test that we can add frames before real stack."""
+        self.build()
+        target, process, thread, bkpt = lldbutil.run_to_source_breakpoint(
+            self, "Break here", lldb.SBFileSpec(self.source), only_one_thread=False
+        )
+
+        # Get original frame count and PC
+        original_frame_count = thread.GetNumFrames()
+        self.assertGreaterEqual(
+            original_frame_count, 2, "Should have at least 2 real frames"
+        )
+
+        # Import and attach Prepend provider
+        script_path = os.path.join(self.getSourceDir(), "test_frame_providers.py")
+        self.runCmd("command script import " + script_path)
+
+        error = lldb.SBError()
+        provider_id = target.RegisterScriptedFrameProvider(
+            "test_frame_providers.PrependFrameProvider",
+            lldb.SBStructuredData(),
+            error,
+        )
+        self.assertTrue(error.Success(), f"Failed to register provider: {error}")
+        self.assertNotEqual(provider_id, 0, "Provider ID should be non-zero")
+
+        # Verify we have 2 more frames
+        new_frame_count = thread.GetNumFrames()
+        self.assertEqual(new_frame_count, original_frame_count + 2)
+
+        # Verify first 2 frames are synthetic (check PCs, not function names)
+        frame0 = thread.GetFrameAtIndex(0)
+        self.assertEqual(frame0.GetPC(), 0x9000)
+
+        frame1 = thread.GetFrameAtIndex(1)
+        self.assertEqual(frame1.GetPC(), 0xA000)
+
+        # Verify frame 2 is the original real frame 0
+        frame2 = thread.GetFrameAtIndex(2)
+        self.assertIn("thread_func", frame2.GetFunctionName())
+
+    def test_append_frames(self):
+        """Test that we can add frames after real stack."""
+        self.build()
+        target, process, thread, bkpt = lldbutil.run_to_source_breakpoint(
+            self, "Break here", lldb.SBFileSpec(self.source), only_one_thread=False
+        )
+
+        # Get original frame count
+        original_frame_count = thread.GetNumFrames()
+
+        # Import and attach Append provider
+        script_path = os.path.join(self.getSourceDir(), "test_frame_providers.py")
+        self.runCmd("command script import " + script_path)
+
+        error = lldb.SBError()
+        provider_id = target.RegisterScriptedFrameProvider(
+            "test_frame_providers.AppendFrameProvider",
+            lldb.SBStructuredData(),
+            error,
+        )
+        self.assertTrue(error.Success(), f"Failed to register provider: {error}")
+        self.assertNotEqual(provider_id, 0, "Provider ID should be non-zero")
+
+        # Verify we have 1 more frame
+        new_frame_count = thread.GetNumFrames()
+        self.assertEqual(new_frame_count, original_frame_count + 1)
+
+        # Verify first frames are still real
+        frame0 = thread.GetFrameAtIndex(0)
+        self.assertIn("thread_func", frame0.GetFunctionName())
+
+        frame_n_plus_1 = thread.GetFrameAtIndex(new_frame_count - 1)
+        self.assertEqual(frame_n_plus_1.GetPC(), 0x10)
+
+    def test_scripted_frame_objects(self):
+        """Test that provider can return ScriptedFrame objects."""
+        self.build()
+        target, process, thread, bkpt = lldbutil.run_to_source_breakpoint(
+            self, "Break here", lldb.SBFileSpec(self.source), only_one_thread=False
+        )
+
+        # Import the provider that returns ScriptedFrame objects
+        script_path = os.path.join(self.getSourceDir(), "test_frame_providers.py")
+        self.runCmd("command script import " + script_path)
+
+        error = lldb.SBError()
+        provider_id = target.RegisterScriptedFrameProvider(
+            "test_frame_providers.ScriptedFrameObjectProvider",
+            lldb.SBStructuredData(),
+            error,
+        )
+        self.assertTrue(error.Success(), f"Failed to register provider: {error}")
+        self.assertNotEqual(provider_id, 0, "Provider ID should be non-zero")
+
+        # Verify we have 5 frames
+        self.assertEqual(
+            thread.GetNumFrames(), 5, "Should have 5 custom scripted frames"
+        )
+
+        # Verify frame properties from CustomScriptedFrame
+        frame0 = thread.GetFrameAtIndex(0)
+        self.assertIsNotNone(frame0)
+        self.assertEqual(frame0.GetFunctionName(), "custom_scripted_frame_0")
+        self.assertEqual(frame0.GetPC(), 0x5000)
+        self.assertTrue(frame0.IsSynthetic(), "Frame should be marked as synthetic")
+
+        frame1 = thread.GetFrameAtIndex(1)
+        self.assertIsNotNone(frame1)
+        self.assertEqual(frame1.GetPC(), 0x6000)
+
+        frame2 = thread.GetFrameAtIndex(2)
+        self.assertIsNotNone(frame2)
+        self.assertEqual(frame2.GetFunctionName(), "custom_scripted_frame_2")
+        self.assertEqual(frame2.GetPC(), 0x7000)
+        self.assertTrue(frame2.IsSynthetic(), "Frame should be marked as synthetic")
+
+    def test_applies_to_thread(self):
+        """Test that applies_to_thread filters which threads get the provider."""
+        self.build()
+        target, process, thread, bkpt = lldbutil.run_to_source_breakpoint(
+            self, "Break here", lldb.SBFileSpec(self.source), only_one_thread=False
+        )
+
+        # We should have at least 2 threads (worker threads) at the breakpoint
+        num_threads = process.GetNumThreads()
+        self.assertGreaterEqual(
+            num_threads, 2, "Should have at least 2 threads at breakpoint"
+        )
+
+        # Import the test frame provider
+        script_path = os.path.join(self.getSourceDir(), "test_frame_providers.py")
+        self.runCmd("command script import " + script_path)
+
+        # Collect original thread info before applying provider
+        thread_info = {}
+        for i in range(num_threads):
+            t = process.GetThreadAtIndex(i)
+            thread_info[t.GetIndexID()] = {
+                "frame_count": t.GetNumFrames(),
+                "pc": t.GetFrameAtIndex(0).GetPC(),
+            }
+
+        # Register the ThreadFilterFrameProvider which only applies to thread ID 1
+        error = lldb.SBError()
+        provider_id = target.RegisterScriptedFrameProvider(
+            "test_frame_providers.ThreadFilterFrameProvider",
+            lldb.SBStructuredData(),
+            error,
+        )
+        self.assertTrue(error.Success(), f"Failed to register provider: {error}")
+        self.assertNotEqual(provider_id, 0, "Provider ID should be non-zero")
+
+        # Check each thread
+        thread_id_1_found = False
+        # On ARM32, FixCodeAddress clears bit 0, so synthetic PCs get modified
+        is_arm_32bit = lldbplatformutil.getArchitecture() == "arm"
+        expected_synthetic_pc = 0xFFFE if is_arm_32bit else 0xFFFF
+
+        for i in range(num_threads):
+            t = process.GetThreadAtIndex(i)
+            thread_id = t.GetIndexID()
+
+            if thread_id == 1:
+                # Thread with ID 1 should have synthetic frame
+                thread_id_1_found = True
+                self.assertEqual(
+                    t.GetNumFrames(),
+                    1,
+                    f"Thread with ID 1 should have 1 synthetic frame",
+                )
+                self.assertEqual(
+                    t.GetFrameAtIndex(0).GetPC(),
+                    expected_synthetic_pc,
+                    f"Thread with ID 1 should have synthetic PC {expected_synthetic_pc:#x}",
+                )
+            else:
+                # Other threads should keep their original frames
+                self.assertEqual(
+                    t.GetNumFrames(),
+                    thread_info[thread_id]["frame_count"],
+                    f"Thread with ID {thread_id} should not be affected by provider",
+                )
+                self.assertEqual(
+                    t.GetFrameAtIndex(0).GetPC(),
+                    thread_info[thread_id]["pc"],
+                    f"Thread with ID {thread_id} should have its original PC",
+                )
+
+        # We should have found at least one thread with ID 1
+        self.assertTrue(
+            thread_id_1_found,
+            "Should have found a thread with ID 1 to test filtering",
+        )
+
+    def test_remove_frame_provider_by_id(self):
+        """Test that RemoveScriptedFrameProvider removes a specific provider by ID."""
+        self.build()
+        target, process, thread, bkpt = lldbutil.run_to_source_breakpoint(
+            self, "Break here", lldb.SBFileSpec(self.source), only_one_thread=False
+        )
+
+        # Import the test frame providers
+        script_path = os.path.join(self.getSourceDir(), "test_frame_providers.py")
+        self.runCmd("command script import " + script_path)
+
+        # Get original frame count
+        original_frame_count = thread.GetNumFrames()
+        original_pc = thread.GetFrameAtIndex(0).GetPC()
+
+        # Register the first provider and get its ID
+        error = lldb.SBError()
+        provider_id_1 = target.RegisterScriptedFrameProvider(
+            "test_frame_providers.ReplaceFrameProvider",
+            lldb.SBStructuredData(),
+            error,
+        )
+        self.assertTrue(error.Success(), f"Failed to register provider 1: {error}")
+
+        # Verify first provider is active (3 synthetic frames)
+        self.assertEqual(thread.GetNumFrames(), 3, "Should have 3 synthetic frames")
+        self.assertEqual(
+            thread.GetFrameAtIndex(0).GetPC(), 0x1000, "Should have first provider's PC"
+        )
+
+        # Register a second provider and get its ID
+        provider_id_2 = target.RegisterScriptedFrameProvider(
+            "test_frame_providers.PrependFrameProvider",
+            lldb.SBStructuredData(),
+            error,
+        )
+        self.assertTrue(error.Success(), f"Failed to register provider 2: {error}")
+
+        # Verify IDs are different
+        self.assertNotEqual(
+            provider_id_1, provider_id_2, "Provider IDs should be unique"
+        )
+
+        # Now remove the first provider by ID
+        result = target.RemoveScriptedFrameProvider(provider_id_1)
+        self.assertSuccess(
+            result, f"Should successfully remove provider with ID {provider_id_1}"
+        )
+
+        # After removing the first provider, the second provider should still be active
+        # The PrependFrameProvider adds 2 frames before the real stack
+        # Since ReplaceFrameProvider had 3 frames, and we removed it, we should now
+        # have the original frames (from real stack) with PrependFrameProvider applied
+        new_frame_count = thread.GetNumFrames()
+        self.assertEqual(
+            new_frame_count,
+            original_frame_count + 2,
+            "Should have original frames + 2 prepended frames",
+        )
+
+        # First two frames should be from PrependFrameProvider
+        self.assertEqual(
+            thread.GetFrameAtIndex(0).GetPC(),
+            0x9000,
+            "First frame should be from PrependFrameProvider",
+        )
+        self.assertEqual(
+            thread.GetFrameAtIndex(1).GetPC(),
+            0xA000,
+            "Second frame should be from PrependFrameProvider",
+        )
+
+        # Remove the second provider
+        result = target.RemoveScriptedFrameProvider(provider_id_2)
+        self.assertSuccess(
+            result, f"Should successfully remove provider with ID {provider_id_2}"
+        )
+
+        # After removing both providers, frames should be back to original
+        self.assertEqual(
+            thread.GetNumFrames(),
+            original_frame_count,
+            "Should restore original frame count",
+        )
+        self.assertEqual(
+            thread.GetFrameAtIndex(0).GetPC(),
+            original_pc,
+            "Should restore original PC",
+        )
+
+        # Try to remove a provider that doesn't exist
+        result = target.RemoveScriptedFrameProvider(999999)
+        self.assertTrue(result.Fail(), "Should fail to remove non-existent provider")
+
+    def test_circular_dependency_fix(self):
+        """Test that accessing input_frames in __init__ doesn't cause circular dependency.
+
+        This test verifies the fix for the circular dependency issue where:
+        1. Thread::GetStackFrameList() creates the frame provider
+        2. Provider's __init__ accesses input_frames and calls methods on frames
+        3. SBFrame methods trigger ExecutionContextRef::GetFrameSP()
+        4. Before the fix: GetFrameSP() would call Thread::GetStackFrameList() again -> circular dependency!
+        5. After the fix: GetFrameSP() uses the remembered frame list -> no circular dependency
+
+        The fix works by:
+        - StackFrame stores m_frame_list_wp (weak pointer to originating list)
+        - ExecutionContextRef stores m_frame_list_wp when created from a frame
+        - ExecutionContextRef::GetFrameSP() tries the remembered list first before asking the thread
+        """
+        self.build()
+        target, process, thread, bkpt = lldbutil.run_to_source_breakpoint(
+            self, "Break here", lldb.SBFileSpec(self.source), only_one_thread=False
+        )
+
+        # Get original frame count and PC
+        original_frame_count = thread.GetNumFrames()
+        original_pc = thread.GetFrameAtIndex(0).GetPC()
+        self.assertGreaterEqual(
+            original_frame_count, 2, "Should have at least 2 real frames"
+        )
+
+        # Import the provider that accesses input frames in __init__
+        script_path = os.path.join(self.getSourceDir(), "test_frame_providers.py")
+        self.runCmd("command script import " + script_path)
+
+        # Register the CircularDependencyTestProvider
+        # Before the fix, this would crash or hang due to circular dependency
+        error = lldb.SBError()
+        provider_id = target.RegisterScriptedFrameProvider(
+            "test_frame_providers.CircularDependencyTestProvider",
+            lldb.SBStructuredData(),
+            error,
+        )
+
+        # If we get here without crashing, the fix is working!
+        self.assertTrue(error.Success(), f"Failed to register provider: {error}")
+        self.assertNotEqual(provider_id, 0, "Provider ID should be non-zero")
+
+        # Verify the provider worked correctly
+        # Should have 1 synthetic frame + all original frames
+        new_frame_count = thread.GetNumFrames()
+        self.assertEqual(
+            new_frame_count,
+            original_frame_count + 1,
+            "Should have original frames + 1 synthetic frame",
+        )
+
+        # On ARM32, FixCodeAddress clears bit 0, so synthetic PCs get modified
+        is_arm_32bit = lldbplatformutil.getArchitecture() == "arm"
+        expected_synthetic_pc = 0xDEADBEEE if is_arm_32bit else 0xDEADBEEF
+
+        # First frame should be synthetic
+        frame0 = thread.GetFrameAtIndex(0)
+        self.assertIsNotNone(frame0)
+        self.assertEqual(
+            frame0.GetPC(),
+            expected_synthetic_pc,
+            f"First frame should be synthetic frame with PC {expected_synthetic_pc:#x}",
+        )
+
+        # Second frame should be the original first frame
+        frame1 = thread.GetFrameAtIndex(1)
+        self.assertIsNotNone(frame1)
+        self.assertEqual(
+            frame1.GetPC(),
+            original_pc,
+            "Second frame should be original first frame",
+        )
+
+        # Verify we can still call methods on frames (no circular dependency!)
+        for i in range(min(3, new_frame_count)):
+            frame = thread.GetFrameAtIndex(i)
+            self.assertIsNotNone(frame)
+            # These calls should not trigger circular dependency
+            pc = frame.GetPC()
+            self.assertNotEqual(pc, 0, f"Frame {i} should have valid PC")
diff --git a/lldb/test/API/functionalities/scripted_frame_provider/circular_dependency/Makefile b/lldb/test/API/functionalities/scripted_frame_provider/circular_dependency/Makefile
new file mode 100644
index 0000000000000..10495940055b6
--- /dev/null
+++ b/lldb/test/API/functionalities/scripted_frame_provider/circular_dependency/Makefile
@@ -0,0 +1,3 @@
+C_SOURCES := main.c
+
+include Makefile.rules
diff --git a/lldb/test/API/functionalities/scripted_frame_provider/circular_dependency/TestFrameProviderCircularDependency.py b/lldb/test/API/functionalities/scripted_frame_provider/circular_dependency/TestFrameProviderCircularDependency.py
new file mode 100644
index 0000000000000..7552de18a58df
--- /dev/null
+++ b/lldb/test/API/functionalities/scripted_frame_provider/circular_dependency/TestFrameProviderCircularDependency.py
@@ -0,0 +1,119 @@
+"""
+Test that frame providers wouldn't cause a hang due to a circular dependency
+during its initialization.
+"""
+
+import os
+import lldb
+from lldbsuite.test.decorators import *
+from lldbsuite.test.lldbtest import TestBase
+from lldbsuite.test import lldbutil
+
+
+class FrameProviderCircularDependencyTestCase(TestBase):
+    NO_DEBUG_INFO_TESTCASE = True
+
+    def setUp(self):
+        TestBase.setUp(self)
+        self.source = "main.c"
+
+    @expectedFailureAll(oslist=["windows"], bugnumber="llvm.org/pr24778")
+    def test_circular_dependency_with_function_replacement(self):
+        """
+        Test the circular dependency fix with a provider that replaces function names.
+        """
+        self.build()
+
+        target = self.dbg.CreateTarget(self.getBuildArtifact("a.out"))
+        self.assertTrue(target, "Target should be valid")
+
+        bkpt = target.BreakpointCreateBySourceRegex(
+            "break here", lldb.SBFileSpec(self.source)
+        )
+        self.assertTrue(bkpt.IsValid(), "Breakpoint should be valid")
+        self.assertEqual(bkpt.GetNumLocations(), 1, "Should have 1 breakpoint location")
+
+        process = target.LaunchSimple(None, None, self.get_process_working_directory())
+        self.assertTrue(process, "Process should be valid")
+        self.assertEqual(
+            process.GetState(), lldb.eStateStopped, "Process should be stopped"
+        )
+
+        thread = process.GetSelectedThread()
+        self.assertTrue(thread.IsValid(), "Thread should be valid")
+
+        frame0 = thread.GetFrameAtIndex(0)
+        self.assertIn("bar", frame0.GetFunctionName(), "Should be stopped in bar()")
+
+        original_frame_count = thread.GetNumFrames()
+        self.assertGreaterEqual(
+            original_frame_count, 3, "Should have at least 3 frames: bar, foo, main"
+        )
+
+        frame_names = [thread.GetFrameAtIndex(i).GetFunctionName() for i in range(3)]
+        self.assertEqual(frame_names[0], "bar", "Frame 0 should be bar")
+        self.assertEqual(frame_names[1], "foo", "Frame 1 should be foo")
+        self.assertEqual(frame_names[2], "main", "Frame 2 should be main")
+
+        script_path = os.path.join(self.getSourceDir(), "frame_provider.py")
+        self.runCmd("command script import " + script_path)
+
+        # Register the frame provider that accesses input_frames.
+        # Before the fix, this registration would trigger the circular dependency:
+        # - Thread::GetStackFrameList() creates provider
+        # - Provider's get_frame_at_index() accesses input_frames[0]
+        # - Calls frame.GetFunctionName() -> ExecutionContextRef::GetFrameSP()
+        # - Before fix: Calls Thread::GetStackFrameList() again -> CIRCULAR!
+        # - After fix: Uses remembered m_frame_list_wp -> Works!
+        error = lldb.SBError()
+        provider_id = target.RegisterScriptedFrameProvider(
+            "frame_provider.ScriptedFrameObjectProvider",
+            lldb.SBStructuredData(),
+            error,
+        )
+
+        # If we reach here without crashing/hanging, the fix is working!
+        self.assertTrue(
+            error.Success(),
+            f"Should successfully register provider (if this fails, circular dependency!): {error}",
+        )
+        self.assertNotEqual(provider_id, 0, "Provider ID should be non-zero")
+
+        # Verify the provider is working correctly.
+        # Frame count should be unchanged (we're replacing frames, not adding).
+        new_frame_count = thread.GetNumFrames()
+        self.assertEqual(
+            new_frame_count,
+            original_frame_count,
+            "Frame count should be unchanged (replacement, not addition)",
+        )
+
+        # Verify that "bar" was replaced with "baz".
+        frame0_new = thread.GetFrameAtIndex(0)
+        self.assertIsNotNone(frame0_new, "Frame 0 should exist")
+        self.assertEqual(
+            frame0_new.GetFunctionName(),
+            "baz",
+            "Frame 0 function should be replaced: bar -> baz",
+        )
+
+        # Verify other frames are unchanged.
+        frame1_new = thread.GetFrameAtIndex(1)
+        self.assertEqual(
+            frame1_new.GetFunctionName(), "foo", "Frame 1 should still be foo"
+        )
+
+        frame2_new = thread.GetFrameAtIndex(2)
+        self.assertEqual(
+            frame2_new.GetFunctionName(), "main", "Frame 2 should still be main"
+        )
+
+        # Verify we can call methods on all frames (no circular dependency!).
+        for i in range(new_frame_count):
+            frame = thread.GetFrameAtIndex(i)
+            self.assertIsNotNone(frame, f"Frame {i} should exist")
+            # These calls should not trigger circular dependency.
+            pc = frame.GetPC()
+            self.assertNotEqual(pc, 0, f"Frame {i} should have valid PC")
+            func_name = frame.GetFunctionName()
+            self.assertIsNotNone(func_name, f"Frame {i} should have function name")
diff --git a/lldb/test/API/functionalities/scripted_frame_provider/circular_dependency/frame_provider.py b/lldb/test/API/functionalities/scripted_frame_provider/circular_dependency/frame_provider.py
new file mode 100644
index 0000000000000..f27f18cd07b7f
--- /dev/null
+++ b/lldb/test/API/functionalities/scripted_frame_provider/circular_dependency/frame_provider.py
@@ -0,0 +1,102 @@
+"""
+Frame provider that reproduces the circular dependency issue.
+
+This provider accesses input_frames and calls methods on them,
+which before the fix would cause a circular dependency.
+"""
+
+import lldb
+from lldb.plugins.scripted_process import ScriptedFrame
+from lldb.plugins.scripted_frame_provider import ScriptedFrameProvider
+
+
+class CustomScriptedFrame(ScriptedFrame):
+    """Custom scripted frame with full control over frame behavior."""
+
+    def __init__(self, thread, idx, pc, function_name):
+        args = lldb.SBStructuredData()
+        super().__init__(thread, args)
+
+        self.idx = idx
+        self.pc = pc
+        self.function_name = function_name
+
+    def get_id(self):
+        """Return the frame index."""
+        return self.idx
+
+    def get_pc(self):
+        """Return the program counter."""
+        return self.pc
+
+    def get_function_name(self):
+        """Return the function name."""
+        return self.function_name
+
+    def is_artificial(self):
+        """Mark as artificial frame."""
+        return False
+
+    def is_hidden(self):
+        """Not hidden."""
+        return False
+
+    def get_register_context(self):
+        return None
+
+
+class ScriptedFrameObjectProvider(ScriptedFrameProvider):
+    """
+    Provider that returns ScriptedFrame objects and accesses input_frames.
+
+    This provider demonstrates the circular dependency bug fix:
+    1. During get_frame_at_index(), we access input_frames[idx]
+    2. We call frame.GetFunctionName() and frame.GetPC() on input frames
+    3. Before the fix: These calls would trigger ExecutionContextRef::GetFrameSP()
+       which would call Thread::GetStackFrameList() -> circular dependency!
+    4. After the fix: ExecutionContextRef uses the remembered frame list -> no circular dependency
+    """
+
+    def __init__(self, input_frames, args):
+        super().__init__(input_frames, args)
+        self.replacement_count = 0
+        if self.target.process:
+            baz_symbol_ctx = self.target.FindFunctions("baz")
+            self.baz_symbol_ctx = None
+            if len(baz_symbol_ctx) == 1:
+                self.baz_symbol_ctx = baz_symbol_ctx[0]
+
+    @staticmethod
+    def get_description():
+        """Return a description of this provider."""
+        return "Provider that replaces 'bar' function with 'baz'"
+
+    def get_frame_at_index(self, idx):
+        """
+        Replace frames named 'bar' with custom frames named 'baz'.
+
+        This accesses input_frames and calls methods on them, which would
+        trigger the circular dependency bug before the fix.
+        """
+        if idx < len(self.input_frames):
+            # This access and method calls would cause circular dependency before fix!
+            frame = self.input_frames[idx]
+
+            # Calling GetFunctionName() triggers ExecutionContextRef resolution.
+            function_name = frame.GetFunctionName()
+
+            if function_name == "bar" and self.baz_symbol_ctx:
+                # Replace "bar" with "baz".
+                baz_func = self.baz_symbol_ctx.GetFunction()
+                new_function_name = baz_func.GetName()
+                pc = baz_func.GetStartAddress().GetLoadAddress(self.target)
+                custom_frame = CustomScriptedFrame(
+                    self.thread, idx, pc, new_function_name
+                )
+                self.replacement_count += 1
+                return custom_frame
+
+            # Pass through other frames by returning their index.
+            return idx
+
+        return None
diff --git a/lldb/test/API/functionalities/scripted_frame_provider/circular_dependency/main.c b/lldb/test/API/functionalities/scripted_frame_provider/circular_dependency/main.c
new file mode 100644
index 0000000000000..bbd1028236f40
--- /dev/null
+++ b/lldb/test/API/functionalities/scripted_frame_provider/circular_dependency/main.c
@@ -0,0 +1,21 @@
+#include <stdio.h>
+
+int baz() {
+  printf("baz\n");
+  return 666;
+}
+
+int bar() {
+  printf("bar\n");
+  return 42; // break here.
+}
+
+int foo() {
+  printf("foo\n");
+  return bar();
+}
+
+int main() {
+  printf("main\n");
+  return foo();
+}
diff --git a/lldb/test/API/functionalities/scripted_frame_provider/main.cpp b/lldb/test/API/functionalities/scripted_frame_provider/main.cpp
new file mode 100644
index 0000000000000..0298e88e4de16
--- /dev/null
+++ b/lldb/test/API/functionalities/scripted_frame_provider/main.cpp
@@ -0,0 +1,53 @@
+// Multi-threaded test program for testing frame providers.
+
+#include <condition_variable>
+#include <iostream>
+#include <mutex>
+#include <thread>
+
+std::mutex mtx;
+std::condition_variable cv;
+int ready_count = 0;
+constexpr int NUM_THREADS = 2;
+
+void thread_func(int thread_num) {
+  std::cout << "Thread " << thread_num << " started\n";
+
+  {
+    std::unique_lock<std::mutex> lock(mtx);
+    ready_count++;
+    if (ready_count == NUM_THREADS + 1) {
+      cv.notify_all();
+    } else {
+      cv.wait(lock, [] { return ready_count == NUM_THREADS + 1; });
+    }
+  }
+
+  std::cout << "Thread " << thread_num << " at breakpoint\n"; // Break here.
+}
+
+int main(int argc, char **argv) {
+  std::thread threads[NUM_THREADS];
+
+  for (int i = 0; i < NUM_THREADS; i++) {
+    threads[i] = std::thread(thread_func, i);
+  }
+
+  {
+    std::unique_lock<std::mutex> lock(mtx);
+    ready_count++;
+    if (ready_count == NUM_THREADS + 1) {
+      cv.notify_all();
+    } else {
+      cv.wait(lock, [] { return ready_count == NUM_THREADS + 1; });
+    }
+  }
+
+  std::cout << "Main thread at barrier\n";
+
+  for (int i = 0; i < NUM_THREADS; i++)
+    threads[i].join();
+
+  std::cout << "All threads completed\n";
+  return 0;
+}
diff --git a/lldb/test/API/functionalities/scripted_frame_provider/test_frame_providers.py b/lldb/test/API/functionalities/scripted_frame_provider/test_frame_providers.py
new file mode 100644
index 0000000000000..b9731fdc0a197
--- /dev/null
+++ b/lldb/test/API/functionalities/scripted_frame_provider/test_frame_providers.py
@@ -0,0 +1,222 @@
+"""
+Test frame providers for scripted frame provider functionality.
+
+These providers demonstrate various merge strategies:
+- Replace: Replace entire stack
+- Prepend: Add frames before real stack
+- Append: Add frames after real stack
+
+It also shows the ability to mix a dictionary, a ScriptedFrame or an SBFrame
+index to create stackframes
+"""
+
+import lldb
+from lldb.plugins.scripted_process import ScriptedFrame
+from lldb.plugins.scripted_frame_provider import ScriptedFrameProvider
+
+
+class ReplaceFrameProvider(ScriptedFrameProvider):
+    """Replace entire stack with custom frames."""
+
+    def __init__(self, input_frames, args):
+        super().__init__(input_frames, args)
+        self.frames = [
+            {
+                "idx": 0,
+                "pc": 0x1000,
+            },
+            0,
+            {
+                "idx": 2,
+                "pc": 0x3000,
+            },
+        ]
+
+    @staticmethod
+    def get_description():
+        """Return a description of this provider."""
+        return "Replace entire stack with 3 custom frames"
+
+    def get_frame_at_index(self, index):
+        if index >= len(self.frames):
+            return None
+        return self.frames[index]
+
+
+class PrependFrameProvider(ScriptedFrameProvider):
+    """Prepend synthetic frames before real stack."""
+
+    def __init__(self, input_frames, args):
+        super().__init__(input_frames, args)
+
+    @staticmethod
+    def get_description():
+        """Return a description of this provider."""
+        return "Prepend 2 synthetic frames before real stack"
+
+    def get_frame_at_index(self, index):
+        if index == 0:
+            return {"pc": 0x9000}
+        elif index == 1:
+            return {"pc": 0xA000}
+        elif index - 2 < len(self.input_frames):
+            return index - 2  # Return real frame index.
+        return None
+
+
+class AppendFrameProvider(ScriptedFrameProvider):
+    """Append synthetic frames after real stack."""
+
+    def __init__(self, input_frames, args):
+        super().__init__(input_frames, args)
+
+    @staticmethod
+    def get_description():
+        """Return a description of this provider."""
+        return "Append 1 synthetic frame after real stack"
+
+    def get_frame_at_index(self, index):
+        if index < len(self.input_frames):
+            return index  # Return real frame index.
+        elif index == len(self.input_frames):
+            return {
+                "idx": 1,
+                "pc": 0x10,
+            }
+        return None
+
+
+class CustomScriptedFrame(ScriptedFrame):
+    """Custom scripted frame with full control over frame behavior."""
+
+    def __init__(self, thread, idx, pc, function_name):
+        args = lldb.SBStructuredData()
+        super().__init__(thread, args)
+
+        self.idx = idx
+        self.pc = pc
+        self.function_name = function_name
+
+    def get_id(self):
+        """Return the frame index."""
+        return self.idx
+
+    def get_pc(self):
+        """Return the program counter."""
+        return self.pc
+
+    def get_function_name(self):
+        """Return the function name."""
+        return self.function_name
+
+    def is_artificial(self):
+        """Mark as artificial frame."""
+        return False
+
+    def is_hidden(self):
+        """Not hidden."""
+        return False
+
+    def get_register_context(self):
+        """No register context for this test."""
+        return None
+
+
+class ScriptedFrameObjectProvider(ScriptedFrameProvider):
+    """Provider that returns ScriptedFrame objects instead of dictionaries."""
+
+    def __init__(self, input_frames, args):
+        super().__init__(input_frames, args)
+
+    @staticmethod
+    def get_description():
+        """Return a description of this provider."""
+        return "Provider returning custom ScriptedFrame objects"
+
+    def get_frame_at_index(self, index):
+        """Return ScriptedFrame objects or dictionaries based on index."""
+        if index == 0:
+            return CustomScriptedFrame(
+                self.thread, 0, 0x5000, "custom_scripted_frame_0"
+            )
+        elif index == 1:
+            return {"pc": 0x6000}
+        elif index == 2:
+            return CustomScriptedFrame(
+                self.thread, 2, 0x7000, "custom_scripted_frame_2"
+            )
+        elif index == 3:
+            return len(self.input_frames) - 2  # Real frame index.
+        elif index == 4:
+            return len(self.input_frames) - 1  # Real frame index.
+        return None
+
+
+class ThreadFilterFrameProvider(ScriptedFrameProvider):
+    """Provider that only applies to thread with ID 1."""
+
+    @staticmethod
+    def applies_to_thread(thread):
+        """Only apply to thread with index ID 1."""
+        return thread.GetIndexID() == 1
+
+    def __init__(self, input_frames, args):
+        super().__init__(input_frames, args)
+
+    @staticmethod
+    def get_description():
+        """Return a description of this provider."""
+        return "Provider that only applies to thread ID 1"
+
+    def get_frame_at_index(self, index):
+        """Return a single synthetic frame."""
+        if index == 0:
+            return {"pc": 0xFFFF}
+        return None
+
+
+class CircularDependencyTestProvider(ScriptedFrameProvider):
+    """
+    Provider that tests the circular dependency fix.
+
+    This provider accesses input_frames during __init__ and calls methods
+    on those frames. Before the fix, this would cause a circular dependency:
+    - Thread::GetStackFrameList() creates provider
+    - Provider's __init__ accesses input_frames[0]
+    - SBFrame::GetPC() tries to resolve ExecutionContextRef
+    - ExecutionContextRef::GetFrameSP() calls Thread::GetStackFrameList()
+    - Re-enters initialization -> circular dependency!
+
+    With the fix, ExecutionContextRef remembers the frame list, so it doesn't
+    re-enter Thread::GetStackFrameList().
+    """
+
+    def __init__(self, input_frames, args):
+        super().__init__(input_frames, args)
+
+        # This would cause circular dependency before the fix!
+        # Accessing frames and calling methods on them during __init__
+        self.original_frame_count = len(input_frames)
+        self.original_pcs = []
+
+        # Call GetPC() on each input frame - this triggers ExecutionContextRef resolution.
+        for i in range(min(3, len(input_frames))):
+            frame = input_frames[i]
+            if frame.IsValid():
+                pc = frame.GetPC()
+                self.original_pcs.append(pc)
+
+    @staticmethod
+    def get_description():
+        """Return a description of this provider."""
+        return "Provider that tests circular dependency fix by accessing frames in __init__"
+
+    def get_frame_at_index(self, index):
+        """Prepend a synthetic frame, then pass through original frames."""
+        if index == 0:
+            # Synthetic frame at index 0.
+            return {"pc": 0xDEADBEEF}
+        elif index - 1 < self.original_frame_count:
+            # Pass through original frames at indices 1, 2, 3, ...
+            return index - 1
+        return None
diff --git a/lldb/test/API/python_api/exprpath_register/Makefile b/lldb/test/API/python_api/exprpath_register/Makefile
new file mode 100644
index 0000000000000..10495940055b6
--- /dev/null
+++ b/lldb/test/API/python_api/exprpath_register/Makefile
@@ -0,0 +1,3 @@
+C_SOURCES := main.c
+
+include Makefile.rules
diff --git a/lldb/test/API/python_api/exprpath_register/TestExprPathRegisters.py b/lldb/test/API/python_api/exprpath_register/TestExprPathRegisters.py
new file mode 100644
index 0000000000000..4ffbc5e49fb0d
--- /dev/null
+++ b/lldb/test/API/python_api/exprpath_register/TestExprPathRegisters.py
@@ -0,0 +1,64 @@
+"""
+Test Getting the expression path for registers works correctly
+"""
+
+import lldb
+from lldbsuite.test import lldbutil
+from lldbsuite.test.lldbtest import TestBase, VALID_BREAKPOINT, VALID_TARGET
+
+
+class TestExprPathRegisters(TestBase):
+    NO_DEBUG_INFO_TESTCASE = True
+
+    def verify_register_path(self, reg_value: lldb.SBValue):
+        stream = lldb.SBStream()
+        reg_name = reg_value.name
+        self.assertTrue(
+            reg_value.GetExpressionPath(stream),
+            f"Expected an expression path for register {reg_name}.",
+        )
+        reg_expr_path = stream.GetData()
+        self.assertEqual(reg_expr_path, f"${reg_name}")
+
+    def test_float_registers(self):
+        """Verify the expression path of the registers is valid."""
+        self.build()
+        _, _, thread, _ = lldbutil.run_to_name_breakpoint(self, "my_foo")
+        frame = thread.GetSelectedFrame()
+        self.assertTrue(frame, "Expected a valid Frame.")
+
+        # possible floating point register on some cpus.
+        register_names = [
+            "xmm0",
+            "ymm0",
+            "v0",
+            "v1",
+            "f0",
+            "f1",
+            "d0",
+            "d1",
+            "vr0",
+            "vr1",
+            "st0",
+            "st1",
+        ]
+        for name in register_names:
+            reg_value = frame.FindRegister(name)
+            # some the register will not be available for the cpu
+            # only verify if it is valid.
+            if reg_value:
+                self.verify_register_path(reg_value)
+
+    def test_all_registers(self):
+        """Test all the registers that is avaiable on the machine"""
+        self.build()
+        _, _, thread, _ = lldbutil.run_to_name_breakpoint(self, "my_foo")
+        frame = thread.GetSelectedFrame()
+        self.assertTrue(frame, "Expected a valid Frame.")
+
+        register_sets = frame.GetRegisters()
+        self.assertTrue(register_sets.IsValid(), "Expected Frame Registers")
+
+        for register_set in register_sets:
+            for register in register_set.children:
+                self.verify_register_path(register)
diff --git a/lldb/test/API/python_api/exprpath_register/main.c b/lldb/test/API/python_api/exprpath_register/main.c
new file mode 100644
index 0000000000000..4809a87cdf210
--- /dev/null
+++ b/lldb/test/API/python_api/exprpath_register/main.c
@@ -0,0 +1,10 @@
+
+float my_foo() {
+  float result = 10.0 + 20.0;
+  return result;
+}
+
+int main(void) {
+  float result = my_foo();
+  return (int)result;
+}
diff --git a/lldb/test/API/python_api/sbframe_extensions/Makefile b/lldb/test/API/python_api/sbframe_extensions/Makefile
new file mode 100644
index 0000000000000..10495940055b6
--- /dev/null
+++ b/lldb/test/API/python_api/sbframe_extensions/Makefile
@@ -0,0 +1,3 @@
+C_SOURCES := main.c
+
+include Makefile.rules
diff --git a/lldb/test/API/python_api/sbframe_extensions/TestSBFrameExtensions.py b/lldb/test/API/python_api/sbframe_extensions/TestSBFrameExtensions.py
new file mode 100644
index 0000000000000..d3eabfdd979c5
--- /dev/null
+++ b/lldb/test/API/python_api/sbframe_extensions/TestSBFrameExtensions.py
@@ -0,0 +1,534 @@
+"""
+Test SBFrameExtensions API.
+"""
+
+import lldb
+from lldbsuite.test.decorators import *
+from lldbsuite.test.lldbtest import *
+from lldbsuite.test import lldbutil
+
+
+class TestSBFrameExtensions(TestBase):
+    NO_DEBUG_INFO_TESTCASE = True
+
+    def setUp(self):
+        TestBase.setUp(self)
+        self.source = "main.c"
+
+    def _get_frame(self):
+        """Helper method to get a valid frame for testing."""
+        self.build()
+        target, process, thread, bkpt = lldbutil.run_to_source_breakpoint(
+            self, "Set breakpoint here", lldb.SBFileSpec(self.source)
+        )
+        frame = thread.GetFrameAtIndex(0)
+        self.assertTrue(frame.IsValid(), "Frame should be valid")
+        return frame, thread
+
+    def test_property_pc(self):
+        """Test SBFrame extension property: pc"""
+        frame, _ = self._get_frame()
+
+        pc = frame.pc
+        self.assertIsInstance(pc, int, "pc should be an integer")
+        self.assertGreater(pc, 0, "pc should be greater than 0")
+        self.assertEqual(pc, frame.GetPC(), "pc property should match GetPC()")
+
+    def test_property_addr(self):
+        """Test SBFrame extension property: addr"""
+        frame, _ = self._get_frame()
+
+        addr = frame.addr
+        self.assertTrue(addr.IsValid(), "addr should be valid")
+        self.assertEqual(addr, frame.GetPCAddress(), "addr should match GetPCAddress()")
+
+    def test_property_fp(self):
+        """Test SBFrame extension property: fp"""
+        frame, _ = self._get_frame()
+
+        fp = frame.fp
+        self.assertIsInstance(fp, int, "fp should be an integer")
+        self.assertEqual(fp, frame.GetFP(), "fp property should match GetFP()")
+
+    def test_property_sp(self):
+        """Test SBFrame extension property: sp"""
+        frame, _ = self._get_frame()
+
+        sp = frame.sp
+        self.assertIsInstance(sp, int, "sp should be an integer")
+        self.assertEqual(sp, frame.GetSP(), "sp property should match GetSP()")
+
+    def test_property_module(self):
+        """Test SBFrame extension property: module"""
+        frame, _ = self._get_frame()
+
+        module = frame.module
+        self.assertTrue(module.IsValid(), "module should be valid")
+        self.assertEqual(module, frame.GetModule(), "module should match GetModule()")
+
+    def test_property_compile_unit(self):
+        """Test SBFrame extension property: compile_unit"""
+        frame, _ = self._get_frame()
+
+        compile_unit = frame.compile_unit
+        self.assertTrue(compile_unit.IsValid(), "compile_unit should be valid")
+        self.assertEqual(
+            compile_unit,
+            frame.GetCompileUnit(),
+            "compile_unit should match GetCompileUnit()",
+        )
+
+    def test_property_function(self):
+        """Test SBFrame extension property: function"""
+        frame, _ = self._get_frame()
+
+        function = frame.function
+        self.assertTrue(function.IsValid(), "function should be valid")
+        self.assertEqual(
+            function, frame.GetFunction(), "function should match GetFunction()"
+        )
+
+    def test_property_symbol(self):
+        """Test SBFrame extension property: symbol"""
+        frame, _ = self._get_frame()
+
+        symbol = frame.symbol
+        self.assertTrue(symbol.IsValid(), "symbol should be valid")
+        self.assertEqual(symbol, frame.GetSymbol(), "symbol should match GetSymbol()")
+
+    def test_property_block(self):
+        """Test SBFrame extension property: block"""
+        frame, _ = self._get_frame()
+
+        block = frame.block
+        self.assertTrue(block.IsValid(), "block should be valid")
+        block_direct = frame.GetBlock()
+        self.assertTrue(block_direct.IsValid(), "GetBlock() should return valid block")
+        # Verify both blocks are valid and have the same ranges
+        # by comparing their first range start address.
+        block_ranges = block.GetRanges()
+        block_direct_ranges = block_direct.GetRanges()
+        if block_ranges.GetSize() > 0 and block_direct_ranges.GetSize() > 0:
+            self.assertEqual(
+                block.GetRangeStartAddress(0),
+                block_direct.GetRangeStartAddress(0),
+                "block should match GetBlock() start address",
+            )
+
+    def test_property_is_inlined(self):
+        """Test SBFrame extension property: is_inlined"""
+        frame, _ = self._get_frame()
+
+        is_inlined = frame.is_inlined
+        self.assertIsInstance(is_inlined, bool, "is_inlined should be a boolean")
+        self.assertEqual(
+            is_inlined, frame.IsInlined(), "is_inlined should match IsInlined()"
+        )
+
+    def test_property_name(self):
+        """Test SBFrame extension property: name"""
+        frame, _ = self._get_frame()
+
+        name = frame.name
+        self.assertIsInstance(name, str, "name should be a string")
+        self.assertEqual(
+            name, frame.GetFunctionName(), "name should match GetFunctionName()"
+        )
+        # Should be one of our functions.
+        self.assertIn(
+            name, ["func1", "func2", "main"], "name should be a known function"
+        )
+
+    def test_property_line_entry(self):
+        """Test SBFrame extension property: line_entry"""
+        frame, _ = self._get_frame()
+
+        line_entry = frame.line_entry
+        self.assertTrue(line_entry.IsValid(), "line_entry should be valid")
+        self.assertEqual(
+            line_entry, frame.GetLineEntry(), "line_entry should match GetLineEntry()"
+        )
+
+    def test_property_thread(self):
+        """Test SBFrame extension property: thread"""
+        frame, thread = self._get_frame()
+
+        thread_prop = frame.thread
+        self.assertTrue(thread_prop.IsValid(), "thread should be valid")
+        self.assertEqual(
+            thread_prop, frame.GetThread(), "thread should match GetThread()"
+        )
+        self.assertEqual(
+            thread_prop.GetThreadID(),
+            thread.GetThreadID(),
+            "thread should be the same thread",
+        )
+
+    def test_property_disassembly(self):
+        """Test SBFrame extension property: disassembly"""
+        frame, _ = self._get_frame()
+
+        disassembly = frame.disassembly
+        self.assertIsInstance(disassembly, str, "disassembly should be a string")
+        self.assertGreater(len(disassembly), 0, "disassembly should not be empty")
+        self.assertEqual(
+            disassembly, frame.Disassemble(), "disassembly should match Disassemble()"
+        )
+
+    def test_property_idx(self):
+        """Test SBFrame extension property: idx"""
+        frame, _ = self._get_frame()
+
+        idx = frame.idx
+        self.assertIsInstance(idx, int, "idx should be an integer")
+        self.assertEqual(idx, frame.GetFrameID(), "idx should match GetFrameID()")
+        self.assertEqual(idx, 0, "First frame should have idx 0")
+
+    def test_property_variables(self):
+        """Test SBFrame extension property: variables"""
+        frame, _ = self._get_frame()
+
+        variables = frame.variables
+        self.assertIsInstance(
+            variables, lldb.SBValueList, "variables should be SBValueList"
+        )
+        all_vars = frame.GetVariables(True, True, True, True)
+        self.assertEqual(
+            variables.GetSize(),
+            all_vars.GetSize(),
+            "variables should match GetVariables(True, True, True, True)",
+        )
+
+    def test_property_vars(self):
+        """Test SBFrame extension property: vars (alias for variables)"""
+        frame, _ = self._get_frame()
+
+        vars_prop = frame.vars
+        self.assertIsInstance(vars_prop, lldb.SBValueList, "vars should be SBValueList")
+        variables = frame.variables
+        self.assertEqual(
+            vars_prop.GetSize(),
+            variables.GetSize(),
+            "vars should match variables",
+        )
+
+    def test_property_locals(self):
+        """Test SBFrame extension property: locals"""
+        frame, _ = self._get_frame()
+
+        locals_prop = frame.locals
+        self.assertIsInstance(
+            locals_prop, lldb.SBValueList, "locals should be SBValueList"
+        )
+        locals_direct = frame.GetVariables(False, True, False, False)
+        self.assertEqual(
+            locals_prop.GetSize(),
+            locals_direct.GetSize(),
+            "locals should match GetVariables(False, True, False, False)",
+        )
+
+    def test_property_args(self):
+        """Test SBFrame extension property: args"""
+        frame, _ = self._get_frame()
+
+        args_prop = frame.args
+        self.assertIsInstance(args_prop, lldb.SBValueList, "args should be SBValueList")
+        args_direct = frame.GetVariables(True, False, False, False)
+        self.assertEqual(
+            args_prop.GetSize(),
+            args_direct.GetSize(),
+            "args should match GetVariables(True, False, False, False)",
+        )
+
+    def test_property_arguments(self):
+        """Test SBFrame extension property: arguments (alias for args)"""
+        frame, _ = self._get_frame()
+
+        arguments_prop = frame.arguments
+        self.assertIsInstance(
+            arguments_prop, lldb.SBValueList, "arguments should be SBValueList"
+        )
+        args_prop = frame.args
+        self.assertEqual(
+            arguments_prop.GetSize(),
+            args_prop.GetSize(),
+            "arguments should match args",
+        )
+
+    def test_property_statics(self):
+        """Test SBFrame extension property: statics"""
+        frame, _ = self._get_frame()
+
+        statics_prop = frame.statics
+        self.assertIsInstance(
+            statics_prop, lldb.SBValueList, "statics should be SBValueList"
+        )
+        statics_direct = frame.GetVariables(False, False, True, False)
+        self.assertEqual(
+            statics_prop.GetSize(),
+            statics_direct.GetSize(),
+            "statics should match GetVariables(False, False, True, False)",
+        )
+
+    def test_property_registers(self):
+        """Test SBFrame extension property: registers"""
+        frame, _ = self._get_frame()
+
+        registers = frame.registers
+        # registers returns an SBValueList that can be iterated.
+        self.assertTrue(hasattr(registers, "__iter__"), "registers should be iterable")
+        registers_direct = frame.GetRegisters()
+        # Compare by iterating and counting.
+        registers_count = sum(1 for _ in registers)
+        registers_direct_count = sum(1 for _ in registers_direct)
+        self.assertEqual(
+            registers_count,
+            registers_direct_count,
+            "registers should match GetRegisters()",
+        )
+
+    def test_property_regs(self):
+        """Test SBFrame extension property: regs (alias for registers)"""
+        frame, _ = self._get_frame()
+
+        regs = frame.regs
+        self.assertTrue(hasattr(regs, "__iter__"), "regs should be iterable")
+        registers = frame.registers
+        regs_count = sum(1 for _ in regs)
+        registers_count = sum(1 for _ in registers)
+        self.assertEqual(regs_count, registers_count, "regs should match registers")
+
+    def test_property_register(self):
+        """Test SBFrame extension property: register (flattened view)"""
+        frame, _ = self._get_frame()
+
+        register = frame.register
+        self.assertIsNotNone(register, "register should not be None")
+        # register is a helper object with __iter__ and __getitem__.
+        reg_names = set()
+        for reg in register:
+            self.assertTrue(reg.IsValid(), "Register should be valid")
+            reg_names.add(reg.name)
+
+        # Test register indexing by name.
+        if len(reg_names) > 0:
+            first_reg_name = list(reg_names)[0]
+            reg_by_name = register[first_reg_name]
+            self.assertTrue(reg_by_name.IsValid(), "Register by name should be valid")
+            self.assertEqual(
+                reg_by_name.name, first_reg_name, "Register name should match"
+            )
+
+    def test_property_reg(self):
+        """Test SBFrame extension property: reg (alias for register)"""
+        frame, _ = self._get_frame()
+
+        reg = frame.reg
+        self.assertIsNotNone(reg, "reg should not be None")
+        register = frame.register
+        reg_names = set()
+        for r in reg:
+            reg_names.add(r.name)
+        reg_names2 = set()
+        for r in register:
+            reg_names2.add(r.name)
+        self.assertEqual(reg_names, reg_names2, "reg should match register")
+
+    def test_property_parent(self):
+        """Test SBFrame extension property: parent"""
+        frame0, thread = self._get_frame()
+
+        # If there's a parent frame (frame 1), test parent property.
+        if thread.GetNumFrames() > 1:
+            frame1 = thread.GetFrameAtIndex(1)
+            parent = frame0.parent
+            self.assertTrue(parent.IsValid(), "parent should be valid")
+            self.assertEqual(
+                parent.GetFrameID(),
+                frame1.GetFrameID(),
+                "parent should be the next frame",
+            )
+            self.assertEqual(
+                parent.pc, frame1.GetPC(), "parent PC should match frame 1"
+            )
+
+    def test_property_child(self):
+        """Test SBFrame extension property: child"""
+        frame0, thread = self._get_frame()
+
+        # Test child property (should be frame -1, which doesn't exist, so should return invalid).
+        child = frame0.child
+        # Child of frame 0 would be frame -1, which doesn't exist.
+        # So it should return an invalid frame.
+        if thread.GetNumFrames() == 1:
+            self.assertFalse(child.IsValid(), "child of only frame should be invalid")
+
+    def test_method_get_all_variables(self):
+        """Test SBFrame extension method: get_all_variables()"""
+        frame, _ = self._get_frame()
+
+        all_vars = frame.get_all_variables()
+        self.assertIsInstance(
+            all_vars, lldb.SBValueList, "get_all_variables should return SBValueList"
+        )
+        all_vars_direct = frame.GetVariables(True, True, True, True)
+        self.assertEqual(
+            all_vars.GetSize(),
+            all_vars_direct.GetSize(),
+            "get_all_variables should match GetVariables(True, True, True, True)",
+        )
+
+    def test_method_get_arguments(self):
+        """Test SBFrame extension method: get_arguments()"""
+        frame, _ = self._get_frame()
+
+        args = frame.get_arguments()
+        self.assertIsInstance(
+            args, lldb.SBValueList, "get_arguments should return SBValueList"
+        )
+        args_direct = frame.GetVariables(True, False, False, False)
+        self.assertEqual(
+            args.GetSize(),
+            args_direct.GetSize(),
+            "get_arguments should match GetVariables(True, False, False, False)",
+        )
+
+    def test_method_get_locals(self):
+        """Test SBFrame extension method: get_locals()"""
+        frame, _ = self._get_frame()
+
+        locals = frame.get_locals()
+        self.assertIsInstance(
+            locals, lldb.SBValueList, "get_locals should return SBValueList"
+        )
+        locals_direct = frame.GetVariables(False, True, False, False)
+        self.assertEqual(
+            locals.GetSize(),
+            locals_direct.GetSize(),
+            "get_locals should match GetVariables(False, True, False, False)",
+        )
+
+    def test_method_get_statics(self):
+        """Test SBFrame extension method: get_statics()"""
+        frame, _ = self._get_frame()
+
+        statics = frame.get_statics()
+        self.assertIsInstance(
+            statics, lldb.SBValueList, "get_statics should return SBValueList"
+        )
+        statics_direct = frame.GetVariables(False, False, True, False)
+        self.assertEqual(
+            statics.GetSize(),
+            statics_direct.GetSize(),
+            "get_statics should match GetVariables(False, False, True, False)",
+        )
+
+    def test_method_var(self):
+        """Test SBFrame extension method: var()"""
+        frame, _ = self._get_frame()
+
+        # Test var() method with a variable that should exist.
+        # First, let's see what variables are available.
+        all_vars = frame.GetVariables(True, True, True, True)
+        if all_vars.GetSize() > 0:
+            var_name = all_vars.GetValueAtIndex(0).GetName()
+            var_value = frame.var(var_name)
+            self.assertTrue(var_value.IsValid(), f"var('{var_name}') should be valid")
+            self.assertEqual(
+                var_value.GetName(),
+                var_name,
+                f"var('{var_name}') should return the correct variable",
+            )
+            # Compare with GetValueForVariablePath.
+            var_direct = frame.GetValueForVariablePath(var_name)
+            self.assertEqual(
+                var_value.GetName(),
+                var_direct.GetName(),
+                "var() should match GetValueForVariablePath()",
+            )
+
+        # Test var() with non-existent variable.
+        invalid_var = frame.var("NonExistentVariable12345")
+        self.assertFalse(
+            invalid_var.IsValid(), "var() with non-existent variable should be invalid"
+        )
+
+    def test_method_get_parent_frame(self):
+        """Test SBFrame extension method: get_parent_frame()"""
+        frame0, thread = self._get_frame()
+
+        # Test get_parent_frame.
+        if thread.GetNumFrames() > 1:
+            parent = frame0.get_parent_frame()
+            self.assertTrue(
+                parent.IsValid(), "get_parent_frame should return valid frame"
+            )
+            frame1 = thread.GetFrameAtIndex(1)
+            self.assertEqual(
+                parent.GetFrameID(),
+                frame1.GetFrameID(),
+                "get_parent_frame should return frame 1",
+            )
+        else:
+            # If there's only one frame, parent should be invalid.
+            parent = frame0.get_parent_frame()
+            # Note: get_parent_frame might return an invalid frame if idx+1 is out of bounds.
+
+    def test_method_get_child_frame(self):
+        """Test SBFrame extension method: get_child_frame()"""
+        frame0, thread = self._get_frame()
+
+        # Test get_child_frame (frame -1 doesn't exist, so should be invalid).
+        child = frame0.get_child_frame()
+        if thread.GetNumFrames() == 1:
+            self.assertFalse(
+                child.IsValid(), "get_child_frame of only frame should be invalid"
+            )
+
+    def test_special_method_int(self):
+        """Test SBFrame extension special method: __int__"""
+        frame0, _ = self._get_frame()
+
+        # Test __int__ (converts frame to its frame ID).
+        frame_id = int(frame0)
+        self.assertIsInstance(frame_id, int, "__int__ should return an integer")
+        self.assertEqual(
+            frame_id, frame0.GetFrameID(), "__int__ should return frame ID"
+        )
+
+    def test_special_method_hex(self):
+        """Test SBFrame extension special method: __hex__"""
+        frame0, _ = self._get_frame()
+
+        # Test __hex__ (converts frame to its PC).
+        # Note: __hex__ returns the PC as an integer, not a hex string.
+        # In Python 3, hex() builtin calls __index__ if __hex__ doesn't exist,
+        # but since __hex__ is defined, it will be called.
+        pc_hex = frame0.__hex__()
+        self.assertIsInstance(pc_hex, int, "__hex__ should return an integer (PC)")
+        self.assertEqual(pc_hex, frame0.GetPC(), "__hex__ should return PC")
+
+    def test_special_method_eq(self):
+        """Test SBFrame extension special method: __eq__ and __ne__"""
+        frame0, thread = self._get_frame()
+
+        # Test __eq__ and __ne__.
+        frame0_copy = thread.GetFrameAtIndex(0)
+        self.assertTrue(frame0 == frame0_copy, "Same frame should be equal")
+        self.assertFalse(frame0 != frame0_copy, "Same frame should not be not-equal")
+
+        if thread.GetNumFrames() > 1:
+            frame1 = thread.GetFrameAtIndex(1)
+            self.assertFalse(frame0 == frame1, "Different frames should not be equal")
+            self.assertTrue(frame0 != frame1, "Different frames should be not-equal")
+
+    def test_pc_property_settable(self):
+        """Test that pc property is settable"""
+        frame, _ = self._get_frame()
+
+        original_pc = frame.GetPC()
+        # Test that we can set pc (though this might not work on all platforms).
+        # We'll just verify the property exists and can be read.
+        pc = frame.pc
+        self.assertIsInstance(pc, int, "pc should be readable")
+        # Note: Setting pc might not be supported on all platforms, so we just test reading.
diff --git a/lldb/test/API/python_api/sbframe_extensions/main.c b/lldb/test/API/python_api/sbframe_extensions/main.c
new file mode 100644
index 0000000000000..8e2d3ed8e5a5f
--- /dev/null
+++ b/lldb/test/API/python_api/sbframe_extensions/main.c
@@ -0,0 +1,33 @@
+#include <stdio.h>
+
+// Global and static variables for testing
+int g_global_var = 42;
+static int g_static_var = 100;
+
+// Function declarations
+int func1(int arg1, char arg2);
+int func2(int arg1, int arg2);
+
+int func1(int arg1, char arg2) {
+  static int static_var = 200;
+  int local1 = arg1 * 2;
+  char local2 = arg2;
+  // Set breakpoint here
+  return local1 + local2 + static_var;
+}
+
+int func2(int arg1, int arg2) {
+  int local1 = arg1 + arg2;
+  int local2 = arg1 * arg2;
+  // Set breakpoint here
+  return func1(local1, 'X');
+}
+
+int main(int argc, char const *argv[]) {
+  int main_local = 10;
+  static int main_static = 50;
+  // Set breakpoint here
+  int result = func2(5, 7);
+  printf("Result: %d\n", result);
+  return 0;
+}
diff --git a/lldb/test/Shell/helper/toolchain.py b/lldb/test/Shell/helper/toolchain.py
index b0d4e272d5646..0c8c39d37e089 100644
--- a/lldb/test/Shell/helper/toolchain.py
+++ b/lldb/test/Shell/helper/toolchain.py
@@ -226,7 +226,7 @@ def use_support_substitutions(config):
         except OSError:
             res = -1
         if res == 0 and out:
-            sdk_path = str(out)
+            sdk_path = out.decode("utf-8")
             llvm_config.lit_config.note("using SDKROOT: %r" % sdk_path)
             host_flags += ["-isysroot", sdk_path]
     elif sys.platform != "win32":
diff --git a/lldb/tools/lldb-dap/Handler/InitializeRequestHandler.cpp b/lldb/tools/lldb-dap/Handler/InitializeRequestHandler.cpp
index 53e1810a5b0e0..2d30e089447f1 100644
--- a/lldb/tools/lldb-dap/Handler/InitializeRequestHandler.cpp
+++ b/lldb/tools/lldb-dap/Handler/InitializeRequestHandler.cpp
@@ -23,7 +23,7 @@ llvm::Expected<InitializeResponse> InitializeRequestHandler::Run(
     const InitializeRequestArguments &arguments) const {
   // Store initialization arguments for later use in Launch/Attach.
   dap.clientFeatures = arguments.supportedFeatures;
-  dap.sourceInitFile = arguments.lldbExtSourceInitFile.value_or(true);
+  dap.sourceInitFile = arguments.lldbExtSourceInitFile;
 
   return dap.GetCapabilities();
 }
diff --git a/lldb/tools/lldb-dap/Protocol/ProtocolRequests.cpp b/lldb/tools/lldb-dap/Protocol/ProtocolRequests.cpp
index d53a520ade39b..0a1d580bffd68 100644
--- a/lldb/tools/lldb-dap/Protocol/ProtocolRequests.cpp
+++ b/lldb/tools/lldb-dap/Protocol/ProtocolRequests.cpp
@@ -216,12 +216,13 @@ bool fromJSON(const json::Value &Params, InitializeRequestArguments &IRA,
   }
 
   return OM.map("adapterID", IRA.adapterID) &&
-         OM.map("clientID", IRA.clientID) &&
-         OM.map("clientName", IRA.clientName) && OM.map("locale", IRA.locale) &&
-         OM.map("linesStartAt1", IRA.linesStartAt1) &&
-         OM.map("columnsStartAt1", IRA.columnsStartAt1) &&
+         OM.mapOptional("clientID", IRA.clientID) &&
+         OM.mapOptional("clientName", IRA.clientName) &&
+         OM.mapOptional("locale", IRA.locale) &&
+         OM.mapOptional("linesStartAt1", IRA.linesStartAt1) &&
+         OM.mapOptional("columnsStartAt1", IRA.columnsStartAt1) &&
          OM.mapOptional("pathFormat", IRA.pathFormat) &&
-         OM.map("$__lldb_sourceInitFile", IRA.lldbExtSourceInitFile);
+         OM.mapOptional("$__lldb_sourceInitFile", IRA.lldbExtSourceInitFile);
 }
 
 bool fromJSON(const json::Value &Params, Configuration &C, json::Path P) {
diff --git a/lldb/tools/lldb-dap/Protocol/ProtocolRequests.h b/lldb/tools/lldb-dap/Protocol/ProtocolRequests.h
index 37fc2465f6a05..6a85033ae7ef2 100644
--- a/lldb/tools/lldb-dap/Protocol/ProtocolRequests.h
+++ b/lldb/tools/lldb-dap/Protocol/ProtocolRequests.h
@@ -108,23 +108,23 @@ struct InitializeRequestArguments {
   std::string adapterID;
 
   /// The ID of the client using this adapter.
-  std::optional<std::string> clientID;
+  std::string clientID;
 
   /// The human-readable name of the client using this adapter.
-  std::optional<std::string> clientName;
+  std::string clientName;
 
   /// The ISO-639 locale of the client using this adapter, e.g. en-US or de-CH.
-  std::optional<std::string> locale;
+  std::string locale;
 
   /// Determines in what format paths are specified. The default is `path`,
   /// which is the native format.
   PathFormat pathFormat = ePatFormatPath;
 
   /// If true all line numbers are 1-based (default).
-  std::optional<bool> linesStartAt1;
+  bool linesStartAt1 = true;
 
   /// If true all column numbers are 1-based (default).
-  std::optional<bool> columnsStartAt1;
+  bool columnsStartAt1 = true;
 
   /// The set of supported features reported by the client.
   llvm::DenseSet<ClientFeature> supportedFeatures;
@@ -133,7 +133,7 @@ struct InitializeRequestArguments {
   /// @{
 
   /// Source init files when initializing lldb::SBDebugger.
-  std::optional<bool> lldbExtSourceInitFile;
+  bool lldbExtSourceInitFile = true;
 
   /// @}
 };
diff --git a/lldb/unittests/DAP/ProtocolRequestsTest.cpp b/lldb/unittests/DAP/ProtocolRequestsTest.cpp
index ba9aef1e5fcc5..a74c369924b8e 100644
--- a/lldb/unittests/DAP/ProtocolRequestsTest.cpp
+++ b/lldb/unittests/DAP/ProtocolRequestsTest.cpp
@@ -77,7 +77,7 @@ TEST(ProtocolRequestsTest, EvaluateArguments) {
   EXPECT_EQ(expected->expression, "hello world");
   EXPECT_EQ(expected->context, eEvaluateContextRepl);
 
-  // Check required keys;
+  // Check required keys.
   EXPECT_THAT_EXPECTED(parse<EvaluateArguments>(R"({})"),
                        FailedWithMessage("missing value at (root).expression"));
 }
@@ -118,3 +118,67 @@ TEST(ProtocolRequestsTest, EvaluateResponseBody) {
   ASSERT_THAT_EXPECTED(expected_opt, llvm::Succeeded());
   EXPECT_EQ(PrettyPrint(*expected_opt), PrettyPrint(body));
 }
+
+TEST(ProtocolRequestsTest, InitializeRequestArguments) {
+  llvm::Expected<InitializeRequestArguments> expected =
+      parse<InitializeRequestArguments>(R"({"adapterID": "myid"})");
+  ASSERT_THAT_EXPECTED(expected, llvm::Succeeded());
+  EXPECT_EQ(expected->adapterID, "myid");
+
+  // Check optional keys.
+  expected = parse<InitializeRequestArguments>(R"({
+    "adapterID": "myid",
+    "clientID": "myclientid",
+    "clientName": "lldb-dap-unit-tests",
+    "locale": "en-US",
+    "linesStartAt1": true,
+    "columnsStartAt1": true,
+    "pathFormat": "uri",
+    "supportsVariableType": true,
+    "supportsVariablePaging": true,
+    "supportsRunInTerminalRequest": true,
+    "supportsMemoryReferences": true,
+    "supportsProgressReporting": true,
+    "supportsInvalidatedEvent": true,
+    "supportsMemoryEvent": true,
+    "supportsArgsCanBeInterpretedByShell": true,
+    "supportsStartDebuggingRequest": true,
+    "supportsANSIStyling": true
+  })");
+  ASSERT_THAT_EXPECTED(expected, llvm::Succeeded());
+  EXPECT_EQ(expected->adapterID, "myid");
+  EXPECT_EQ(expected->clientID, "myclientid");
+  EXPECT_EQ(expected->clientName, "lldb-dap-unit-tests");
+  EXPECT_EQ(expected->locale, "en-US");
+  EXPECT_EQ(expected->linesStartAt1, true);
+  EXPECT_EQ(expected->columnsStartAt1, true);
+  EXPECT_EQ(expected->pathFormat, ePathFormatURI);
+  EXPECT_EQ(expected->supportedFeatures.contains(eClientFeatureVariableType),
+            true);
+  EXPECT_EQ(
+      expected->supportedFeatures.contains(eClientFeatureRunInTerminalRequest),
+      true);
+  EXPECT_EQ(
+      expected->supportedFeatures.contains(eClientFeatureMemoryReferences),
+      true);
+  EXPECT_EQ(
+      expected->supportedFeatures.contains(eClientFeatureProgressReporting),
+      true);
+  EXPECT_EQ(
+      expected->supportedFeatures.contains(eClientFeatureInvalidatedEvent),
+      true);
+  EXPECT_EQ(expected->supportedFeatures.contains(eClientFeatureMemoryEvent),
+            true);
+  EXPECT_EQ(expected->supportedFeatures.contains(
+                eClientFeatureArgsCanBeInterpretedByShell),
+            true);
+  EXPECT_EQ(
+      expected->supportedFeatures.contains(eClientFeatureStartDebuggingRequest),
+      true);
+  EXPECT_EQ(expected->supportedFeatures.contains(eClientFeatureANSIStyling),
+            true);
+
+  // Check required keys.
+  EXPECT_THAT_EXPECTED(parse<InitializeRequestArguments>(R"({})"),
+                       FailedWithMessage("missing value at (root).adapterID"));
+}
diff --git a/lldb/unittests/Expression/DWARFExpressionTest.cpp b/lldb/unittests/Expression/DWARFExpressionTest.cpp
index 0126c408d8696..f264fb3ce94e5 100644
--- a/lldb/unittests/Expression/DWARFExpressionTest.cpp
+++ b/lldb/unittests/Expression/DWARFExpressionTest.cpp
@@ -1217,3 +1217,107 @@ TEST_F(DWARFExpressionMockProcessTestWithAArch, DW_op_deref_no_ptr_fixing) {
   llvm::Expected<Value> result_deref = evaluate_expr(expr_deref);
   EXPECT_THAT_EXPECTED(result_deref, ExpectLoadAddress(expected_value));
 }
+
+TEST_F(DWARFExpressionMockProcessTest, deref_register) {
+  TestContext test_ctx;
+  constexpr uint32_t reg_r0 = 0x504;
+  MockMemory::Map memory = {
+      {{0x004, 4}, {0x1, 0x2, 0x3, 0x4}},
+      {{0x504, 4}, {0xa, 0xb, 0xc, 0xd}},
+      {{0x505, 4}, {0x5, 0x6, 0x7, 0x8}},
+  };
+  ASSERT_TRUE(CreateTestContext(&test_ctx, "i386-pc-linux",
+                                RegisterValue(reg_r0), memory, memory));
+
+  ExecutionContext exe_ctx(test_ctx.process_sp);
+  MockDwarfDelegate delegate = MockDwarfDelegate::Dwarf5();
+  auto Eval = [&](llvm::ArrayRef<uint8_t> expr_data) {
+    ExecutionContext exe_ctx(test_ctx.process_sp);
+    return Evaluate(expr_data, {}, &delegate, &exe_ctx,
+                    test_ctx.reg_ctx_sp.get());
+  };
+
+  // Reads from the register r0.
+  // Sets the context to RegisterInfo so we know this is a register location.
+  EXPECT_THAT_EXPECTED(Eval({DW_OP_reg0}),
+                       ExpectScalar(reg_r0, Value::ContextType::RegisterInfo));
+
+  // Reads from the location(register r0).
+  // Clears the context so we know this is a value not a location.
+  EXPECT_THAT_EXPECTED(Eval({DW_OP_reg0, DW_OP_deref}),
+                       ExpectLoadAddress(reg_r0, Value::ContextType::Invalid));
+
+  // Reads from the location(register r0) and adds the value to the host buffer.
+  // The evaluator should implicitly convert it to a memory location when
+  // added to a composite value and should add the contents of memory[r0]
+  // to the host buffer.
+  EXPECT_THAT_EXPECTED(Eval({DW_OP_reg0, DW_OP_deref, DW_OP_piece, 4}),
+                       ExpectHostAddress({0xa, 0xb, 0xc, 0xd}));
+
+  // Reads from the location(register r0) and truncates the value to one byte.
+  // Clears the context so we know this is a value not a location.
+  EXPECT_THAT_EXPECTED(
+      Eval({DW_OP_reg0, DW_OP_deref_size, 1}),
+      ExpectLoadAddress(reg_r0 & 0xff, Value::ContextType::Invalid));
+
+  // Reads from the location(register r0) and truncates to one byte then adds
+  // the value to the host buffer. The evaluator should implicitly convert it to
+  // a memory location when added to a composite value and should add the
+  // contents of memory[r0 & 0xff] to the host buffer.
+  EXPECT_THAT_EXPECTED(Eval({DW_OP_reg0, DW_OP_deref_size, 1, DW_OP_piece, 4}),
+                       ExpectHostAddress({0x1, 0x2, 0x3, 0x4}));
+
+  // Reads from the register r0 + 1.
+  EXPECT_THAT_EXPECTED(
+      Eval({DW_OP_breg0, 1}),
+      ExpectLoadAddress(reg_r0 + 1, Value::ContextType::Invalid));
+
+  // Reads from address r0 + 1, which contains the bytes [5,6,7,8].
+  EXPECT_THAT_EXPECTED(
+      Eval({DW_OP_breg0, 1, DW_OP_deref}),
+      ExpectLoadAddress(0x08070605, Value::ContextType::Invalid));
+}
+
+TEST_F(DWARFExpressionMockProcessTest, deref_implicit_value) {
+  TestContext test_ctx;
+  MockMemory::Map memory = {
+      {{0x4, 1}, {0x1}},
+      {{0x4, 4}, {0x1, 0x2, 0x3, 0x4}},
+  };
+  ASSERT_TRUE(CreateTestContext(&test_ctx, "i386-pc-linux", {}, memory));
+
+  ExecutionContext exe_ctx(test_ctx.process_sp);
+  MockDwarfDelegate delegate = MockDwarfDelegate::Dwarf5();
+  auto Eval = [&](llvm::ArrayRef<uint8_t> expr_data) {
+    ExecutionContext exe_ctx(test_ctx.process_sp);
+    return Evaluate(expr_data, {}, &delegate, &exe_ctx,
+                    test_ctx.reg_ctx_sp.get());
+  };
+
+  // Creates an implicit location with a value of 4.
+  EXPECT_THAT_EXPECTED(Eval({DW_OP_lit4, DW_OP_stack_value}),
+                       ExpectScalar(0x4));
+
+  // Creates an implicit location with a value of 4. The deref reads the value
+  // out of the location and implicitly converts it to a load address.
+  EXPECT_THAT_EXPECTED(Eval({DW_OP_lit4, DW_OP_stack_value, DW_OP_deref}),
+                       ExpectLoadAddress(0x4));
+
+  // Creates an implicit location with a value of 0x504 (uleb128(0x504) =
+  // 0xa84). The deref reads the low byte out of the location and implicitly
+  // converts it to a load address.
+  EXPECT_THAT_EXPECTED(
+      Eval({DW_OP_constu, 0x84, 0xa, DW_OP_stack_value, DW_OP_deref_size, 1}),
+      ExpectLoadAddress(0x4));
+
+  // The tests below are similar to the ones above, but there is no implicit
+  // location created by a stack_value operation. They are provided here as a
+  // reference to contrast with the above tests.
+  EXPECT_THAT_EXPECTED(Eval({DW_OP_lit4}), ExpectLoadAddress(0x4));
+
+  EXPECT_THAT_EXPECTED(Eval({DW_OP_lit4, DW_OP_deref}),
+                       ExpectLoadAddress(0x04030201));
+
+  EXPECT_THAT_EXPECTED(Eval({DW_OP_lit4, DW_OP_deref_size, 1}),
+                       ExpectLoadAddress(0x01));
+}
diff --git a/lldb/unittests/ScriptInterpreter/Python/PythonTestSuite.cpp b/lldb/unittests/ScriptInterpreter/Python/PythonTestSuite.cpp
index a63b740d9472f..5694aeeff3e5b 100644
--- a/lldb/unittests/ScriptInterpreter/Python/PythonTestSuite.cpp
+++ b/lldb/unittests/ScriptInterpreter/Python/PythonTestSuite.cpp
@@ -136,6 +136,11 @@ lldb_private::python::LLDBSWIGPython_CastPyObjectToSBStream(PyObject *data) {
   return nullptr;
 }
 
+void *
+lldb_private::python::LLDBSWIGPython_CastPyObjectToSBThread(PyObject *data) {
+  return nullptr;
+}
+
 void *
 lldb_private::python::LLDBSWIGPython_CastPyObjectToSBFrame(PyObject *data) {
   return nullptr;
diff --git a/lldb/unittests/Symbol/TestClangASTImporter.cpp b/lldb/unittests/Symbol/TestClangASTImporter.cpp
index f1b3d7911c4bd..07c42088b9101 100644
--- a/lldb/unittests/Symbol/TestClangASTImporter.cpp
+++ b/lldb/unittests/Symbol/TestClangASTImporter.cpp
@@ -287,7 +287,7 @@ TEST_F(TestClangASTImporter, RecordLayoutFromOrigin) {
   clang_utils::SourceASTWithRecord source;
 
   auto *dwarf_parser =
-      static_cast<DWARFASTParserClang *>(source.ast->GetDWARFParser());
+      llvm::cast<DWARFASTParserClang>(source.ast->GetDWARFParser());
   auto &importer = dwarf_parser->GetClangASTImporter();
 
   // Set the layout for the origin decl in the origin ClangASTImporter.
diff --git a/lldb/unittests/SymbolFile/DWARF/DWARFASTParserClangTests.cpp b/lldb/unittests/SymbolFile/DWARF/DWARFASTParserClangTests.cpp
index cef3a25a4a960..642dde9924f3a 100644
--- a/lldb/unittests/SymbolFile/DWARF/DWARFASTParserClangTests.cpp
+++ b/lldb/unittests/SymbolFile/DWARF/DWARFASTParserClangTests.cpp
@@ -42,6 +42,52 @@ class DWARFASTParserClangStub : public DWARFASTParserClang {
     return keys;
   }
 };
+
+/// Helper structure for DWARFASTParserClang tests that want to parse DWARF
+/// generated using yaml2obj. On construction parses the supplied YAML data
+/// into a DWARF module and thereafter vends a DWARFASTParserClang and
+/// TypeSystemClang that are guaranteed to live for the duration of this object.
+class DWARFASTParserClangYAMLTester {
+public:
+  DWARFASTParserClangYAMLTester(llvm::StringRef yaml_data)
+      : m_module_tester(yaml_data) {}
+
+  DWARFDIE GetCUDIE() {
+    DWARFUnit *unit = m_module_tester.GetDwarfUnit();
+    assert(unit);
+
+    const DWARFDebugInfoEntry *cu_entry = unit->DIE().GetDIE();
+    assert(cu_entry->Tag() == DW_TAG_compile_unit);
+
+    return DWARFDIE(unit, cu_entry);
+  }
+
+  DWARFASTParserClang &GetParser() {
+    auto *parser = GetTypeSystem().GetDWARFParser();
+
+    assert(llvm::isa_and_nonnull<DWARFASTParserClang>(parser));
+
+    return *llvm::cast<DWARFASTParserClang>(parser);
+  }
+
+  TypeSystemClang &GetTypeSystem() {
+    ModuleSP module_sp = m_module_tester.GetModule();
+    assert(module_sp);
+
+    SymbolFile *symfile = module_sp->GetSymbolFile();
+    assert(symfile);
+
+    TypeSystemSP ts_sp = llvm::cantFail(
+        symfile->GetTypeSystemForLanguage(lldb::eLanguageTypeC_plus_plus));
+
+    assert(llvm::isa_and_nonnull<TypeSystemClang>(ts_sp.get()));
+
+    return llvm::cast<TypeSystemClang>(*ts_sp);
+  }
+
+private:
+  YAMLModuleTester m_module_tester;
+};
 } // namespace
 
 // If your implementation needs to dereference the dummy pointers we are
@@ -248,17 +294,9 @@ TEST_F(DWARFASTParserClangTests, TestCallingConventionParsing) {
         - AbbrCode:        0x0
 ...
 )";
-  YAMLModuleTester t(yamldata);
+  DWARFASTParserClangYAMLTester tester(yamldata);
 
-  DWARFUnit *unit = t.GetDwarfUnit();
-  ASSERT_NE(unit, nullptr);
-  const DWARFDebugInfoEntry *cu_entry = unit->DIE().GetDIE();
-  ASSERT_EQ(cu_entry->Tag(), DW_TAG_compile_unit);
-  DWARFDIE cu_die(unit, cu_entry);
-
-  auto holder = std::make_unique<clang_utils::TypeSystemClangHolder>("ast");
-  auto &ast_ctx = *holder->GetAST();
-  DWARFASTParserClangStub ast_parser(ast_ctx);
+  DWARFDIE cu_die = tester.GetCUDIE();
 
   std::vector<std::string> found_function_types;
   // The DWARF above is just a list of functions. Parse all of them to
@@ -267,7 +305,8 @@ TEST_F(DWARFASTParserClangTests, TestCallingConventionParsing) {
     ASSERT_EQ(func.Tag(), DW_TAG_subprogram);
     SymbolContext sc;
     bool new_type = false;
-    lldb::TypeSP type = ast_parser.ParseTypeFromDWARF(sc, func, &new_type);
+    lldb::TypeSP type =
+        tester.GetParser().ParseTypeFromDWARF(sc, func, &new_type);
     found_function_types.push_back(
         type->GetForwardCompilerType().GetTypeName().AsCString());
   }
@@ -394,18 +433,9 @@ TEST_F(DWARFASTParserClangTests, TestPtrAuthParsing) {
         - AbbrCode:        0x00 # end of child tags of 0x0c
 ...
 )";
-  YAMLModuleTester t(yamldata);
-
-  DWARFUnit *unit = t.GetDwarfUnit();
-  ASSERT_NE(unit, nullptr);
-  const DWARFDebugInfoEntry *cu_entry = unit->DIE().GetDIE();
-  ASSERT_EQ(cu_entry->Tag(), DW_TAG_compile_unit);
-  DWARFDIE cu_die(unit, cu_entry);
-
-  auto holder = std::make_unique<clang_utils::TypeSystemClangHolder>("ast");
-  auto &ast_ctx = *holder->GetAST();
-  DWARFASTParserClangStub ast_parser(ast_ctx);
+  DWARFASTParserClangYAMLTester tester(yamldata);
 
+  DWARFDIE cu_die = tester.GetCUDIE();
   DWARFDIE ptrauth_variable = cu_die.GetFirstChild();
   ASSERT_EQ(ptrauth_variable.Tag(), DW_TAG_variable);
   DWARFDIE ptrauth_type =
@@ -415,7 +445,7 @@ TEST_F(DWARFASTParserClangTests, TestPtrAuthParsing) {
   SymbolContext sc;
   bool new_type = false;
   lldb::TypeSP type_sp =
-      ast_parser.ParseTypeFromDWARF(sc, ptrauth_type, &new_type);
+      tester.GetParser().ParseTypeFromDWARF(sc, ptrauth_type, &new_type);
   CompilerType compiler_type = type_sp->GetForwardCompilerType();
   ASSERT_EQ(compiler_type.GetPtrAuthKey(), 0U);
   ASSERT_EQ(compiler_type.GetPtrAuthAddressDiversity(), false);
@@ -554,24 +584,17 @@ TEST_F(DWARFASTParserClangTests, TestDefaultTemplateParamParsing) {
   auto BufferOrError = llvm::MemoryBuffer::getFile(
       GetInputFilePath("DW_AT_default_value-test.yaml"), /*IsText=*/true);
   ASSERT_TRUE(BufferOrError);
-  YAMLModuleTester t(BufferOrError.get()->getBuffer());
 
-  DWARFUnit *unit = t.GetDwarfUnit();
-  ASSERT_NE(unit, nullptr);
-  const DWARFDebugInfoEntry *cu_entry = unit->DIE().GetDIE();
-  ASSERT_EQ(cu_entry->Tag(), DW_TAG_compile_unit);
-  DWARFDIE cu_die(unit, cu_entry);
-
-  auto holder = std::make_unique<clang_utils::TypeSystemClangHolder>("ast");
-  auto &ast_ctx = *holder->GetAST();
-  DWARFASTParserClangStub ast_parser(ast_ctx);
+  DWARFASTParserClangYAMLTester tester(BufferOrError.get()->getBuffer());
+  DWARFDIE cu_die = tester.GetCUDIE();
 
   llvm::SmallVector<lldb::TypeSP, 2> types;
   for (DWARFDIE die : cu_die.children()) {
     if (die.Tag() == DW_TAG_class_type) {
       SymbolContext sc;
       bool new_type = false;
-      types.push_back(ast_parser.ParseTypeFromDWARF(sc, die, &new_type));
+      types.push_back(
+          tester.GetParser().ParseTypeFromDWARF(sc, die, &new_type));
     }
   }
 
@@ -605,23 +628,14 @@ TEST_F(DWARFASTParserClangTests, TestSpecDeclExistsError) {
   auto BufferOrError = llvm::MemoryBuffer::getFile(
       GetInputFilePath("DW_AT_spec_decl_exists-test.yaml"), /*IsText=*/true);
   ASSERT_TRUE(BufferOrError);
-  YAMLModuleTester t(BufferOrError.get()->getBuffer());
-
-  DWARFUnit *unit = t.GetDwarfUnit();
-  ASSERT_NE(unit, nullptr);
-  const DWARFDebugInfoEntry *cu_entry = unit->DIE().GetDIE();
-  ASSERT_EQ(cu_entry->Tag(), DW_TAG_compile_unit);
-  DWARFDIE cu_die(unit, cu_entry);
-
-  auto holder = std::make_unique<clang_utils::TypeSystemClangHolder>("ast");
-  auto &ast_ctx = *holder->GetAST();
-  DWARFASTParserClangStub ast_parser(ast_ctx);
+  DWARFASTParserClangYAMLTester tester(BufferOrError.get()->getBuffer());
+  DWARFDIE cu_die = tester.GetCUDIE();
 
   llvm::SmallVector<lldb::TypeSP, 2> specializations;
   for (DWARFDIE die : cu_die.children()) {
     SymbolContext sc;
     bool new_type = false;
-    auto type = ast_parser.ParseTypeFromDWARF(sc, die, &new_type);
+    auto type = tester.GetParser().ParseTypeFromDWARF(sc, die, &new_type);
     llvm::StringRef die_name = llvm::StringRef(die.GetName());
     if (die_name.starts_with("_Optional_payload")) {
       specializations.push_back(std::move(type));
@@ -730,18 +744,8 @@ TEST_F(DWARFASTParserClangTests, TestUniqueDWARFASTTypeMap_CppInsertMapFind) {
         - AbbrCode:        0x00 # end of child tags of 0x0c
 ...
 )";
-  YAMLModuleTester t(yamldata);
-
-  DWARFUnit *unit = t.GetDwarfUnit();
-  ASSERT_NE(unit, nullptr);
-  const DWARFDebugInfoEntry *cu_entry = unit->DIE().GetDIE();
-  ASSERT_EQ(cu_entry->Tag(), DW_TAG_compile_unit);
-  ASSERT_EQ(unit->GetDWARFLanguageType(), DW_LANG_C_plus_plus);
-  DWARFDIE cu_die(unit, cu_entry);
-
-  auto holder = std::make_unique<clang_utils::TypeSystemClangHolder>("ast");
-  auto &ast_ctx = *holder->GetAST();
-  DWARFASTParserClangStub ast_parser(ast_ctx);
+  DWARFASTParserClangYAMLTester tester(yamldata);
+  DWARFDIE cu_die = tester.GetCUDIE();
 
   DWARFDIE decl_die;
   DWARFDIE def_die;
@@ -762,6 +766,8 @@ TEST_F(DWARFASTParserClangTests, TestUniqueDWARFASTTypeMap_CppInsertMapFind) {
   ParsedDWARFTypeAttributes attrs(def_die);
   ASSERT_TRUE(attrs.decl.IsValid());
 
+  DWARFASTParserClang &ast_parser = tester.GetParser();
+
   SymbolContext sc;
   bool new_type = false;
   lldb::TypeSP type_sp = ast_parser.ParseTypeFromDWARF(sc, decl_die, &new_type);
@@ -906,18 +912,8 @@ TEST_F(DWARFASTParserClangTests, TestObjectPointer) {
         - AbbrCode: 0x0
 ...
 )";
-  YAMLModuleTester t(yamldata);
-
-  DWARFUnit *unit = t.GetDwarfUnit();
-  ASSERT_NE(unit, nullptr);
-  const DWARFDebugInfoEntry *cu_entry = unit->DIE().GetDIE();
-  ASSERT_EQ(cu_entry->Tag(), DW_TAG_compile_unit);
-  ASSERT_EQ(unit->GetDWARFLanguageType(), DW_LANG_C_plus_plus);
-  DWARFDIE cu_die(unit, cu_entry);
-
-  auto holder = std::make_unique<clang_utils::TypeSystemClangHolder>("ast");
-  auto &ast_ctx = *holder->GetAST();
-  DWARFASTParserClangStub ast_parser(ast_ctx);
+  DWARFASTParserClangYAMLTester tester(yamldata);
+  DWARFDIE cu_die = tester.GetCUDIE();
 
   auto context_die = cu_die.GetFirstChild();
   ASSERT_TRUE(context_die.IsValid());
@@ -932,7 +928,8 @@ TEST_F(DWARFASTParserClangTests, TestObjectPointer) {
     auto param_die = decl_die.GetFirstChild();
     ASSERT_TRUE(param_die.IsValid());
 
-    EXPECT_EQ(param_die, ast_parser.GetObjectParameter(decl_die, context_die));
+    EXPECT_EQ(param_die,
+              tester.GetParser().GetObjectParameter(decl_die, context_die));
   }
 
   {
@@ -945,8 +942,8 @@ TEST_F(DWARFASTParserClangTests, TestObjectPointer) {
     auto param_die = subprogram_definition.GetFirstChild();
     ASSERT_TRUE(param_die.IsValid());
 
-    EXPECT_EQ(param_die, ast_parser.GetObjectParameter(subprogram_definition,
-                                                       context_die));
+    EXPECT_EQ(param_die, tester.GetParser().GetObjectParameter(
+                             subprogram_definition, context_die));
   }
 }
 
@@ -1076,18 +1073,8 @@ TEST_F(DWARFASTParserClangTests,
         - AbbrCode: 0x0
 ...
 )";
-  YAMLModuleTester t(yamldata);
-
-  DWARFUnit *unit = t.GetDwarfUnit();
-  ASSERT_NE(unit, nullptr);
-  const DWARFDebugInfoEntry *cu_entry = unit->DIE().GetDIE();
-  ASSERT_EQ(cu_entry->Tag(), DW_TAG_compile_unit);
-  ASSERT_EQ(unit->GetDWARFLanguageType(), DW_LANG_C_plus_plus);
-  DWARFDIE cu_die(unit, cu_entry);
-
-  auto holder = std::make_unique<clang_utils::TypeSystemClangHolder>("ast");
-  auto &ast_ctx = *holder->GetAST();
-  DWARFASTParserClangStub ast_parser(ast_ctx);
+  DWARFASTParserClangYAMLTester tester(yamldata);
+  DWARFDIE cu_die = tester.GetCUDIE();
 
   auto context_die = cu_die.GetFirstChild();
   ASSERT_TRUE(context_die.IsValid());
@@ -1105,7 +1092,7 @@ TEST_F(DWARFASTParserClangTests,
   auto param_die = subprogram_definition.GetFirstChild();
   ASSERT_TRUE(param_die.IsValid());
   EXPECT_EQ(param_die,
-            ast_parser.GetObjectParameter(subprogram_definition, {}));
+            tester.GetParser().GetObjectParameter(subprogram_definition, {}));
 }
 
 TEST_F(DWARFASTParserClangTests, TestParseSubroutine_ExplicitObjectParameter) {
@@ -1243,21 +1230,15 @@ TEST_F(DWARFASTParserClangTests, TestParseSubroutine_ExplicitObjectParameter) {
         - AbbrCode: 0x0
 ...
 )";
-  YAMLModuleTester t(yamldata);
-
-  DWARFUnit *unit = t.GetDwarfUnit();
-  ASSERT_NE(unit, nullptr);
-  const DWARFDebugInfoEntry *cu_entry = unit->DIE().GetDIE();
-  ASSERT_EQ(cu_entry->Tag(), DW_TAG_compile_unit);
-  ASSERT_EQ(unit->GetDWARFLanguageType(), DW_LANG_C_plus_plus);
-  DWARFDIE cu_die(unit, cu_entry);
+  DWARFASTParserClangYAMLTester tester(yamldata);
+  DWARFDIE cu_die = tester.GetCUDIE();
 
   auto ts_or_err =
       cu_die.GetDWARF()->GetTypeSystemForLanguage(eLanguageTypeC_plus_plus);
   ASSERT_TRUE(static_cast<bool>(ts_or_err));
   llvm::consumeError(ts_or_err.takeError());
   auto *parser =
-      static_cast<DWARFASTParserClang *>((*ts_or_err)->GetDWARFParser());
+      llvm::cast<DWARFASTParserClang>((*ts_or_err)->GetDWARFParser());
 
   auto context_die = cu_die.GetFirstChild();
   ASSERT_TRUE(context_die.IsValid());
@@ -1419,22 +1400,8 @@ TEST_F(DWARFASTParserClangTests, TestParseSubroutine_ParameterCreation) {
         - AbbrCode: 0x0
 ...
 )";
-  YAMLModuleTester t(yamldata);
-
-  DWARFUnit *unit = t.GetDwarfUnit();
-  ASSERT_NE(unit, nullptr);
-  const DWARFDebugInfoEntry *cu_entry = unit->DIE().GetDIE();
-  ASSERT_EQ(cu_entry->Tag(), DW_TAG_compile_unit);
-  ASSERT_EQ(unit->GetDWARFLanguageType(), DW_LANG_C_plus_plus);
-  DWARFDIE cu_die(unit, cu_entry);
-
-  auto ts_or_err =
-      cu_die.GetDWARF()->GetTypeSystemForLanguage(eLanguageTypeC_plus_plus);
-  ASSERT_TRUE(static_cast<bool>(ts_or_err));
-  llvm::consumeError(ts_or_err.takeError());
-
-  auto *ts = static_cast<TypeSystemClang *>(ts_or_err->get());
-  auto *parser = static_cast<DWARFASTParserClang *>(ts->GetDWARFParser());
+  DWARFASTParserClangYAMLTester tester(yamldata);
+  DWARFDIE cu_die = tester.GetCUDIE();
 
   auto subprogram = cu_die.GetFirstChild();
   ASSERT_TRUE(subprogram.IsValid());
@@ -1442,11 +1409,13 @@ TEST_F(DWARFASTParserClangTests, TestParseSubroutine_ParameterCreation) {
 
   SymbolContext sc;
   bool new_type;
-  auto type_sp = parser->ParseTypeFromDWARF(sc, subprogram, &new_type);
+  auto type_sp =
+      tester.GetParser().ParseTypeFromDWARF(sc, subprogram, &new_type);
   ASSERT_NE(type_sp, nullptr);
 
-  auto result = ts->GetTranslationUnitDecl()->lookup(
-      clang_utils::getDeclarationName(*ts, "func"));
+  TypeSystemClang &ts = tester.GetTypeSystem();
+  auto result = ts.GetTranslationUnitDecl()->lookup(
+      clang_utils::getDeclarationName(ts, "func"));
   ASSERT_TRUE(result.isSingleResult());
 
   auto const *func = llvm::cast<clang::FunctionDecl>(result.front());
@@ -1609,19 +1578,8 @@ TEST_F(DWARFASTParserClangTests, TestObjectPointer_IndexEncoding) {
         - AbbrCode: 0x0
 ...
 )";
-
-  YAMLModuleTester t(yamldata);
-
-  DWARFUnit *unit = t.GetDwarfUnit();
-  ASSERT_NE(unit, nullptr);
-  const DWARFDebugInfoEntry *cu_entry = unit->DIE().GetDIE();
-  ASSERT_EQ(cu_entry->Tag(), DW_TAG_compile_unit);
-  ASSERT_EQ(unit->GetDWARFLanguageType(), DW_LANG_C_plus_plus);
-  DWARFDIE cu_die(unit, cu_entry);
-
-  auto holder = std::make_unique<clang_utils::TypeSystemClangHolder>("ast");
-  auto &ast_ctx = *holder->GetAST();
-  DWARFASTParserClangStub ast_parser(ast_ctx);
+  DWARFASTParserClangYAMLTester tester(yamldata);
+  DWARFDIE cu_die = tester.GetCUDIE();
 
   auto context_die = cu_die.GetFirstChild();
   ASSERT_TRUE(context_die.IsValid());
@@ -1640,7 +1598,8 @@ TEST_F(DWARFASTParserClangTests, TestObjectPointer_IndexEncoding) {
     auto param_die = sub1.GetFirstChild().GetSibling();
     ASSERT_TRUE(param_die.IsValid());
 
-    EXPECT_EQ(param_die, ast_parser.GetObjectParameter(sub1, context_die));
+    EXPECT_EQ(param_die,
+              tester.GetParser().GetObjectParameter(sub1, context_die));
   }
 
   // Object parameter is at constant index 0
@@ -1648,7 +1607,8 @@ TEST_F(DWARFASTParserClangTests, TestObjectPointer_IndexEncoding) {
     auto param_die = sub2.GetFirstChild();
     ASSERT_TRUE(param_die.IsValid());
 
-    EXPECT_EQ(param_die, ast_parser.GetObjectParameter(sub2, context_die));
+    EXPECT_EQ(param_die,
+              tester.GetParser().GetObjectParameter(sub2, context_die));
   }
 }
 
@@ -1711,19 +1671,8 @@ TEST_F(DWARFASTParserClangTests, TestTypeBitSize) {
             - Value: 0x02
 ...
 )";
-
-  YAMLModuleTester t(yamldata);
-
-  DWARFUnit *unit = t.GetDwarfUnit();
-  ASSERT_NE(unit, nullptr);
-  const DWARFDebugInfoEntry *cu_entry = unit->DIE().GetDIE();
-  ASSERT_EQ(cu_entry->Tag(), DW_TAG_compile_unit);
-  ASSERT_EQ(unit->GetDWARFLanguageType(), DW_LANG_C_plus_plus);
-  DWARFDIE cu_die(unit, cu_entry);
-
-  auto holder = std::make_unique<clang_utils::TypeSystemClangHolder>("ast");
-  auto &ast_ctx = *holder->GetAST();
-  DWARFASTParserClangStub ast_parser(ast_ctx);
+  DWARFASTParserClangYAMLTester tester(yamldata);
+  DWARFDIE cu_die = tester.GetCUDIE();
 
   auto type_die = cu_die.GetFirstChild();
   ASSERT_TRUE(type_die.IsValid());
@@ -1734,8 +1683,8 @@ TEST_F(DWARFASTParserClangTests, TestTypeBitSize) {
   EXPECT_EQ(attrs.data_bit_size.value_or(0), 2U);
 
   SymbolContext sc;
-  auto type_sp =
-      ast_parser.ParseTypeFromDWARF(sc, type_die, /*type_is_new_ptr=*/nullptr);
+  auto type_sp = tester.GetParser().ParseTypeFromDWARF(
+      sc, type_die, /*type_is_new_ptr=*/nullptr);
   ASSERT_NE(type_sp, nullptr);
 
   EXPECT_EQ(llvm::expectedToOptional(type_sp->GetByteSize(nullptr)).value_or(0),
@@ -1857,27 +1806,17 @@ TEST_F(DWARFASTParserClangTests, TestBitIntParsing) {
 ...
 
 )";
-
-  YAMLModuleTester t(yamldata);
-
-  DWARFUnit *unit = t.GetDwarfUnit();
-  ASSERT_NE(unit, nullptr);
-  const DWARFDebugInfoEntry *cu_entry = unit->DIE().GetDIE();
-  ASSERT_EQ(cu_entry->Tag(), DW_TAG_compile_unit);
-  ASSERT_EQ(unit->GetDWARFLanguageType(), DW_LANG_C_plus_plus);
-  DWARFDIE cu_die(unit, cu_entry);
-
-  auto holder = std::make_unique<clang_utils::TypeSystemClangHolder>("ast");
-  auto &ast_ctx = *holder->GetAST();
-  DWARFASTParserClangStub ast_parser(ast_ctx);
+  DWARFASTParserClangYAMLTester tester(yamldata);
+  DWARFDIE cu_die = tester.GetCUDIE();
 
   auto type_die = cu_die.GetFirstChild();
   ASSERT_TRUE(type_die.IsValid());
 
   {
     SymbolContext sc;
-    auto type_sp = ast_parser.ParseTypeFromDWARF(sc, type_die,
-                                                 /*type_is_new_ptr=*/nullptr);
+    auto type_sp =
+        tester.GetParser().ParseTypeFromDWARF(sc, type_die,
+                                              /*type_is_new_ptr=*/nullptr);
     ASSERT_NE(type_sp, nullptr);
 
     EXPECT_EQ(
@@ -1891,8 +1830,9 @@ TEST_F(DWARFASTParserClangTests, TestBitIntParsing) {
   {
     type_die = type_die.GetSibling();
     SymbolContext sc;
-    auto type_sp = ast_parser.ParseTypeFromDWARF(sc, type_die,
-                                                 /*type_is_new_ptr=*/nullptr);
+    auto type_sp =
+        tester.GetParser().ParseTypeFromDWARF(sc, type_die,
+                                              /*type_is_new_ptr=*/nullptr);
     ASSERT_NE(type_sp, nullptr);
 
     EXPECT_EQ(
@@ -1906,8 +1846,9 @@ TEST_F(DWARFASTParserClangTests, TestBitIntParsing) {
   {
     type_die = type_die.GetSibling();
     SymbolContext sc;
-    auto type_sp = ast_parser.ParseTypeFromDWARF(sc, type_die,
-                                                 /*type_is_new_ptr=*/nullptr);
+    auto type_sp =
+        tester.GetParser().ParseTypeFromDWARF(sc, type_die,
+                                              /*type_is_new_ptr=*/nullptr);
     ASSERT_NE(type_sp, nullptr);
 
     EXPECT_EQ(
@@ -1922,8 +1863,9 @@ TEST_F(DWARFASTParserClangTests, TestBitIntParsing) {
   {
     type_die = type_die.GetSibling();
     SymbolContext sc;
-    auto type_sp = ast_parser.ParseTypeFromDWARF(sc, type_die,
-                                                 /*type_is_new_ptr=*/nullptr);
+    auto type_sp =
+        tester.GetParser().ParseTypeFromDWARF(sc, type_die,
+                                              /*type_is_new_ptr=*/nullptr);
     ASSERT_NE(type_sp, nullptr);
 
     EXPECT_EQ(
@@ -1938,8 +1880,9 @@ TEST_F(DWARFASTParserClangTests, TestBitIntParsing) {
   {
     type_die = type_die.GetSibling();
     SymbolContext sc;
-    auto type_sp = ast_parser.ParseTypeFromDWARF(sc, type_die,
-                                                 /*type_is_new_ptr=*/nullptr);
+    auto type_sp =
+        tester.GetParser().ParseTypeFromDWARF(sc, type_die,
+                                              /*type_is_new_ptr=*/nullptr);
     ASSERT_NE(type_sp, nullptr);
 
     EXPECT_EQ(
diff --git a/llvm/include/llvm/ADT/SetVector.h b/llvm/include/llvm/ADT/SetVector.h
index 0fde14126c79b..4d0a20f4f95f7 100644
--- a/llvm/include/llvm/ADT/SetVector.h
+++ b/llvm/include/llvm/ADT/SetVector.h
@@ -39,8 +39,7 @@ namespace llvm {
 ///
 /// The key and value types are derived from the Set and Vector types
 /// respectively. This allows the vector-type operations and set-type operations
-/// to have different types. In particular, this is useful when storing pointers
-/// as "Foo *" values but looking them up as "const Foo *" keys.
+/// to have different types.
 ///
 /// No constraint is placed on the key and value types, although it is assumed
 /// that value_type can be converted into key_type for insertion. Users must be
@@ -60,6 +59,9 @@ class SetVector {
   // excessively long linear scans from occuring.
   static_assert(N <= 32, "Small size should be less than or equal to 32!");
 
+  using const_arg_type =
+      typename const_pointer_or_const_ref<typename Set::key_type>::type;
+
 public:
   using value_type = typename Vector::value_type;
   using key_type = typename Set::key_type;
@@ -247,17 +249,17 @@ class SetVector {
   }
 
   /// Check if the SetVector contains the given key.
-  [[nodiscard]] bool contains(const key_type &key) const {
+  [[nodiscard]] bool contains(const_arg_type key) const {
     if constexpr (canBeSmall())
       if (isSmall())
         return is_contained(vector_, key);
 
-    return set_.find(key) != set_.end();
+    return is_contained(set_, key);
   }
 
   /// Count the number of elements of a given key in the SetVector.
   /// \returns 0 if the element is not in the SetVector, 1 if it is.
-  [[nodiscard]] size_type count(const key_type &key) const {
+  [[nodiscard]] size_type count(const_arg_type key) const {
     return contains(key) ? 1 : 0;
   }
 
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/InstructionSelector.h b/llvm/include/llvm/CodeGen/GlobalISel/InstructionSelector.h
index 569407963695e..483afb426fa10 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/InstructionSelector.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/InstructionSelector.h
@@ -35,9 +35,6 @@ class LLVM_ABI InstructionSelector : public GIMatchTableExecutor {
   ///       !isPreISelGenericOpcode(I.getOpcode())
   virtual bool select(MachineInstr &I) = 0;
 
-  // FIXME: Eliminate dependency on TargetPassConfig for NewPM transition
-  const TargetPassConfig *TPC = nullptr;
-
   MachineOptimizationRemarkEmitter *MORE = nullptr;
 
   /// Note: InstructionSelect does not track changed instructions.
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/RegBankSelect.h b/llvm/include/llvm/CodeGen/GlobalISel/RegBankSelect.h
index 076c70d21bbdf..6060bb6144c62 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/RegBankSelect.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/RegBankSelect.h
@@ -510,9 +510,6 @@ class RegBankSelect : public MachineFunctionPass {
   /// Optimization mode of the pass.
   Mode OptMode;
 
-  /// Current target configuration. Controls how the pass handles errors.
-  const TargetPassConfig *TPC;
-
   /// Assign the register bank of each operand of \p MI.
   /// \return True on success, false otherwise.
   bool assignInstr(MachineInstr &MI);
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/Utils.h b/llvm/include/llvm/CodeGen/GlobalISel/Utils.h
index e1aa8eceefd3f..da2742e089f8f 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/Utils.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/Utils.h
@@ -155,12 +155,10 @@ LLVM_ABI bool isTriviallyDead(const MachineInstr &MI,
 /// Report an ISel error as a missed optimization remark to the LLVMContext's
 /// diagnostic stream.  Set the FailedISel MachineFunction property.
 LLVM_ABI void reportGISelFailure(MachineFunction &MF,
-                                 const TargetPassConfig &TPC,
                                  MachineOptimizationRemarkEmitter &MORE,
                                  MachineOptimizationRemarkMissed &R);
 
 LLVM_ABI void reportGISelFailure(MachineFunction &MF,
-                                 const TargetPassConfig &TPC,
                                  MachineOptimizationRemarkEmitter &MORE,
                                  const char *PassName, StringRef Msg,
                                  const MachineInstr &MI);
@@ -168,7 +166,6 @@ LLVM_ABI void reportGISelFailure(MachineFunction &MF,
 /// Report an ISel warning as a missed optimization remark to the LLVMContext's
 /// diagnostic stream.
 LLVM_ABI void reportGISelWarning(MachineFunction &MF,
-                                 const TargetPassConfig &TPC,
                                  MachineOptimizationRemarkEmitter &MORE,
                                  MachineOptimizationRemarkMissed &R);
 
diff --git a/llvm/include/llvm/CodeGen/TargetLowering.h b/llvm/include/llvm/CodeGen/TargetLowering.h
index b2697c81fd825..149366c69bdcc 100644
--- a/llvm/include/llvm/CodeGen/TargetLowering.h
+++ b/llvm/include/llvm/CodeGen/TargetLowering.h
@@ -1243,7 +1243,7 @@ class LLVM_ABI TargetLoweringBase {
   /// to a MemIntrinsicNode (touches memory). If this is the case, it returns
   /// true and store the intrinsic information into the IntrinsicInfo that was
   /// passed to the function.
-  virtual bool getTgtMemIntrinsic(IntrinsicInfo &, const CallInst &,
+  virtual bool getTgtMemIntrinsic(IntrinsicInfo &, const CallBase &,
                                   MachineFunction &,
                                   unsigned /*Intrinsic*/) const {
     return false;
diff --git a/llvm/include/llvm/IR/IntrinsicsARM.td b/llvm/include/llvm/IR/IntrinsicsARM.td
index 3787e2591a4c1..3b475c8d5614d 100644
--- a/llvm/include/llvm/IR/IntrinsicsARM.td
+++ b/llvm/include/llvm/IR/IntrinsicsARM.td
@@ -972,6 +972,13 @@ def int_arm_mve_vmaxnma_predicated: DefaultAttrsIntrinsic<[llvm_anyvector_ty],
    [LLVMMatchType<0>, LLVMMatchType<0>, llvm_anyvector_ty],
     [IntrNoMem]>;
 
+def int_arm_mve_vminnm: DefaultAttrsIntrinsic<[llvm_anyvector_ty],
+   [LLVMMatchType<0>, LLVMMatchType<0>],
+    [IntrNoMem]>;
+def int_arm_mve_vmaxnm: DefaultAttrsIntrinsic<[llvm_anyvector_ty],
+   [LLVMMatchType<0>, LLVMMatchType<0>],
+    [IntrNoMem]>;
+
 multiclass MVEPredicated<list<LLVMType> rets, list<LLVMType> params,
                          LLVMType pred = llvm_anyvector_ty,
                          list<IntrinsicProperty> props = [IntrNoMem],
diff --git a/llvm/include/llvm/Transforms/Utils/Local.h b/llvm/include/llvm/Transforms/Utils/Local.h
index 817fffdde2642..7ceed39ad1859 100644
--- a/llvm/include/llvm/Transforms/Utils/Local.h
+++ b/llvm/include/llvm/Transforms/Utils/Local.h
@@ -431,7 +431,7 @@ LLVM_ABI void combineAAMetadata(Instruction *K, const Instruction *J);
 
 /// Copy the metadata from the source instruction to the destination (the
 /// replacement for the source instruction).
-LLVM_ABI void copyMetadataForAccess(Instruction &Dest, Instruction &Source);
+LLVM_ABI void copyMetadataForLoad(LoadInst &Dest, const LoadInst &Source);
 
 /// Patch the replacement so that it is not more restrictive than the value
 /// being replaced. It assumes that the replacement does not get moved from
diff --git a/llvm/lib/Analysis/ScalarEvolution.cpp b/llvm/lib/Analysis/ScalarEvolution.cpp
index 60d3535ea6097..5f6718d6cbcd8 100644
--- a/llvm/lib/Analysis/ScalarEvolution.cpp
+++ b/llvm/lib/Analysis/ScalarEvolution.cpp
@@ -13998,7 +13998,15 @@ static void PrintLoopInfo(raw_ostream &OS, ScalarEvolution *SE,
 }
 
 namespace llvm {
-raw_ostream &operator<<(raw_ostream &OS, ScalarEvolution::LoopDisposition LD) {
+// Note: these overloaded operators need to be in the llvm namespace for them
+// to be resolved correctly. If we put them outside the llvm namespace, the
+//
+// OS << ": " << SE.getLoopDisposition(SV, InnerL);
+//
+// code below "breaks" and start printing raw enum values as opposed to the
+// string values.
+static raw_ostream &operator<<(raw_ostream &OS,
+                               ScalarEvolution::LoopDisposition LD) {
   switch (LD) {
   case ScalarEvolution::LoopVariant:
     OS << "Variant";
@@ -14013,7 +14021,8 @@ raw_ostream &operator<<(raw_ostream &OS, ScalarEvolution::LoopDisposition LD) {
   return OS;
 }
 
-raw_ostream &operator<<(raw_ostream &OS, ScalarEvolution::BlockDisposition BD) {
+static raw_ostream &operator<<(raw_ostream &OS,
+                               llvm::ScalarEvolution::BlockDisposition BD) {
   switch (BD) {
   case ScalarEvolution::DoesNotDominateBlock:
     OS << "DoesNotDominate";
diff --git a/llvm/lib/Analysis/ScalarEvolutionDivision.cpp b/llvm/lib/Analysis/ScalarEvolutionDivision.cpp
index bce41f9f5329e..4e422539ff9f6 100644
--- a/llvm/lib/Analysis/ScalarEvolutionDivision.cpp
+++ b/llvm/lib/Analysis/ScalarEvolutionDivision.cpp
@@ -29,8 +29,6 @@ class Type;
 
 using namespace llvm;
 
-namespace {
-
 static inline int sizeOfSCEV(const SCEV *S) {
   struct FindSCEVSize {
     int Size = 0;
@@ -52,8 +50,6 @@ static inline int sizeOfSCEV(const SCEV *S) {
   return F.Size;
 }
 
-} // namespace
-
 // Computes the Quotient and Remainder of the division of Numerator by
 // Denominator.
 void SCEVDivision::divide(ScalarEvolution &SE, const SCEV *Numerator,
diff --git a/llvm/lib/Analysis/ValueTracking.cpp b/llvm/lib/Analysis/ValueTracking.cpp
index dbceb8e557849..eb8650fd0eb60 100644
--- a/llvm/lib/Analysis/ValueTracking.cpp
+++ b/llvm/lib/Analysis/ValueTracking.cpp
@@ -5877,6 +5877,12 @@ void computeKnownFPClass(const Value *V, const APInt &DemandedElts,
     break;
   }
   case Instruction::ShuffleVector: {
+    // Handle vector splat idiom
+    if (Value *Splat = getSplatValue(V)) {
+      computeKnownFPClass(Splat, Known, InterestedClasses, Q, Depth + 1);
+      break;
+    }
+
     // For undef elements, we don't know anything about the common state of
     // the shuffle result.
     APInt DemandedLHS, DemandedRHS;
diff --git a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
index 88e554244c663..ad821231035f0 100644
--- a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
@@ -111,17 +111,18 @@ INITIALIZE_PASS_END(IRTranslator, DEBUG_TYPE, "IRTranslator LLVM IR -> MI",
                 false, false)
 
 static void reportTranslationError(MachineFunction &MF,
-                                   const TargetPassConfig &TPC,
                                    OptimizationRemarkEmitter &ORE,
                                    OptimizationRemarkMissed &R) {
   MF.getProperties().setFailedISel();
+  bool IsGlobalISelAbortEnabled =
+      MF.getTarget().Options.GlobalISelAbort == GlobalISelAbortMode::Enable;
 
   // Print the function name explicitly if we don't have a debug location (which
   // makes the diagnostic less useful) or if we're going to emit a raw error.
-  if (!R.getLocation().isValid() || TPC.isGlobalISelAbortEnabled())
+  if (!R.getLocation().isValid() || IsGlobalISelAbortEnabled)
     R << (" (in function: " + MF.getName() + ")").str();
 
-  if (TPC.isGlobalISelAbortEnabled())
+  if (IsGlobalISelAbortEnabled)
     report_fatal_error(Twine(R.getMsg()));
   else
     ORE.emit(R);
@@ -242,7 +243,7 @@ ArrayRef<Register> IRTranslator::getOrCreateVRegs(const Value &Val) {
                                  MF->getFunction().getSubprogram(),
                                  &MF->getFunction().getEntryBlock());
       R << "unable to translate constant: " << ore::NV("Type", Val.getType());
-      reportTranslationError(*MF, *TPC, *ORE, R);
+      reportTranslationError(*MF, *ORE, R);
       return *VRegs;
     }
   }
@@ -279,7 +280,7 @@ Align IRTranslator::getMemOpAlign(const Instruction &I) {
 
   OptimizationRemarkMissed R("gisel-irtranslator", "", &I);
   R << "unable to translate memop: " << ore::NV("Opcode", &I);
-  reportTranslationError(*MF, *TPC, *ORE, R);
+  reportTranslationError(*MF, *ORE, R);
   return Align(1);
 }
 
@@ -4150,7 +4151,7 @@ bool IRTranslator::runOnMachineFunction(MachineFunction &CurMF) {
     OptimizationRemarkMissed R("gisel-irtranslator", "GISelFailure",
                                F.getSubprogram(), &F.getEntryBlock());
     R << "unable to translate in big endian mode";
-    reportTranslationError(*MF, *TPC, *ORE, R);
+    reportTranslationError(*MF, *ORE, R);
     return false;
   }
 
@@ -4194,7 +4195,7 @@ bool IRTranslator::runOnMachineFunction(MachineFunction &CurMF) {
                                F.getSubprogram(), &F.getEntryBlock());
     R << "unable to lower function: "
       << ore::NV("Prototype", F.getFunctionType());
-    reportTranslationError(*MF, *TPC, *ORE, R);
+    reportTranslationError(*MF, *ORE, R);
     return false;
   }
 
@@ -4217,7 +4218,7 @@ bool IRTranslator::runOnMachineFunction(MachineFunction &CurMF) {
                                F.getSubprogram(), &F.getEntryBlock());
     R << "unable to lower arguments: "
       << ore::NV("Prototype", F.getFunctionType());
-    reportTranslationError(*MF, *TPC, *ORE, R);
+    reportTranslationError(*MF, *ORE, R);
     return false;
   }
 
@@ -4268,7 +4269,7 @@ bool IRTranslator::runOnMachineFunction(MachineFunction &CurMF) {
           R << ": '" << InstStrStorage << "'";
         }
 
-        reportTranslationError(*MF, *TPC, *ORE, R);
+        reportTranslationError(*MF, *ORE, R);
         return false;
       }
 
@@ -4276,7 +4277,7 @@ bool IRTranslator::runOnMachineFunction(MachineFunction &CurMF) {
         OptimizationRemarkMissed R("gisel-irtranslator", "GISelFailure",
                                    BB->getTerminator()->getDebugLoc(), BB);
         R << "unable to translate basic block";
-        reportTranslationError(*MF, *TPC, *ORE, R);
+        reportTranslationError(*MF, *ORE, R);
         return false;
       }
     }
diff --git a/llvm/lib/CodeGen/GlobalISel/InstructionSelect.cpp b/llvm/lib/CodeGen/GlobalISel/InstructionSelect.cpp
index 2dd22c8a7e8ba..1d281ab83aacc 100644
--- a/llvm/lib/CodeGen/GlobalISel/InstructionSelect.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/InstructionSelect.cpp
@@ -137,7 +137,6 @@ bool InstructionSelect::runOnMachineFunction(MachineFunction &MF) {
     return false;
 
   ISel = MF.getSubtarget().getInstructionSelector();
-  ISel->TPC = &getAnalysis<TargetPassConfig>();
 
   // FIXME: Properly override OptLevel in TargetMachine. See OptLevelChanger
   CodeGenOptLevel OldOptLevel = OptLevel;
@@ -159,7 +158,6 @@ bool InstructionSelect::selectMachineFunction(MachineFunction &MF) {
   LLVM_DEBUG(dbgs() << "Selecting function: " << MF.getName() << '\n');
   assert(ISel && "Cannot work without InstructionSelector");
 
-  const TargetPassConfig &TPC = *ISel->TPC;
   CodeGenCoverage CoverageInfo;
   ISel->setupMF(MF, VT, &CoverageInfo, PSI, BFI);
 
@@ -177,8 +175,8 @@ bool InstructionSelect::selectMachineFunction(MachineFunction &MF) {
   // property check already is.
   if (!DisableGISelLegalityCheck)
     if (const MachineInstr *MI = machineFunctionIsIllegal(MF)) {
-      reportGISelFailure(MF, TPC, MORE, "gisel-select",
-                         "instruction is not legal", *MI);
+      reportGISelFailure(MF, MORE, "gisel-select", "instruction is not legal",
+                         *MI);
       return false;
     }
   // FIXME: We could introduce new blocks and will need to fix the outer loop.
@@ -215,8 +213,7 @@ bool InstructionSelect::selectMachineFunction(MachineFunction &MF) {
         if (!selectInstr(MI)) {
           LLVM_DEBUG(dbgs() << "Selection failed!\n";
                      MIIMaintainer.reportFullyCreatedInstrs());
-          reportGISelFailure(MF, TPC, MORE, "gisel-select", "cannot select",
-                             MI);
+          reportGISelFailure(MF, MORE, "gisel-select", "cannot select", MI);
           return false;
         }
         LLVM_DEBUG(MIIMaintainer.reportFullyCreatedInstrs());
@@ -279,7 +276,7 @@ bool InstructionSelect::selectMachineFunction(MachineFunction &MF) {
 
     const TargetRegisterClass *RC = MRI.getRegClassOrNull(VReg);
     if (!RC) {
-      reportGISelFailure(MF, TPC, MORE, "gisel-select",
+      reportGISelFailure(MF, MORE, "gisel-select",
                          "VReg has no regclass after selection", *MI);
       return false;
     }
@@ -288,7 +285,7 @@ bool InstructionSelect::selectMachineFunction(MachineFunction &MF) {
     if (Ty.isValid() &&
         TypeSize::isKnownGT(Ty.getSizeInBits(), TRI.getRegSizeInBits(*RC))) {
       reportGISelFailure(
-          MF, TPC, MORE, "gisel-select",
+          MF, MORE, "gisel-select",
           "VReg's low-level type and register class have different sizes", *MI);
       return false;
     }
@@ -299,7 +296,7 @@ bool InstructionSelect::selectMachineFunction(MachineFunction &MF) {
                                       MF.getFunction().getSubprogram(),
                                       /*MBB=*/nullptr);
     R << "inserting blocks is not supported yet";
-    reportGISelFailure(MF, TPC, MORE, R);
+    reportGISelFailure(MF, MORE, R);
     return false;
   }
 #endif
diff --git a/llvm/lib/CodeGen/GlobalISel/Legalizer.cpp b/llvm/lib/CodeGen/GlobalISel/Legalizer.cpp
index aef16b5f33af4..0f0656aaa4f45 100644
--- a/llvm/lib/CodeGen/GlobalISel/Legalizer.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/Legalizer.cpp
@@ -348,7 +348,7 @@ bool Legalizer::runOnMachineFunction(MachineFunction &MF) {
                                             *MIRBuilder, VT);
 
   if (Result.FailedOn) {
-    reportGISelFailure(MF, TPC, MORE, "gisel-legalize",
+    reportGISelFailure(MF, MORE, "gisel-legalize",
                        "unable to legalize instruction", *Result.FailedOn);
     return false;
   }
@@ -360,7 +360,7 @@ bool Legalizer::runOnMachineFunction(MachineFunction &MF) {
     R << "lost "
       << ore::NV("NumLostDebugLocs", LocObserver.getNumLostDebugLocs())
       << " debug locations during pass";
-    reportGISelWarning(MF, TPC, MORE, R);
+    reportGISelWarning(MF, MORE, R);
     // Example remark:
     // --- !Missed
     // Pass:            gisel-legalize
diff --git a/llvm/lib/CodeGen/GlobalISel/RegBankSelect.cpp b/llvm/lib/CodeGen/GlobalISel/RegBankSelect.cpp
index bcb4f1c551cfd..5db631be32acd 100644
--- a/llvm/lib/CodeGen/GlobalISel/RegBankSelect.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/RegBankSelect.cpp
@@ -39,6 +39,7 @@
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/raw_ostream.h"
+#include "llvm/Target/TargetMachine.h"
 #include <algorithm>
 #include <cassert>
 #include <cstdint>
@@ -83,7 +84,6 @@ void RegBankSelect::init(MachineFunction &MF) {
   assert(RBI && "Cannot work without RegisterBankInfo");
   MRI = &MF.getRegInfo();
   TRI = MF.getSubtarget().getRegisterInfo();
-  TPC = &getAnalysis<TargetPassConfig>();
   if (OptMode != Mode::Fast) {
     MBFI = &getAnalysis<MachineBlockFrequencyInfoWrapperPass>().getMBFI();
     MBPI = &getAnalysis<MachineBranchProbabilityInfoWrapperPass>().getMBPI();
@@ -308,7 +308,8 @@ const RegisterBankInfo::InstructionMapping &RegBankSelect::findBestMapping(
         RepairPts.emplace_back(std::move(RepairPt));
     }
   }
-  if (!BestMapping && !TPC->isGlobalISelAbortEnabled()) {
+  if (!BestMapping && MI.getMF()->getTarget().Options.GlobalISelAbort !=
+                          GlobalISelAbortMode::Enable) {
     // If none of the mapping worked that means they are all impossible.
     // Thus, pick the first one and set an impossible repairing point.
     // It will trigger the failed isel mode.
@@ -708,7 +709,7 @@ bool RegBankSelect::assignRegisterBanks(MachineFunction &MF) {
         continue;
 
       if (!assignInstr(MI)) {
-        reportGISelFailure(MF, *TPC, *MORE, "gisel-regbankselect",
+        reportGISelFailure(MF, *MORE, "gisel-regbankselect",
                            "unable to map instruction", MI);
         return false;
       }
@@ -722,7 +723,7 @@ bool RegBankSelect::checkFunctionIsLegal(MachineFunction &MF) const {
 #ifndef NDEBUG
   if (!DisableGISelLegalityCheck) {
     if (const MachineInstr *MI = machineFunctionIsIllegal(MF)) {
-      reportGISelFailure(MF, *TPC, *MORE, "gisel-regbankselect",
+      reportGISelFailure(MF, *MORE, "gisel-regbankselect",
                          "instruction is not legal", *MI);
       return false;
     }
diff --git a/llvm/lib/CodeGen/GlobalISel/Utils.cpp b/llvm/lib/CodeGen/GlobalISel/Utils.cpp
index bc01cb65c4a69..15e81f5773b69 100644
--- a/llvm/lib/CodeGen/GlobalISel/Utils.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/Utils.cpp
@@ -234,11 +234,11 @@ bool llvm::isTriviallyDead(const MachineInstr &MI,
 
 static void reportGISelDiagnostic(DiagnosticSeverity Severity,
                                   MachineFunction &MF,
-                                  const TargetPassConfig &TPC,
                                   MachineOptimizationRemarkEmitter &MORE,
                                   MachineOptimizationRemarkMissed &R) {
-  bool IsFatal = Severity == DS_Error &&
-                 TPC.isGlobalISelAbortEnabled();
+  bool IsGlobalISelAbortEnabled =
+      MF.getTarget().Options.GlobalISelAbort == GlobalISelAbortMode::Enable;
+  bool IsFatal = Severity == DS_Error && IsGlobalISelAbortEnabled;
   // Print the function name explicitly if we don't have a debug location (which
   // makes the diagnostic less useful) or if we're going to emit a raw error.
   if (!R.getLocation().isValid() || IsFatal)
@@ -250,20 +250,20 @@ static void reportGISelDiagnostic(DiagnosticSeverity Severity,
     MORE.emit(R);
 }
 
-void llvm::reportGISelWarning(MachineFunction &MF, const TargetPassConfig &TPC,
+void llvm::reportGISelWarning(MachineFunction &MF,
                               MachineOptimizationRemarkEmitter &MORE,
                               MachineOptimizationRemarkMissed &R) {
-  reportGISelDiagnostic(DS_Warning, MF, TPC, MORE, R);
+  reportGISelDiagnostic(DS_Warning, MF, MORE, R);
 }
 
-void llvm::reportGISelFailure(MachineFunction &MF, const TargetPassConfig &TPC,
+void llvm::reportGISelFailure(MachineFunction &MF,
                               MachineOptimizationRemarkEmitter &MORE,
                               MachineOptimizationRemarkMissed &R) {
   MF.getProperties().setFailedISel();
-  reportGISelDiagnostic(DS_Error, MF, TPC, MORE, R);
+  reportGISelDiagnostic(DS_Error, MF, MORE, R);
 }
 
-void llvm::reportGISelFailure(MachineFunction &MF, const TargetPassConfig &TPC,
+void llvm::reportGISelFailure(MachineFunction &MF,
                               MachineOptimizationRemarkEmitter &MORE,
                               const char *PassName, StringRef Msg,
                               const MachineInstr &MI) {
@@ -271,9 +271,10 @@ void llvm::reportGISelFailure(MachineFunction &MF, const TargetPassConfig &TPC,
                                     MI.getDebugLoc(), MI.getParent());
   R << Msg;
   // Printing MI is expensive;  only do it if expensive remarks are enabled.
-  if (TPC.isGlobalISelAbortEnabled() || MORE.allowExtraAnalysis(PassName))
+  if (MF.getTarget().Options.GlobalISelAbort == GlobalISelAbortMode::Enable ||
+      MORE.allowExtraAnalysis(PassName))
     R << ": " << ore::MNV("Inst", MI);
-  reportGISelFailure(MF, TPC, MORE, R);
+  reportGISelFailure(MF, MORE, R);
 }
 
 unsigned llvm::getInverseGMinMaxOpcode(unsigned MinMaxOpc) {
diff --git a/llvm/lib/LTO/LTO.cpp b/llvm/lib/LTO/LTO.cpp
index a02af59600c44..4e242311e290f 100644
--- a/llvm/lib/LTO/LTO.cpp
+++ b/llvm/lib/LTO/LTO.cpp
@@ -19,7 +19,6 @@
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/Analysis/OptimizationRemarkEmitter.h"
 #include "llvm/Analysis/StackSafetyAnalysis.h"
-#include "llvm/Analysis/TargetLibraryInfo.h"
 #include "llvm/Analysis/TargetTransformInfo.h"
 #include "llvm/Bitcode/BitcodeReader.h"
 #include "llvm/Bitcode/BitcodeWriter.h"
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index d3d57b99a6345..a83185d6ade20 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -17227,7 +17227,7 @@ SDValue AArch64TargetLowering::LowerVSCALE(SDValue Op,
 template <unsigned NumVecs>
 static bool
 setInfoSVEStN(const AArch64TargetLowering &TLI, const DataLayout &DL,
-              AArch64TargetLowering::IntrinsicInfo &Info, const CallInst &CI) {
+              AArch64TargetLowering::IntrinsicInfo &Info, const CallBase &CI) {
   Info.opc = ISD::INTRINSIC_VOID;
   // Retrieve EC from first vector argument.
   const EVT VT = TLI.getMemValueType(DL, CI.getArgOperand(0)->getType());
@@ -17252,7 +17252,7 @@ setInfoSVEStN(const AArch64TargetLowering &TLI, const DataLayout &DL,
 /// MemIntrinsicNodes.  The associated MachineMemOperands record the alignment
 /// specified in the intrinsic calls.
 bool AArch64TargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
-                                               const CallInst &I,
+                                               const CallBase &I,
                                                MachineFunction &MF,
                                                unsigned Intrinsic) const {
   auto &DL = I.getDataLayout();
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.h b/llvm/lib/Target/AArch64/AArch64ISelLowering.h
index b6d34f97c7b48..1d4446d287462 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.h
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.h
@@ -206,7 +206,7 @@ class AArch64TargetLowering : public TargetLowering {
   EmitInstrWithCustomInserter(MachineInstr &MI,
                               MachineBasicBlock *MBB) const override;
 
-  bool getTgtMemIntrinsic(IntrinsicInfo &Info, const CallInst &I,
+  bool getTgtMemIntrinsic(IntrinsicInfo &Info, const CallBase &I,
                           MachineFunction &MF,
                           unsigned Intrinsic) const override;
 
diff --git a/llvm/lib/Target/AMDGPU/AMDGPU.td b/llvm/lib/Target/AMDGPU/AMDGPU.td
index 5dea64844e64e..ed8ae2b16c5d4 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPU.td
+++ b/llvm/lib/Target/AMDGPU/AMDGPU.td
@@ -905,25 +905,25 @@ def FeatureCubeInsts : SubtargetFeature<"cube-insts",
   "HasCubeInsts",
   "true",
   "Has v_cube* instructions"
->; 
+>;
 
 def FeatureLerpInst : SubtargetFeature<"lerp-inst",
   "HasLerpInst",
   "true",
   "Has v_lerp_u8 instruction"
->; 
+>;
 
 def FeatureSadInsts : SubtargetFeature<"sad-insts",
   "HasSadInsts",
   "true",
   "Has v_sad* instructions"
->; 
+>;
 
 def FeatureQsadInsts : SubtargetFeature<"qsad-insts",
   "HasQsadInsts",
   "true",
   "Has v_qsad* instructions"
->; 
+>;
 
 def FeatureCvtNormInsts : SubtargetFeature<"cvt-norm-insts",
   "HasCvtNormInsts",
@@ -1568,8 +1568,8 @@ def FeatureVolcanicIslands : GCNSubtargetFeatureGeneration<"VOLCANIC_ISLANDS",
    FeatureGFX7GFX8GFX9Insts, FeatureSMemTimeInst, FeatureMadMacF32Insts,
    FeatureDsSrc2Insts, FeatureExtendedImageInsts, FeatureFastDenormalF32,
    FeatureUnalignedBufferAccess, FeatureImageInsts, FeatureGDS, FeatureGWS,
-   FeatureDefaultComponentZero, FeatureVmemWriteVgprInOrder, FeatureCubeInsts, 
-   FeatureLerpInst, FeatureSadInsts, FeatureQsadInsts, 
+   FeatureDefaultComponentZero, FeatureVmemWriteVgprInOrder, FeatureCubeInsts,
+   FeatureLerpInst, FeatureSadInsts, FeatureQsadInsts,
    FeatureCvtPkNormVOP2Insts
   ]
 >;
@@ -1590,8 +1590,8 @@ def FeatureGFX9 : GCNSubtargetFeatureGeneration<"GFX9",
    FeatureUnalignedBufferAccess, FeatureUnalignedScratchAccess,
    FeatureUnalignedDSAccess, FeatureNegativeScratchOffsetBug, FeatureGWS,
    FeatureDefaultComponentZero,FeatureVmemWriteVgprInOrder, FeatureMemToLDSLoad,
-   FeatureCubeInsts, FeatureLerpInst, FeatureSadInsts, FeatureQsadInsts, 
-   FeatureCvtNormInsts, FeatureCvtPkNormVOP2Insts, 
+   FeatureCubeInsts, FeatureLerpInst, FeatureSadInsts, FeatureQsadInsts,
+   FeatureCvtNormInsts, FeatureCvtPkNormVOP2Insts,
    FeatureCvtPkNormVOP3Insts
   ]
 >;
@@ -1616,9 +1616,9 @@ def FeatureGFX10 : GCNSubtargetFeatureGeneration<"GFX10",
    FeatureDefaultComponentZero, FeatureMaxHardClauseLength63,
    FeatureAtomicFMinFMaxF32GlobalInsts, FeatureAtomicFMinFMaxF64GlobalInsts,
    FeatureAtomicFMinFMaxF32FlatInsts, FeatureAtomicFMinFMaxF64FlatInsts,
-   FeatureVmemWriteVgprInOrder, FeatureMemToLDSLoad, FeatureCubeInsts, 
-   FeatureLerpInst, FeatureSadInsts, FeatureQsadInsts, 
-   FeatureCvtNormInsts, FeatureCvtPkNormVOP2Insts, 
+   FeatureVmemWriteVgprInOrder, FeatureMemToLDSLoad, FeatureCubeInsts,
+   FeatureLerpInst, FeatureSadInsts, FeatureQsadInsts,
+   FeatureCvtNormInsts, FeatureCvtPkNormVOP2Insts,
    FeatureCvtPkNormVOP3Insts
   ]
 >;
@@ -1643,7 +1643,7 @@ def FeatureGFX11 : GCNSubtargetFeatureGeneration<"GFX11",
    FeatureDefaultComponentZero, FeatureMaxHardClauseLength32,
    FeatureAtomicFMinFMaxF32GlobalInsts, FeatureAtomicFMinFMaxF32FlatInsts,
    FeatureVmemWriteVgprInOrder, FeatureCubeInsts, FeatureLerpInst,
-   FeatureSadInsts, FeatureQsadInsts, FeatureCvtNormInsts, 
+   FeatureSadInsts, FeatureQsadInsts, FeatureCvtNormInsts,
    FeatureCvtPkNormVOP2Insts, FeatureCvtPkNormVOP3Insts
   ]
 >;
@@ -2124,12 +2124,12 @@ def FeatureISAVersion12 : FeatureSet<
    FeatureBVHDualAndBVH8Insts,
    FeatureWaitsBeforeSystemScopeStores,
    FeatureD16Writes32BitVgpr,
-   FeatureCubeInsts, 
-   FeatureLerpInst, 
+   FeatureCubeInsts,
+   FeatureLerpInst,
    FeatureSadInsts,
-   FeatureQsadInsts, 
-   FeatureCvtNormInsts, 
-   FeatureCvtPkNormVOP2Insts, 
+   FeatureQsadInsts,
+   FeatureCvtNormInsts,
+   FeatureCvtPkNormVOP2Insts,
    FeatureCvtPkNormVOP3Insts
    ]>;
 
@@ -2137,7 +2137,6 @@ def FeatureISAVersion12_50_Common : FeatureSet<
   [FeatureGFX12,
    FeatureGFX1250Insts,
    FeatureRequiresAlignedVGPRs,
-   FeatureAddressableLocalMemorySize327680,
    FeatureCuMode,
    Feature1024AddressableVGPRs,
    Feature64BitLiterals,
@@ -2206,17 +2205,18 @@ def FeatureISAVersion12_50_Common : FeatureSet<
    FeatureXNACK,
    FeatureClusters,
    FeatureD16Writes32BitVgpr,
+   FeatureCubeInsts,
+   FeatureLerpInst,
+   FeatureSadInsts,
+   FeatureQsadInsts,
+   FeatureCvtNormInsts,
+   FeatureCvtPkNormVOP2Insts,
+   FeatureCvtPkNormVOP3Insts
 ]>;
 
 def FeatureISAVersion12_50 : FeatureSet<
   !listconcat(FeatureISAVersion12_50_Common.Features,
-  [FeatureCubeInsts, 
-   FeatureLerpInst, 
-   FeatureSadInsts, 
-   FeatureQsadInsts, 
-   FeatureCvtNormInsts, 
-   FeatureCvtPkNormVOP2Insts, 
-   FeatureCvtPkNormVOP3Insts])>;
+  [FeatureAddressableLocalMemorySize327680])>;
 
 def FeatureISAVersion12_51 : FeatureSet<
   !listconcat(FeatureISAVersion12_50.Features,
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
index 78a3ec7f0c266..8698e816ddbb9 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
@@ -4451,16 +4451,14 @@ bool AMDGPUDAGToDAGISel::isUniformLoad(const SDNode *N) const {
   const auto *Ld = cast<LoadSDNode>(N);
   const MachineMemOperand *MMO = Ld->getMemOperand();
 
-  if (Ld->isDivergent()) {
-    // FIXME: We ought to able able to take the direct isDivergent result. We
-    // cannot rely on the MMO for a uniformity check, and should stop using
-    // it. This is a hack for 2 ways that the IR divergence analysis is superior
-    // to the DAG divergence: Recognizing shift-of-workitem-id as always
-    // uniform, and isSingleLaneExecution. These should be handled in the DAG
-    // version, and then this can be dropped.
-    if (!MMO->getValue() || !AMDGPU::isUniformMMO(MMO))
-      return false;
-  }
+  // FIXME: We ought to able able to take the direct isDivergent result. We
+  // cannot rely on the MMO for a uniformity check, and should stop using
+  // it. This is a hack for 2 ways that the IR divergence analysis is superior
+  // to the DAG divergence: Recognizing shift-of-workitem-id as always
+  // uniform, and isSingleLaneExecution. These should be handled in the DAG
+  // version, and then this can be dropped.
+  if (Ld->isDivergent() && !AMDGPU::isUniformMMO(MMO))
+    return false;
 
   return MMO->getSize().hasValue() &&
          Ld->getAlign() >=
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstrInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPUInstrInfo.cpp
index b7b87674ee658..2b1f4048947bf 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUInstrInfo.cpp
@@ -35,15 +35,13 @@ bool AMDGPU::isUniformMMO(const MachineMemOperand *MMO) {
              PSV->isJumpTable();
     }
 
-    // FIXME: null value is should be treated as unknown, not as uniform.
-    return true;
+    // Unknown value.
+    return false;
   }
 
   // UndefValue means this is a load of a kernel input.  These are uniform.
   // Sometimes LDS instructions have constant pointers.
-  // If Ptr is null, then that means this mem operand contains a
-  // PseudoSourceValue like GOT.
-  if (!Ptr || isa<UndefValue, Constant, GlobalValue>(Ptr))
+  if (isa<UndefValue, Constant, GlobalValue>(Ptr))
     return true;
 
   if (MMO->getAddrSpace() == AMDGPUAS::CONSTANT_ADDRESS_32BIT)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
index 6e41134bb3bc1..ae62dbe1cc706 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
@@ -2362,7 +2362,7 @@ Register AMDGPULegalizerInfo::getSegmentAperture(
   if (!loadInputValue(QueuePtr, B, AMDGPUFunctionArgInfo::QUEUE_PTR))
     return Register();
 
-  // TODO: can we be smarter about machine pointer info?
+  // TODO: Use custom PseudoSourceValue
   MachinePointerInfo PtrInfo(AMDGPUAS::CONSTANT_ADDRESS);
 
   // Offset into amd_queue_t for group_segment_aperture_base_hi /
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULowerBufferFatPointers.cpp b/llvm/lib/Target/AMDGPU/AMDGPULowerBufferFatPointers.cpp
index 98884c441096e..fdff21b6ef8df 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULowerBufferFatPointers.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULowerBufferFatPointers.cpp
@@ -1035,7 +1035,7 @@ bool LegalizeBufferContentTypesVisitor::visitLoadImpl(
       LoadInst *NewLI = IRB.CreateAlignedLoad(
           LoadableType, NewPtr, commonAlignment(OrigLI.getAlign(), ByteOffset),
           Name + ".off." + Twine(ByteOffset));
-      copyMetadataForAccess(*NewLI, OrigLI);
+      copyMetadataForLoad(*NewLI, OrigLI);
       NewLI->setAAMetadata(
           AANodes.adjustForAccess(ByteOffset, LoadableType, DL));
       NewLI->setAtomic(OrigLI.getOrdering(), OrigLI.getSyncScopeID());
diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalize.cpp b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalize.cpp
index 396d64625fb5c..839120da89711 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalize.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalize.cpp
@@ -435,7 +435,8 @@ bool AMDGPURegBankLegalize::runOnMachineFunction(MachineFunction &MF) {
     unsigned Opc = MI->getOpcode();
     // Insert point for use operands needs some calculation.
     if (Opc == AMDGPU::G_PHI) {
-      RBLHelper.applyMappingPHI(*MI);
+      if (!RBLHelper.applyMappingPHI(*MI))
+        return false;
       continue;
     }
 
@@ -466,7 +467,8 @@ bool AMDGPURegBankLegalize::runOnMachineFunction(MachineFunction &MF) {
       // S1 rules are in RegBankLegalizeRules.
     }
 
-    RBLHelper.findRuleAndApplyMapping(*MI);
+    if (!RBLHelper.findRuleAndApplyMapping(*MI))
+      return false;
   }
 
   // Sgpr S1 clean up combines:
diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeHelper.cpp b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeHelper.cpp
index 123fc5bf37a19..cc31d7d5c55ac 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeHelper.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeHelper.cpp
@@ -32,28 +32,48 @@ using namespace AMDGPU;
 RegBankLegalizeHelper::RegBankLegalizeHelper(
     MachineIRBuilder &B, const MachineUniformityInfo &MUI,
     const RegisterBankInfo &RBI, const RegBankLegalizeRules &RBLRules)
-    : ST(B.getMF().getSubtarget<GCNSubtarget>()), B(B), MRI(*B.getMRI()),
-      MUI(MUI), RBI(RBI), RBLRules(RBLRules), IsWave32(ST.isWave32()),
+    : MF(B.getMF()), ST(MF.getSubtarget<GCNSubtarget>()), B(B),
+      MRI(*B.getMRI()), MUI(MUI), RBI(RBI), MORE(MF, nullptr),
+      RBLRules(RBLRules), IsWave32(ST.isWave32()),
       SgprRB(&RBI.getRegBank(AMDGPU::SGPRRegBankID)),
       VgprRB(&RBI.getRegBank(AMDGPU::VGPRRegBankID)),
       VccRB(&RBI.getRegBank(AMDGPU::VCCRegBankID)) {}
 
-void RegBankLegalizeHelper::findRuleAndApplyMapping(MachineInstr &MI) {
-  const SetOfRulesForOpcode &RuleSet = RBLRules.getRulesForOpc(MI);
-  const RegBankLLTMapping &Mapping = RuleSet.findMappingForMI(MI, MRI, MUI);
+bool RegBankLegalizeHelper::findRuleAndApplyMapping(MachineInstr &MI) {
+  const SetOfRulesForOpcode *RuleSet = RBLRules.getRulesForOpc(MI);
+  if (!RuleSet) {
+    reportGISelFailure(MF, MORE, "amdgpu-regbanklegalize",
+                       "No AMDGPU RegBankLegalize rules defined for opcode",
+                       MI);
+    return false;
+  }
+
+  const RegBankLLTMapping *Mapping = RuleSet->findMappingForMI(MI, MRI, MUI);
+  if (!Mapping) {
+    reportGISelFailure(MF, MORE, "amdgpu-regbanklegalize",
+                       "AMDGPU RegBankLegalize: none of the rules defined with "
+                       "'Any' for MI's opcode matched MI",
+                       MI);
+    return false;
+  }
 
   SmallSet<Register, 4> WaterfallSgprs;
   unsigned OpIdx = 0;
-  if (Mapping.DstOpMapping.size() > 0) {
+  if (Mapping->DstOpMapping.size() > 0) {
     B.setInsertPt(*MI.getParent(), std::next(MI.getIterator()));
-    applyMappingDst(MI, OpIdx, Mapping.DstOpMapping);
+    if (!applyMappingDst(MI, OpIdx, Mapping->DstOpMapping))
+      return false;
   }
-  if (Mapping.SrcOpMapping.size() > 0) {
+  if (Mapping->SrcOpMapping.size() > 0) {
     B.setInstr(MI);
-    applyMappingSrc(MI, OpIdx, Mapping.SrcOpMapping, WaterfallSgprs);
+    if (!applyMappingSrc(MI, OpIdx, Mapping->SrcOpMapping, WaterfallSgprs))
+      return false;
   }
 
-  lower(MI, Mapping, WaterfallSgprs);
+  if (!lower(MI, *Mapping, WaterfallSgprs))
+    return false;
+
+  return true;
 }
 
 bool RegBankLegalizeHelper::executeInWaterfallLoop(
@@ -274,7 +294,7 @@ bool RegBankLegalizeHelper::executeInWaterfallLoop(
   return true;
 }
 
-void RegBankLegalizeHelper::splitLoad(MachineInstr &MI,
+bool RegBankLegalizeHelper::splitLoad(MachineInstr &MI,
                                       ArrayRef<LLT> LLTBreakdown, LLT MergeTy) {
   MachineFunction &MF = B.getMF();
   assert(MI.getNumMemOperands() == 1);
@@ -322,9 +342,10 @@ void RegBankLegalizeHelper::splitLoad(MachineInstr &MI,
     B.buildMergeLikeInstr(Dst, MergeTyParts);
   }
   MI.eraseFromParent();
+  return true;
 }
 
-void RegBankLegalizeHelper::widenLoad(MachineInstr &MI, LLT WideTy,
+bool RegBankLegalizeHelper::widenLoad(MachineInstr &MI, LLT WideTy,
                                       LLT MergeTy) {
   MachineFunction &MF = B.getMF();
   assert(MI.getNumMemOperands() == 1);
@@ -350,9 +371,10 @@ void RegBankLegalizeHelper::widenLoad(MachineInstr &MI, LLT WideTy,
     B.buildMergeLikeInstr(Dst, MergeTyParts);
   }
   MI.eraseFromParent();
+  return true;
 }
 
-void RegBankLegalizeHelper::widenMMOToS32(GAnyLoad &MI) const {
+bool RegBankLegalizeHelper::widenMMOToS32(GAnyLoad &MI) const {
   Register Dst = MI.getDstReg();
   Register Ptr = MI.getPointerReg();
   MachineMemOperand &MMO = MI.getMMO();
@@ -376,9 +398,10 @@ void RegBankLegalizeHelper::widenMMOToS32(GAnyLoad &MI) const {
   }
 
   MI.eraseFromParent();
+  return true;
 }
 
-void RegBankLegalizeHelper::lowerVccExtToSel(MachineInstr &MI) {
+bool RegBankLegalizeHelper::lowerVccExtToSel(MachineInstr &MI) {
   Register Dst = MI.getOperand(0).getReg();
   LLT Ty = MRI.getType(Dst);
   Register Src = MI.getOperand(1).getReg();
@@ -404,15 +427,22 @@ void RegBankLegalizeHelper::lowerVccExtToSel(MachineInstr &MI) {
       Hi = B.buildUndef({VgprRB_S32});
       break;
     default:
-      llvm_unreachable("Opcode not supported");
+      reportGISelFailure(
+          MF, MORE, "amdgpu-regbanklegalize",
+          "AMDGPU RegBankLegalize: lowerVccExtToSel, Opcode not supported", MI);
+      return false;
     }
 
     B.buildMergeValues(Dst, {Lo.getReg(0), Hi.getReg(0)});
   } else {
-    llvm_unreachable("Type not supported");
+    reportGISelFailure(
+        MF, MORE, "amdgpu-regbanklegalize",
+        "AMDGPU RegBankLegalize: lowerVccExtToSel, Type not supported", MI);
+    return false;
   }
 
   MI.eraseFromParent();
+  return true;
 }
 
 std::pair<Register, Register> RegBankLegalizeHelper::unpackZExt(Register Reg) {
@@ -444,7 +474,7 @@ RegBankLegalizeHelper::unpackAExtTruncS16(Register Reg) {
           B.buildTrunc(SgprRB_S16, Hi32).getReg(0)};
 }
 
-void RegBankLegalizeHelper::lowerUnpackBitShift(MachineInstr &MI) {
+bool RegBankLegalizeHelper::lowerUnpackBitShift(MachineInstr &MI) {
   Register Lo, Hi;
   switch (MI.getOpcode()) {
   case AMDGPU::G_SHL: {
@@ -469,13 +499,18 @@ void RegBankLegalizeHelper::lowerUnpackBitShift(MachineInstr &MI) {
     break;
   }
   default:
-    llvm_unreachable("Unpack lowering not implemented");
+    reportGISelFailure(
+        MF, MORE, "amdgpu-regbanklegalize",
+        "AMDGPU RegBankLegalize: lowerUnpackBitShift, case not implemented",
+        MI);
+    return false;
   }
   B.buildBuildVectorTrunc(MI.getOperand(0).getReg(), {Lo, Hi});
   MI.eraseFromParent();
+  return true;
 }
 
-void RegBankLegalizeHelper::lowerUnpackMinMax(MachineInstr &MI) {
+bool RegBankLegalizeHelper::lowerUnpackMinMax(MachineInstr &MI) {
   Register Lo, Hi;
   switch (MI.getOpcode()) {
   case AMDGPU::G_SMIN:
@@ -501,13 +536,17 @@ void RegBankLegalizeHelper::lowerUnpackMinMax(MachineInstr &MI) {
     break;
   }
   default:
-    llvm_unreachable("Unpack min/max lowering not implemented");
+    reportGISelFailure(
+        MF, MORE, "amdgpu-regbanklegalize",
+        "AMDGPU RegBankLegalize: lowerUnpackMinMax, case not implemented", MI);
+    return false;
   }
   B.buildBuildVectorTrunc(MI.getOperand(0).getReg(), {Lo, Hi});
   MI.eraseFromParent();
+  return true;
 }
 
-void RegBankLegalizeHelper::lowerUnpackAExt(MachineInstr &MI) {
+bool RegBankLegalizeHelper::lowerUnpackAExt(MachineInstr &MI) {
   auto [Op1Lo, Op1Hi] = unpackAExt(MI.getOperand(1).getReg());
   auto [Op2Lo, Op2Hi] = unpackAExt(MI.getOperand(2).getReg());
   auto ResLo = B.buildInstr(MI.getOpcode(), {SgprRB_S32}, {Op1Lo, Op2Lo});
@@ -515,6 +554,7 @@ void RegBankLegalizeHelper::lowerUnpackAExt(MachineInstr &MI) {
   B.buildBuildVectorTrunc(MI.getOperand(0).getReg(),
                           {ResLo.getReg(0), ResHi.getReg(0)});
   MI.eraseFromParent();
+  return true;
 }
 
 static bool isSignedBFE(MachineInstr &MI) {
@@ -524,7 +564,7 @@ static bool isSignedBFE(MachineInstr &MI) {
   return MI.getOpcode() == AMDGPU::G_SBFX;
 }
 
-void RegBankLegalizeHelper::lowerV_BFE(MachineInstr &MI) {
+bool RegBankLegalizeHelper::lowerV_BFE(MachineInstr &MI) {
   Register Dst = MI.getOperand(0).getReg();
   assert(MRI.getType(Dst) == LLT::scalar(64));
   bool Signed = isSignedBFE(MI);
@@ -551,7 +591,7 @@ void RegBankLegalizeHelper::lowerV_BFE(MachineInstr &MI) {
     auto SignBit = B.buildShl({VgprRB, S64}, SHRSrc, Amt);
     B.buildInstr(SHROpc, {Dst}, {SignBit, Amt});
     MI.eraseFromParent();
-    return;
+    return true;
   }
 
   uint64_t WidthImm = ConstWidth->Value.getZExtValue();
@@ -581,9 +621,10 @@ void RegBankLegalizeHelper::lowerV_BFE(MachineInstr &MI) {
   }
 
   MI.eraseFromParent();
+  return true;
 }
 
-void RegBankLegalizeHelper::lowerS_BFE(MachineInstr &MI) {
+bool RegBankLegalizeHelper::lowerS_BFE(MachineInstr &MI) {
   Register DstReg = MI.getOperand(0).getReg();
   LLT Ty = MRI.getType(DstReg);
   bool Signed = isSignedBFE(MI);
@@ -609,14 +650,19 @@ void RegBankLegalizeHelper::lowerS_BFE(MachineInstr &MI) {
   auto S_BFE = B.buildInstr(Opc, {{SgprRB, Ty}},
                             {B.buildCopy(Ty, Src), B.buildCopy(S32, Src1)});
   if (!constrainSelectedInstRegOperands(*S_BFE, *ST.getInstrInfo(),
-                                        *ST.getRegisterInfo(), RBI))
-    llvm_unreachable("failed to constrain BFE");
+                                        *ST.getRegisterInfo(), RBI)) {
+    reportGISelFailure(
+        MF, MORE, "amdgpu-regbanklegalize",
+        "AMDGPU RegBankLegalize: lowerS_BFE, failed to constrain BFE", MI);
+    return false;
+  }
 
   B.buildCopy(DstReg, S_BFE->getOperand(0).getReg());
   MI.eraseFromParent();
+  return true;
 }
 
-void RegBankLegalizeHelper::lowerSplitTo32(MachineInstr &MI) {
+bool RegBankLegalizeHelper::lowerSplitTo32(MachineInstr &MI) {
   Register Dst = MI.getOperand(0).getReg();
   LLT DstTy = MRI.getType(Dst);
   assert(DstTy == V4S16 || DstTy == V2S32 || DstTy == S64);
@@ -631,9 +677,10 @@ void RegBankLegalizeHelper::lowerSplitTo32(MachineInstr &MI) {
       B.buildInstr(Opc, {{VgprRB, Ty}}, {Op1.getReg(1), Op2.getReg(1)}, Flags);
   B.buildMergeLikeInstr(Dst, {Lo, Hi});
   MI.eraseFromParent();
+  return true;
 }
 
-void RegBankLegalizeHelper::lowerSplitTo16(MachineInstr &MI) {
+bool RegBankLegalizeHelper::lowerSplitTo16(MachineInstr &MI) {
   Register Dst = MI.getOperand(0).getReg();
   assert(MRI.getType(Dst) == V2S16);
   unsigned Opc = MI.getOpcode();
@@ -645,7 +692,7 @@ void RegBankLegalizeHelper::lowerSplitTo16(MachineInstr &MI) {
     auto Hi = B.buildInstr(Opc, {SgprRB_S16}, {Op1Hi}, Flags);
     B.buildMergeLikeInstr(Dst, {Lo, Hi});
     MI.eraseFromParent();
-    return;
+    return true;
   }
 
   assert(MI.getNumOperands() == 3);
@@ -655,9 +702,10 @@ void RegBankLegalizeHelper::lowerSplitTo16(MachineInstr &MI) {
   auto Hi = B.buildInstr(Opc, {SgprRB_S16}, {Op1Hi, Op2Hi}, Flags);
   B.buildMergeLikeInstr(Dst, {Lo, Hi});
   MI.eraseFromParent();
+  return true;
 }
 
-void RegBankLegalizeHelper::lowerSplitTo32Select(MachineInstr &MI) {
+bool RegBankLegalizeHelper::lowerSplitTo32Select(MachineInstr &MI) {
   Register Dst = MI.getOperand(0).getReg();
   LLT DstTy = MRI.getType(Dst);
   assert(DstTy == V4S16 || DstTy == V2S32 || DstTy == S64 ||
@@ -674,9 +722,10 @@ void RegBankLegalizeHelper::lowerSplitTo32Select(MachineInstr &MI) {
 
   B.buildMergeLikeInstr(Dst, {Lo, Hi});
   MI.eraseFromParent();
+  return true;
 }
 
-void RegBankLegalizeHelper::lowerSplitTo32SExtInReg(MachineInstr &MI) {
+bool RegBankLegalizeHelper::lowerSplitTo32SExtInReg(MachineInstr &MI) {
   auto Op1 = B.buildUnmerge(VgprRB_S32, MI.getOperand(1).getReg());
   int Amt = MI.getOperand(2).getImm();
   Register Lo, Hi;
@@ -701,9 +750,10 @@ void RegBankLegalizeHelper::lowerSplitTo32SExtInReg(MachineInstr &MI) {
 
   B.buildMergeLikeInstr(MI.getOperand(0).getReg(), {Lo, Hi});
   MI.eraseFromParent();
+  return true;
 }
 
-void RegBankLegalizeHelper::lower(MachineInstr &MI,
+bool RegBankLegalizeHelper::lower(MachineInstr &MI,
                                   const RegBankLLTMapping &Mapping,
                                   SmallSet<Register, 4> &WaterfallSgprs) {
 
@@ -723,7 +773,7 @@ void RegBankLegalizeHelper::lower(MachineInstr &MI,
     B.buildSelect(MI.getOperand(0).getReg(), MI.getOperand(1).getReg(), True,
                   False);
     MI.eraseFromParent();
-    return;
+    return true;
   }
   case UnpackBitShift:
     return lowerUnpackBitShift(MI);
@@ -750,20 +800,23 @@ void RegBankLegalizeHelper::lower(MachineInstr &MI,
       break;
     }
     default:
-      llvm_unreachable("Unsuported Opcode in Ext32To64");
+      reportGISelFailure(MF, MORE, "amdgpu-regbanklegalize",
+                         "AMDGPU RegBankLegalize: Ext32To64, unsuported opcode",
+                         MI);
+      return false;
     }
 
     B.buildMergeLikeInstr(MI.getOperand(0).getReg(),
                           {MI.getOperand(1).getReg(), Hi});
     MI.eraseFromParent();
-    return;
+    return true;
   }
   case UniCstExt: {
     uint64_t ConstVal = MI.getOperand(1).getCImm()->getZExtValue();
     B.buildConstant(MI.getOperand(0).getReg(), ConstVal);
 
     MI.eraseFromParent();
-    return;
+    return true;
   }
   case VgprToVccCopy: {
     Register Src = MI.getOperand(1).getReg();
@@ -787,7 +840,7 @@ void RegBankLegalizeHelper::lower(MachineInstr &MI,
     auto Zero = B.buildConstant({VgprRB, Ty}, 0);
     B.buildICmp(CmpInst::ICMP_NE, MI.getOperand(0).getReg(), BoolSrc, Zero);
     MI.eraseFromParent();
-    return;
+    return true;
   }
   case V_BFE:
     return lowerV_BFE(MI);
@@ -816,8 +869,10 @@ void RegBankLegalizeHelper::lower(MachineInstr &MI,
       else if (Size / 128 == 4)
         splitLoad(MI, {B128, B128, B128, B128});
       else {
-        LLVM_DEBUG(dbgs() << "MI: "; MI.dump(););
-        llvm_unreachable("SplitLoad type not supported for MI");
+        reportGISelFailure(MF, MORE, "amdgpu-regbanklegalize",
+                           "AMDGPU RegBankLegalize: SplitLoad, unsuported type",
+                           MI);
+        return false;
       }
     }
     // 64 and 32 bit load
@@ -828,10 +883,12 @@ void RegBankLegalizeHelper::lower(MachineInstr &MI,
     else if (DstTy == V6S16)
       splitLoad(MI, {V4S16, V2S16}, V2S16);
     else {
-      LLVM_DEBUG(dbgs() << "MI: "; MI.dump(););
-      llvm_unreachable("SplitLoad type not supported for MI");
+      reportGISelFailure(MF, MORE, "amdgpu-regbanklegalize",
+                         "AMDGPU RegBankLegalize: SplitLoad, unsuported type",
+                         MI);
+      return false;
     }
-    break;
+    return true;
   }
   case WidenLoad: {
     LLT DstTy = MRI.getType(MI.getOperand(0).getReg());
@@ -842,10 +899,12 @@ void RegBankLegalizeHelper::lower(MachineInstr &MI,
     else if (DstTy == V6S16)
       widenLoad(MI, V8S16, V2S16);
     else {
-      LLVM_DEBUG(dbgs() << "MI: "; MI.dump(););
-      llvm_unreachable("WidenLoad type not supported for MI");
+      reportGISelFailure(MF, MORE, "amdgpu-regbanklegalize",
+                         "AMDGPU RegBankLegalize: WidenLoad, unsuported type",
+                         MI);
+      return false;
     }
-    break;
+    return true;
   }
   case UnpackAExt:
     return lowerUnpackAExt(MI);
@@ -855,8 +914,10 @@ void RegBankLegalizeHelper::lower(MachineInstr &MI,
 
   if (!WaterfallSgprs.empty()) {
     MachineBasicBlock::iterator I = MI.getIterator();
-    executeInWaterfallLoop(B, make_range(I, std::next(I)), WaterfallSgprs);
+    if (!executeInWaterfallLoop(B, make_range(I, std::next(I)), WaterfallSgprs))
+      return false;
   }
+  return true;
 }
 
 LLT RegBankLegalizeHelper::getTyFromID(RegBankLLTMappingApplyID ID) {
@@ -1055,7 +1116,7 @@ RegBankLegalizeHelper::getRegBankFromID(RegBankLLTMappingApplyID ID) {
   }
 }
 
-void RegBankLegalizeHelper::applyMappingDst(
+bool RegBankLegalizeHelper::applyMappingDst(
     MachineInstr &MI, unsigned &OpIdx,
     const SmallVectorImpl<RegBankLLTMappingApplyID> &MethodIDs) {
   // Defs start from operand 0
@@ -1180,16 +1241,23 @@ void RegBankLegalizeHelper::applyMappingDst(
       break;
     }
     case InvalidMapping: {
-      LLVM_DEBUG(dbgs() << "Instruction with Invalid mapping: "; MI.dump(););
-      llvm_unreachable("missing fast rule for MI");
+      reportGISelFailure(
+          MF, MORE, "amdgpu-regbanklegalize",
+          "AMDGPU RegBankLegalize: missing fast rule ('Div' or 'Uni') for", MI);
+      return false;
     }
     default:
-      llvm_unreachable("ID not supported");
+      reportGISelFailure(
+          MF, MORE, "amdgpu-regbanklegalize",
+          "AMDGPU RegBankLegalize: applyMappingDst, ID not supported", MI);
+      return false;
     }
   }
+
+  return true;
 }
 
-void RegBankLegalizeHelper::applyMappingSrc(
+bool RegBankLegalizeHelper::applyMappingSrc(
     MachineInstr &MI, unsigned &OpIdx,
     const SmallVectorImpl<RegBankLLTMappingApplyID> &MethodIDs,
     SmallSet<Register, 4> &SgprWaterfallOperandRegs) {
@@ -1343,12 +1411,16 @@ void RegBankLegalizeHelper::applyMappingSrc(
       break;
     }
     default:
-      llvm_unreachable("ID not supported");
+      reportGISelFailure(
+          MF, MORE, "amdgpu-regbanklegalize",
+          "AMDGPU RegBankLegalize: applyMappingSrc, ID not supported", MI);
+      return false;
     }
   }
+  return true;
 }
 
-void RegBankLegalizeHelper::applyMappingPHI(MachineInstr &MI) {
+bool RegBankLegalizeHelper::applyMappingPHI(MachineInstr &MI) {
   Register Dst = MI.getOperand(0).getReg();
   LLT Ty = MRI.getType(Dst);
 
@@ -1371,16 +1443,17 @@ void RegBankLegalizeHelper::applyMappingPHI(MachineInstr &MI) {
       MI.getOperand(i).setReg(NewUse.getReg(0));
     }
 
-    return;
+    return true;
   }
 
-  // ALL divergent i1 phis should be already lowered and inst-selected into PHI
-  // with sgpr reg class and S1 LLT.
+  // ALL divergent i1 phis should have been lowered and inst-selected into PHI
+  // with sgpr reg class and S1 LLT in AMDGPUGlobalISelDivergenceLowering pass.
   // Note: this includes divergent phis that don't require lowering.
   if (Ty == LLT::scalar(1) && MUI.isDivergent(Dst)) {
-    LLVM_DEBUG(dbgs() << "Divergent S1 G_PHI: "; MI.dump(););
-    llvm_unreachable("Make sure to run AMDGPUGlobalISelDivergenceLowering "
-                     "before RegBankLegalize to lower lane mask(vcc) phis");
+    reportGISelFailure(MF, MORE, "amdgpu-regbanklegalize",
+                       "AMDGPU RegBankLegalize: Can't lower divergent S1 G_PHI",
+                       MI);
+    return false;
   }
 
   // We accept all types that can fit in some register class.
@@ -1388,11 +1461,13 @@ void RegBankLegalizeHelper::applyMappingPHI(MachineInstr &MI) {
   // Divergent G_PHIs have vgpr dst but inputs can be sgpr or vgpr.
   if (Ty == LLT::scalar(32) || Ty == LLT::pointer(1, 64) ||
       Ty == LLT::pointer(4, 64)) {
-    return;
+    return true;
   }
 
-  LLVM_DEBUG(dbgs() << "G_PHI not handled: "; MI.dump(););
-  llvm_unreachable("type not supported");
+  reportGISelFailure(MF, MORE, "amdgpu-regbanklegalize",
+                     "AMDGPU RegBankLegalize: type not supported for G_PHI",
+                     MI);
+  return false;
 }
 
 [[maybe_unused]] static bool verifyRegBankOnOperands(MachineInstr &MI,
diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeHelper.h b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeHelper.h
index 4f1c3c02fa5d6..1dc0278d6d90d 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeHelper.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeHelper.h
@@ -12,6 +12,7 @@
 #include "AMDGPURegBankLegalizeRules.h"
 #include "llvm/ADT/SmallSet.h"
 #include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
+#include "llvm/CodeGen/MachineOptimizationRemarkEmitter.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
 
 namespace llvm {
@@ -27,11 +28,13 @@ namespace AMDGPU {
 // to replace instruction. In other case InstApplyMethod will create new
 // instruction(s).
 class RegBankLegalizeHelper {
+  MachineFunction &MF;
   const GCNSubtarget &ST;
   MachineIRBuilder &B;
   MachineRegisterInfo &MRI;
   const MachineUniformityInfo &MUI;
   const RegisterBankInfo &RBI;
+  MachineOptimizationRemarkEmitter MORE;
   const RegBankLegalizeRules &RBLRules;
   const bool IsWave32;
   const RegisterBank *SgprRB;
@@ -81,10 +84,10 @@ class RegBankLegalizeHelper {
                         const RegisterBankInfo &RBI,
                         const RegBankLegalizeRules &RBLRules);
 
-  void findRuleAndApplyMapping(MachineInstr &MI);
+  bool findRuleAndApplyMapping(MachineInstr &MI);
 
   // Manual apply helpers.
-  void applyMappingPHI(MachineInstr &MI);
+  bool applyMappingPHI(MachineInstr &MI);
   void applyMappingTrivial(MachineInstr &MI);
 
 private:
@@ -97,37 +100,37 @@ class RegBankLegalizeHelper {
 
   const RegisterBank *getRegBankFromID(RegBankLLTMappingApplyID ID);
 
-  void
+  bool
   applyMappingDst(MachineInstr &MI, unsigned &OpIdx,
                   const SmallVectorImpl<RegBankLLTMappingApplyID> &MethodIDs);
 
-  void
+  bool
   applyMappingSrc(MachineInstr &MI, unsigned &OpIdx,
                   const SmallVectorImpl<RegBankLLTMappingApplyID> &MethodIDs,
                   SmallSet<Register, 4> &SgprWaterfallOperandRegs);
 
-  void splitLoad(MachineInstr &MI, ArrayRef<LLT> LLTBreakdown,
+  bool splitLoad(MachineInstr &MI, ArrayRef<LLT> LLTBreakdown,
                  LLT MergeTy = LLT());
-  void widenLoad(MachineInstr &MI, LLT WideTy, LLT MergeTy = LLT());
-  void widenMMOToS32(GAnyLoad &MI) const;
+  bool widenLoad(MachineInstr &MI, LLT WideTy, LLT MergeTy = LLT());
+  bool widenMMOToS32(GAnyLoad &MI) const;
 
-  void lower(MachineInstr &MI, const RegBankLLTMapping &Mapping,
+  bool lower(MachineInstr &MI, const RegBankLLTMapping &Mapping,
              SmallSet<Register, 4> &SgprWaterfallOperandRegs);
 
-  void lowerVccExtToSel(MachineInstr &MI);
+  bool lowerVccExtToSel(MachineInstr &MI);
   std::pair<Register, Register> unpackZExt(Register Reg);
   std::pair<Register, Register> unpackSExt(Register Reg);
   std::pair<Register, Register> unpackAExt(Register Reg);
   std::pair<Register, Register> unpackAExtTruncS16(Register Reg);
-  void lowerUnpackBitShift(MachineInstr &MI);
-  void lowerV_BFE(MachineInstr &MI);
-  void lowerS_BFE(MachineInstr &MI);
-  void lowerSplitTo32(MachineInstr &MI);
-  void lowerSplitTo16(MachineInstr &MI);
-  void lowerSplitTo32Select(MachineInstr &MI);
-  void lowerSplitTo32SExtInReg(MachineInstr &MI);
-  void lowerUnpackMinMax(MachineInstr &MI);
-  void lowerUnpackAExt(MachineInstr &MI);
+  bool lowerUnpackBitShift(MachineInstr &MI);
+  bool lowerV_BFE(MachineInstr &MI);
+  bool lowerS_BFE(MachineInstr &MI);
+  bool lowerSplitTo32(MachineInstr &MI);
+  bool lowerSplitTo16(MachineInstr &MI);
+  bool lowerSplitTo32Select(MachineInstr &MI);
+  bool lowerSplitTo32SExtInReg(MachineInstr &MI);
+  bool lowerUnpackMinMax(MachineInstr &MI);
+  bool lowerUnpackAExt(MachineInstr &MI);
 };
 
 } // end namespace AMDGPU
diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp
index 0c1daefd493b6..9de309279a247 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp
@@ -243,7 +243,7 @@ UniformityLLTOpPredicateID LLTToBId(LLT Ty) {
   return _;
 }
 
-const RegBankLLTMapping &
+const RegBankLLTMapping *
 SetOfRulesForOpcode::findMappingForMI(const MachineInstr &MI,
                                       const MachineRegisterInfo &MRI,
                                       const MachineUniformityInfo &MUI) const {
@@ -260,17 +260,16 @@ SetOfRulesForOpcode::findMappingForMI(const MachineInstr &MI,
       Slot = getFastPredicateSlot(LLTToId(MRI.getType(Reg)));
 
     if (Slot != -1)
-      return MUI.isUniform(Reg) ? Uni[Slot] : Div[Slot];
+      return MUI.isUniform(Reg) ? &Uni[Slot] : &Div[Slot];
   }
 
   // Slow search for more complex rules.
   for (const RegBankLegalizeRule &Rule : Rules) {
     if (Rule.Predicate.match(MI, MUI, MRI))
-      return Rule.OperandMapping;
+      return &Rule.OperandMapping;
   }
 
-  LLVM_DEBUG(dbgs() << "MI: "; MI.dump(););
-  llvm_unreachable("None of the rules defined for MI's opcode matched MI");
+  return nullptr;
 }
 
 void SetOfRulesForOpcode::addRule(RegBankLegalizeRule Rule) {
@@ -353,7 +352,7 @@ RegBankLegalizeRules::addRulesForIOpcs(std::initializer_list<unsigned> OpcList,
   return RuleSetInitializer(OpcList, IRulesAlias, IRules, FastTypes);
 }
 
-const SetOfRulesForOpcode &
+const SetOfRulesForOpcode *
 RegBankLegalizeRules::getRulesForOpc(MachineInstr &MI) const {
   unsigned Opc = MI.getOpcode();
   if (Opc == AMDGPU::G_INTRINSIC || Opc == AMDGPU::G_INTRINSIC_CONVERGENT ||
@@ -361,19 +360,15 @@ RegBankLegalizeRules::getRulesForOpc(MachineInstr &MI) const {
       Opc == AMDGPU::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS) {
     unsigned IntrID = cast<GIntrinsic>(MI).getIntrinsicID();
     auto IRAIt = IRulesAlias.find(IntrID);
-    if (IRAIt == IRulesAlias.end()) {
-      LLVM_DEBUG(dbgs() << "MI: "; MI.dump(););
-      llvm_unreachable("No rules defined for intrinsic opcode");
-    }
-    return IRules.at(IRAIt->second);
+    if (IRAIt == IRulesAlias.end())
+      return nullptr;
+    return &IRules.at(IRAIt->second);
   }
 
   auto GRAIt = GRulesAlias.find(Opc);
-  if (GRAIt == GRulesAlias.end()) {
-    LLVM_DEBUG(dbgs() << "MI: "; MI.dump(););
-    llvm_unreachable("No rules defined for generic opcode");
-  }
-  return GRules.at(GRAIt->second);
+  if (GRAIt == GRulesAlias.end())
+    return nullptr;
+  return &GRules.at(GRAIt->second);
 }
 
 // Syntactic sugar wrapper for predicate lambda that enables '&&', '||' and '!'.
diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.h b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.h
index 7e4ce7b43dc3b..1ac117304b76f 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.h
@@ -287,7 +287,7 @@ class SetOfRulesForOpcode {
   SetOfRulesForOpcode();
   SetOfRulesForOpcode(FastRulesTypes FastTypes);
 
-  const RegBankLLTMapping &
+  const RegBankLLTMapping *
   findMappingForMI(const MachineInstr &MI, const MachineRegisterInfo &MRI,
                    const MachineUniformityInfo &MUI) const;
 
@@ -385,7 +385,7 @@ class RegBankLegalizeRules {
     MRI = &_MRI;
   };
 
-  const SetOfRulesForOpcode &getRulesForOpc(MachineInstr &MI) const;
+  const SetOfRulesForOpcode *getRulesForOpc(MachineInstr &MI) const;
 };
 
 } // end namespace AMDGPU
diff --git a/llvm/lib/Target/AMDGPU/AMDGPURewriteAGPRCopyMFMA.cpp b/llvm/lib/Target/AMDGPU/AMDGPURewriteAGPRCopyMFMA.cpp
index 5c80a9762ff69..11cafdec8d3c3 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPURewriteAGPRCopyMFMA.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPURewriteAGPRCopyMFMA.cpp
@@ -100,8 +100,8 @@ class AMDGPURewriteAGPRCopyMFMAImpl {
 
   /// Compute the register class constraints based on the uses of \p Reg,
   /// excluding MFMA uses from which can be rewritten to change the register
-  /// class constraint. This should be nearly identical to
-  /// MachineRegisterInfo::recomputeRegClass.
+  /// class constraint. MFMA scale operands need to be constraint checked.
+  /// This should be nearly identical to MachineRegisterInfo::recomputeRegClass.
 
   /// \p RewriteCandidates will collect the set of MFMA instructions that need
   /// to have the opcode mutated to perform the replacement.
@@ -155,9 +155,16 @@ bool AMDGPURewriteAGPRCopyMFMAImpl::recomputeRegClassExceptRewritable(
 
       // We can swap the classes of dst + src2 as a pair to AGPR, so ignore the
       // effects of rewrite candidates. It just so happens that we can use
-      // either AGPR or VGPR in src0/src1, so don't bother checking the
-      // constraint effects of the individual operands.
+      // either AGPR or VGPR in src0/src1. We still need to check constraint
+      // effects for scale variant, which does not allow AGPR.
       if (isRewriteCandidate(*MI)) {
+        int AGPROp = AMDGPU::getMFMASrcCVDstAGPROp(MI->getOpcode());
+        const MCInstrDesc &AGPRDesc = TII.get(AGPROp);
+        const TargetRegisterClass *NewRC =
+            TII.getRegClass(AGPRDesc, MO.getOperandNo());
+        if (!TRI.hasAGPRs(NewRC))
+          return false;
+
         const MachineOperand *VDst =
             TII.getNamedOperand(*MI, AMDGPU::OpName::vdst);
         const MachineOperand *Src2 =
diff --git a/llvm/lib/Target/AMDGPU/GCNSubtarget.h b/llvm/lib/Target/AMDGPU/GCNSubtarget.h
index e567176e658b3..34eb8b2266311 100644
--- a/llvm/lib/Target/AMDGPU/GCNSubtarget.h
+++ b/llvm/lib/Target/AMDGPU/GCNSubtarget.h
@@ -1868,6 +1868,12 @@ class GCNSubtarget final : public AMDGPUGenSubtargetInfo,
     return GFX1250Insts && getGeneration() == GFX12;
   }
 
+  // src_flat_scratch_hi cannot be used as a source in SALU producing a 64-bit
+  // result.
+  bool hasFlatScratchHiInB64InstHazard() const {
+    return GFX1250Insts && getGeneration() == GFX12;
+  }
+
   /// \returns true if the subtarget supports clusters of workgroups.
   bool hasClusters() const { return HasClusters; }
 
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 55370a54f07a0..70f8c0cc2ae9c 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -1309,7 +1309,7 @@ static unsigned getIntrMemWidth(unsigned IntrID) {
   }
 }
 
-static void getCoopAtomicOperandsInfo(const CallInst &CI, bool IsLoad,
+static void getCoopAtomicOperandsInfo(const CallBase &CI, bool IsLoad,
                                       TargetLoweringBase::IntrinsicInfo &Info) {
   Value *OrderingArg = CI.getArgOperand(IsLoad ? 1 : 2);
   unsigned Ord = cast<ConstantInt>(OrderingArg)->getZExtValue();
@@ -1339,7 +1339,7 @@ static void getCoopAtomicOperandsInfo(const CallInst &CI, bool IsLoad,
 }
 
 bool SITargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
-                                          const CallInst &CI,
+                                          const CallBase &CI,
                                           MachineFunction &MF,
                                           unsigned IntrID) const {
   Info.flags = MachineMemOperand::MONone;
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.h b/llvm/lib/Target/AMDGPU/SIISelLowering.h
index 40c03ca024c6c..fb162948caf4c 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.h
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.h
@@ -334,7 +334,7 @@ class SITargetLowering final : public AMDGPUTargetLowering {
   MVT getPointerTy(const DataLayout &DL, unsigned AS) const override;
   MVT getPointerMemTy(const DataLayout &DL, unsigned AS) const override;
 
-  bool getTgtMemIntrinsic(IntrinsicInfo &, const CallInst &,
+  bool getTgtMemIntrinsic(IntrinsicInfo &, const CallBase &,
                           MachineFunction &MF,
                           unsigned IntrinsicID) const override;
 
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
index fa5d2e5b8336b..49c956fd0d893 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -6307,6 +6307,18 @@ bool SIInstrInfo::isLegalRegOperand(const MachineInstr &MI, unsigned OpIdx,
       (int)OpIdx == AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0) &&
       RI.isSGPRReg(MRI, MO.getReg()))
     return false;
+
+  if (ST.hasFlatScratchHiInB64InstHazard() &&
+      MO.getReg() == AMDGPU::SRC_FLAT_SCRATCH_BASE_HI && isSALU(MI)) {
+    if (const MachineOperand *Dst = getNamedOperand(MI, AMDGPU::OpName::sdst)) {
+      if (AMDGPU::getRegBitWidth(*RI.getRegClassForReg(MRI, Dst->getReg())) ==
+          64)
+        return false;
+    }
+    if (Opc == AMDGPU::S_BITCMP0_B64 || Opc == AMDGPU::S_BITCMP1_B64)
+      return false;
+  }
+
   return true;
 }
 
diff --git a/llvm/lib/Target/ARM/ARMISelLowering.cpp b/llvm/lib/Target/ARM/ARMISelLowering.cpp
index 1a8c470600394..2d26c67a8077a 100644
--- a/llvm/lib/Target/ARM/ARMISelLowering.cpp
+++ b/llvm/lib/Target/ARM/ARMISelLowering.cpp
@@ -20665,7 +20665,7 @@ bool ARMTargetLowering::isFPImmLegal(const APFloat &Imm, EVT VT,
 /// MemIntrinsicNodes.  The associated MachineMemOperands record the alignment
 /// specified in the intrinsic calls.
 bool ARMTargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
-                                           const CallInst &I,
+                                           const CallBase &I,
                                            MachineFunction &MF,
                                            unsigned Intrinsic) const {
   switch (Intrinsic) {
diff --git a/llvm/lib/Target/ARM/ARMISelLowering.h b/llvm/lib/Target/ARM/ARMISelLowering.h
index 8191eb40a712a..d0fb58c764edd 100644
--- a/llvm/lib/Target/ARM/ARMISelLowering.h
+++ b/llvm/lib/Target/ARM/ARMISelLowering.h
@@ -315,8 +315,7 @@ class VectorType;
     bool isFPImmLegal(const APFloat &Imm, EVT VT,
                       bool ForCodeSize = false) const override;
 
-    bool getTgtMemIntrinsic(IntrinsicInfo &Info,
-                            const CallInst &I,
+    bool getTgtMemIntrinsic(IntrinsicInfo &Info, const CallBase &I,
                             MachineFunction &MF,
                             unsigned Intrinsic) const override;
 
diff --git a/llvm/lib/Target/ARM/ARMInstrMVE.td b/llvm/lib/Target/ARM/ARMInstrMVE.td
index f9aaacb7f5250..097318711d137 100644
--- a/llvm/lib/Target/ARM/ARMInstrMVE.td
+++ b/llvm/lib/Target/ARM/ARMInstrMVE.td
@@ -393,6 +393,12 @@ def vsub : PatFrags<(ops node:$lhs, node:$rhs),
 def vmul : PatFrags<(ops node:$lhs, node:$rhs),
                     [(fmul node:$lhs, node:$rhs),
                      (int_arm_mve_vmul node:$lhs, node:$rhs)]>;
+def vminnm : PatFrags<(ops node:$lhs, node:$rhs),
+                    [(fminnum node:$lhs, node:$rhs),
+                     (int_arm_mve_vminnm node:$lhs, node:$rhs)]>;
+def vmaxnm : PatFrags<(ops node:$lhs, node:$rhs),
+                    [(fmaxnum node:$lhs, node:$rhs),
+                     (int_arm_mve_vmaxnm node:$lhs, node:$rhs)]>;
 
 // --------- Start of base classes for the instructions themselves
 
@@ -1489,7 +1495,7 @@ class MVE_VMINMAXNM<string iname, string suffix, bits<2> sz, bit bit_21,
   let validForTailPredication = 1;
 }
 
-multiclass MVE_VMINMAXNM_m<string iname, bit bit_4, MVEVectorVTInfo VTI, SDNode Op, Intrinsic PredInt> {
+multiclass MVE_VMINMAXNM_m<string iname, bit bit_4, MVEVectorVTInfo VTI, SDPatternOperator Op, Intrinsic PredInt> {
   def "" : MVE_VMINMAXNM<iname, VTI.Suffix, VTI.Size, bit_4>;
 
   let Predicates = [HasMVEFloat] in {
@@ -1497,10 +1503,10 @@ multiclass MVE_VMINMAXNM_m<string iname, bit bit_4, MVEVectorVTInfo VTI, SDNode
   }
 }
 
-defm MVE_VMAXNMf32 : MVE_VMINMAXNM_m<"vmaxnm", 0b0, MVE_v4f32, fmaxnum, int_arm_mve_max_predicated>;
-defm MVE_VMAXNMf16 : MVE_VMINMAXNM_m<"vmaxnm", 0b0, MVE_v8f16, fmaxnum, int_arm_mve_max_predicated>;
-defm MVE_VMINNMf32 : MVE_VMINMAXNM_m<"vminnm", 0b1, MVE_v4f32, fminnum, int_arm_mve_min_predicated>;
-defm MVE_VMINNMf16 : MVE_VMINMAXNM_m<"vminnm", 0b1, MVE_v8f16, fminnum, int_arm_mve_min_predicated>;
+defm MVE_VMAXNMf32 : MVE_VMINMAXNM_m<"vmaxnm", 0b0, MVE_v4f32, vmaxnm, int_arm_mve_max_predicated>;
+defm MVE_VMAXNMf16 : MVE_VMINMAXNM_m<"vmaxnm", 0b0, MVE_v8f16, vmaxnm, int_arm_mve_max_predicated>;
+defm MVE_VMINNMf32 : MVE_VMINMAXNM_m<"vminnm", 0b1, MVE_v4f32, vminnm, int_arm_mve_min_predicated>;
+defm MVE_VMINNMf16 : MVE_VMINMAXNM_m<"vminnm", 0b1, MVE_v8f16, vminnm, int_arm_mve_min_predicated>;
 
 
 class MVE_VMINMAX<string iname, string suffix, bit U, bits<2> size,
@@ -4148,7 +4154,7 @@ class MVE_VMAXMINNMA<string iname, string suffix, bits<2> size, bit bit_12,
 }
 
 multiclass MVE_VMAXMINNMA_m<string iname, MVEVectorVTInfo VTI,
-                      SDNode unpred_op, Intrinsic pred_int,
+                      SDPatternOperator unpred_op, Intrinsic pred_int,
                       bit bit_12> {
   def "" : MVE_VMAXMINNMA<iname, VTI.Suffix, VTI.Size, bit_12>;
   defvar Inst = !cast<Instruction>(NAME);
@@ -4168,13 +4174,13 @@ multiclass MVE_VMAXMINNMA_m<string iname, MVEVectorVTInfo VTI,
 }
 
 multiclass MVE_VMAXNMA<MVEVectorVTInfo VTI, bit bit_12>
-  : MVE_VMAXMINNMA_m<"vmaxnma", VTI, fmaxnum, int_arm_mve_vmaxnma_predicated, bit_12>;
+  : MVE_VMAXMINNMA_m<"vmaxnma", VTI, vmaxnm, int_arm_mve_vmaxnma_predicated, bit_12>;
 
 defm MVE_VMAXNMAf32 : MVE_VMAXNMA<MVE_v4f32, 0b0>;
 defm MVE_VMAXNMAf16 : MVE_VMAXNMA<MVE_v8f16, 0b0>;
 
 multiclass MVE_VMINNMA<MVEVectorVTInfo VTI, bit bit_12>
-  : MVE_VMAXMINNMA_m<"vminnma", VTI, fminnum, int_arm_mve_vminnma_predicated, bit_12>;
+  : MVE_VMAXMINNMA_m<"vminnma", VTI, vminnm, int_arm_mve_vminnma_predicated, bit_12>;
 
 defm MVE_VMINNMAf32 : MVE_VMINNMA<MVE_v4f32, 0b1>;
 defm MVE_VMINNMAf16 : MVE_VMINNMA<MVE_v8f16, 0b1>;
diff --git a/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp b/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp
index 5767a74513e8d..bae9d705f5a7a 100644
--- a/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp
@@ -2115,7 +2115,7 @@ static Value *getUnderLyingObjectForBrevLdIntr(Value *V) {
 /// true and store the intrinsic information into the IntrinsicInfo that was
 /// passed to the function.
 bool HexagonTargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
-                                               const CallInst &I,
+                                               const CallBase &I,
                                                MachineFunction &MF,
                                                unsigned Intrinsic) const {
   switch (Intrinsic) {
diff --git a/llvm/lib/Target/Hexagon/HexagonISelLowering.h b/llvm/lib/Target/Hexagon/HexagonISelLowering.h
index f4d2a79051c10..cde8b5ba8d8a7 100644
--- a/llvm/lib/Target/Hexagon/HexagonISelLowering.h
+++ b/llvm/lib/Target/Hexagon/HexagonISelLowering.h
@@ -145,7 +145,7 @@ class HexagonTargetLowering : public TargetLowering {
       const SmallVectorImpl<SDValue> &OutVals,
       const SmallVectorImpl<ISD::InputArg> &Ins, SelectionDAG& DAG) const;
 
-  bool getTgtMemIntrinsic(IntrinsicInfo &Info, const CallInst &I,
+  bool getTgtMemIntrinsic(IntrinsicInfo &Info, const CallBase &I,
                           MachineFunction &MF,
                           unsigned Intrinsic) const override;
 
diff --git a/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp b/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp
index ba9d0682b26dd..32ea2198f7898 100644
--- a/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp
+++ b/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp
@@ -8912,7 +8912,7 @@ bool LoongArchTargetLowering::hasAndNot(SDValue Y) const {
 }
 
 bool LoongArchTargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
-                                                 const CallInst &I,
+                                                 const CallBase &I,
                                                  MachineFunction &MF,
                                                  unsigned Intrinsic) const {
   switch (Intrinsic) {
diff --git a/llvm/lib/Target/LoongArch/LoongArchISelLowering.h b/llvm/lib/Target/LoongArch/LoongArchISelLowering.h
index 0c09fb6afd2d1..5277e7e3e74ca 100644
--- a/llvm/lib/Target/LoongArch/LoongArchISelLowering.h
+++ b/llvm/lib/Target/LoongArch/LoongArchISelLowering.h
@@ -78,7 +78,7 @@ class LoongArchTargetLowering : public TargetLowering {
                                           Value *NewVal, Value *Mask,
                                           AtomicOrdering Ord) const override;
 
-  bool getTgtMemIntrinsic(IntrinsicInfo &Info, const CallInst &I,
+  bool getTgtMemIntrinsic(IntrinsicInfo &Info, const CallBase &I,
                           MachineFunction &MF,
                           unsigned Intrinsic) const override;
 
diff --git a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
index 8b72b1e1f3a52..5081a093d4c34 100644
--- a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
@@ -4077,9 +4077,10 @@ void NVPTXTargetLowering::LowerAsmOperandForConstraint(
 // because we need the information that is only available in the "Value" type
 // of destination
 // pointer. In particular, the address space information.
-bool NVPTXTargetLowering::getTgtMemIntrinsic(
-    IntrinsicInfo &Info, const CallInst &I,
-    MachineFunction &MF, unsigned Intrinsic) const {
+bool NVPTXTargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
+                                             const CallBase &I,
+                                             MachineFunction &MF,
+                                             unsigned Intrinsic) const {
   switch (Intrinsic) {
   default:
     return false;
diff --git a/llvm/lib/Target/NVPTX/NVPTXISelLowering.h b/llvm/lib/Target/NVPTX/NVPTXISelLowering.h
index dd8e49de7aa6a..cb0a1aa5dc892 100644
--- a/llvm/lib/Target/NVPTX/NVPTXISelLowering.h
+++ b/llvm/lib/Target/NVPTX/NVPTXISelLowering.h
@@ -32,7 +32,7 @@ class NVPTXTargetLowering : public TargetLowering {
                                const NVPTXSubtarget &STI);
   SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const override;
 
-  bool getTgtMemIntrinsic(IntrinsicInfo &Info, const CallInst &I,
+  bool getTgtMemIntrinsic(IntrinsicInfo &Info, const CallBase &I,
                           MachineFunction &MF,
                           unsigned Intrinsic) const override;
 
diff --git a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
index 1a9310c46cd1d..51212837fbb17 100644
--- a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
+++ b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
@@ -18495,7 +18495,7 @@ PPCTargetLowering::isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const {
 }
 
 bool PPCTargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
-                                           const CallInst &I,
+                                           const CallBase &I,
                                            MachineFunction &MF,
                                            unsigned Intrinsic) const {
   switch (Intrinsic) {
diff --git a/llvm/lib/Target/PowerPC/PPCISelLowering.h b/llvm/lib/Target/PowerPC/PPCISelLowering.h
index 74af055ed5d30..daae839479c3c 100644
--- a/llvm/lib/Target/PowerPC/PPCISelLowering.h
+++ b/llvm/lib/Target/PowerPC/PPCISelLowering.h
@@ -492,8 +492,7 @@ namespace llvm {
 
     bool isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const override;
 
-    bool getTgtMemIntrinsic(IntrinsicInfo &Info,
-                            const CallInst &I,
+    bool getTgtMemIntrinsic(IntrinsicInfo &Info, const CallBase &I,
                             MachineFunction &MF,
                             unsigned Intrinsic) const override;
 
diff --git a/llvm/lib/Target/RISCV/GISel/RISCVInstructionSelector.cpp b/llvm/lib/Target/RISCV/GISel/RISCVInstructionSelector.cpp
index 3d5a55c631301..1e5d0a4297465 100644
--- a/llvm/lib/Target/RISCV/GISel/RISCVInstructionSelector.cpp
+++ b/llvm/lib/Target/RISCV/GISel/RISCVInstructionSelector.cpp
@@ -1569,7 +1569,7 @@ bool RISCVInstructionSelector::selectAddr(MachineInstr &MI,
 
   switch (TM.getCodeModel()) {
   default: {
-    reportGISelFailure(*MF, *TPC, *MORE, getName(),
+    reportGISelFailure(*MF, *MORE, getName(),
                        "Unsupported code model for lowering", MI);
     return false;
   }
diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
index 039bd55718e24..ab2652eac3823 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
@@ -1868,7 +1868,7 @@ bool RISCVTargetLowering::shouldExpandCttzElements(EVT VT) const {
 }
 
 bool RISCVTargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
-                                             const CallInst &I,
+                                             const CallBase &I,
                                              MachineFunction &MF,
                                              unsigned Intrinsic) const {
   auto &DL = I.getDataLayout();
diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.h b/llvm/lib/Target/RISCV/RISCVISelLowering.h
index 69fcada6494a2..8a55a5634452c 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.h
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.h
@@ -35,7 +35,7 @@ class RISCVTargetLowering : public TargetLowering {
 
   const RISCVSubtarget &getSubtarget() const { return Subtarget; }
 
-  bool getTgtMemIntrinsic(IntrinsicInfo &Info, const CallInst &I,
+  bool getTgtMemIntrinsic(IntrinsicInfo &Info, const CallBase &I,
                           MachineFunction &MF,
                           unsigned Intrinsic) const override;
   bool isLegalAddressingMode(const DataLayout &DL, const AddrMode &AM, Type *Ty,
diff --git a/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp b/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp
index ae81d38579c18..0fb44052527f0 100644
--- a/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp
@@ -883,10 +883,12 @@ SPIRVType *SPIRVGlobalRegistry::getOpTypeArray(uint32_t NumElems,
           .addUse(NumElementsVReg);
     });
   } else {
-    assert(ST.isShader() && "Runtime arrays are not allowed in non-shader "
-                            "SPIR-V modules.");
-    if (!ST.isShader())
+    if (!ST.isShader()) {
+      llvm::reportFatalUsageError(
+          "Runtime arrays are not allowed in non-shader "
+          "SPIR-V modules");
       return nullptr;
+    }
     ArrayType = createOpType(MIRBuilder, [&](MachineIRBuilder &MIRBuilder) {
       return MIRBuilder.buildInstr(SPIRV::OpTypeRuntimeArray)
           .addDef(createTypeVReg(MIRBuilder))
diff --git a/llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp b/llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp
index 0ba6589c68944..36fa5fa9a70cb 100644
--- a/llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp
@@ -94,7 +94,7 @@ MVT SPIRVTargetLowering::getRegisterTypeForCallingConv(LLVMContext &Context,
 }
 
 bool SPIRVTargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
-                                             const CallInst &I,
+                                             const CallBase &I,
                                              MachineFunction &MF,
                                              unsigned Intrinsic) const {
   unsigned AlignIdx = 3;
diff --git a/llvm/lib/Target/SPIRV/SPIRVISelLowering.h b/llvm/lib/Target/SPIRV/SPIRVISelLowering.h
index 3d31a116bad4a..5746832c8fd95 100644
--- a/llvm/lib/Target/SPIRV/SPIRVISelLowering.h
+++ b/llvm/lib/Target/SPIRV/SPIRVISelLowering.h
@@ -48,7 +48,7 @@ class SPIRVTargetLowering : public TargetLowering {
                                          EVT VT) const override;
   MVT getRegisterTypeForCallingConv(LLVMContext &Context, CallingConv::ID CC,
                                     EVT VT) const override;
-  bool getTgtMemIntrinsic(IntrinsicInfo &Info, const CallInst &I,
+  bool getTgtMemIntrinsic(IntrinsicInfo &Info, const CallBase &I,
                           MachineFunction &MF,
                           unsigned Intrinsic) const override;
 
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp
index 98cb7aba562c4..e0c527b9b2581 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp
@@ -1060,7 +1060,7 @@ EVT WebAssemblyTargetLowering::getSetCCResultType(const DataLayout &DL,
 }
 
 bool WebAssemblyTargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
-                                                   const CallInst &I,
+                                                   const CallBase &I,
                                                    MachineFunction &MF,
                                                    unsigned Intrinsic) const {
   switch (Intrinsic) {
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.h b/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.h
index f7052989b3c75..c37970f458e36 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.h
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.h
@@ -58,7 +58,7 @@ class WebAssemblyTargetLowering final : public TargetLowering {
   bool isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const override;
   EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Context,
                          EVT VT) const override;
-  bool getTgtMemIntrinsic(IntrinsicInfo &Info, const CallInst &I,
+  bool getTgtMemIntrinsic(IntrinsicInfo &Info, const CallBase &I,
                           MachineFunction &MF,
                           unsigned Intrinsic) const override;
 
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyInstrInteger.td b/llvm/lib/Target/WebAssembly/WebAssemblyInstrInteger.td
index eb692679f5971..991507e883f28 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyInstrInteger.td
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyInstrInteger.td
@@ -109,6 +109,10 @@ def : Pat<(rotr I64:$lhs, (and I64:$rhs, 63)), (ROTR_I64 I64:$lhs, I64:$rhs)>;
 
 def : Pat<(shl I64:$lhs, (zext (and I32:$rhs, 63))),
                                (SHL_I64 I64:$lhs, (I64_EXTEND_U_I32 I32:$rhs))>;
+def : Pat<(sra I64:$lhs, (zext (and I32:$rhs, 63))),
+                               (SHR_S_I64 I64:$lhs, (I64_EXTEND_U_I32 I32:$rhs))>;
+def : Pat<(srl I64:$lhs, (zext (and I32:$rhs, 63))),
+                               (SHR_U_I64 I64:$lhs, (I64_EXTEND_U_I32 I32:$rhs))>;
 
 defm SELECT_I32 : I<(outs I32:$dst), (ins I32:$lhs, I32:$rhs, I32:$cond),
                     (outs), (ins),
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 9da121dd9ab87..d46f0f1572f1f 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -3104,7 +3104,7 @@ static bool useVPTERNLOG(const X86Subtarget &Subtarget, MVT VT) {
 }
 
 bool X86TargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
-                                           const CallInst &I,
+                                           const CallBase &I,
                                            MachineFunction &MF,
                                            unsigned Intrinsic) const {
   Info.flags = MachineMemOperand::MONone;
diff --git a/llvm/lib/Target/X86/X86ISelLowering.h b/llvm/lib/Target/X86/X86ISelLowering.h
index c5085299716ed..848fe4bf86d2c 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.h
+++ b/llvm/lib/Target/X86/X86ISelLowering.h
@@ -1482,7 +1482,7 @@ namespace llvm {
     /// to a MemIntrinsicNode (touches memory). If this is the case, it returns
     /// true and stores the intrinsic information into the IntrinsicInfo that was
     /// passed to the function.
-    bool getTgtMemIntrinsic(IntrinsicInfo &Info, const CallInst &I,
+    bool getTgtMemIntrinsic(IntrinsicInfo &Info, const CallBase &I,
                             MachineFunction &MF,
                             unsigned Intrinsic) const override;
 
diff --git a/llvm/lib/Transforms/IPO/LowerTypeTests.cpp b/llvm/lib/Transforms/IPO/LowerTypeTests.cpp
index fa35eef2c00f5..f7aeda95e41b3 100644
--- a/llvm/lib/Transforms/IPO/LowerTypeTests.cpp
+++ b/llvm/lib/Transforms/IPO/LowerTypeTests.cpp
@@ -1554,12 +1554,7 @@ void LowerTypeTestsModule::createJumpTable(
 
   // Align the whole table by entry size.
   F->setAlignment(Align(getJumpTableEntrySize(JumpTableArch)));
-  // Skip prologue.
-  // Disabled on win32 due to https://llvm.org/bugs/show_bug.cgi?id=28641#c3.
-  // Luckily, this function does not get any prologue even without the
-  // attribute.
-  if (OS != Triple::Win32)
-    F->addFnAttr(Attribute::Naked);
+  F->addFnAttr(Attribute::Naked);
   if (JumpTableArch == Triple::arm)
     F->addFnAttr("target-features", "-thumb-mode");
   if (JumpTableArch == Triple::thumb) {
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineLoadStoreAlloca.cpp b/llvm/lib/Transforms/InstCombine/InstCombineLoadStoreAlloca.cpp
index 3e04aeb675d2a..9491610190c10 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineLoadStoreAlloca.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineLoadStoreAlloca.cpp
@@ -415,7 +415,7 @@ void PointerReplacer::replace(Instruction *I) {
                               LT->getAlign(), LT->getOrdering(),
                               LT->getSyncScopeID());
     NewI->takeName(LT);
-    copyMetadataForAccess(*NewI, *LT);
+    copyMetadataForLoad(*NewI, *LT);
 
     IC.InsertNewInstWith(NewI, LT->getIterator());
     IC.replaceInstUsesWith(*LT, NewI);
@@ -606,7 +606,7 @@ LoadInst *InstCombinerImpl::combineLoadToNewType(LoadInst &LI, Type *NewTy,
       Builder.CreateAlignedLoad(NewTy, LI.getPointerOperand(), LI.getAlign(),
                                 LI.isVolatile(), LI.getName() + Suffix);
   NewLoad->setAtomic(LI.getOrdering(), LI.getSyncScopeID());
-  copyMetadataForAccess(*NewLoad, LI);
+  copyMetadataForLoad(*NewLoad, LI);
   return NewLoad;
 }
 
diff --git a/llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp b/llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp
index 001215abcfb26..3af67ff6ac3f5 100644
--- a/llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp
@@ -6024,33 +6024,34 @@ void LSRInstance::Rewrite(const LSRUse &LU, const LSRFixup &LF,
     DeadInsts.emplace_back(OperandIsInstr);
 }
 
-// Trying to hoist the IVInc to loop header if all IVInc users are in
-// the loop header. It will help backend to generate post index load/store
-// when the latch block is different from loop header block.
-static bool canHoistIVInc(const TargetTransformInfo &TTI, const LSRFixup &Fixup,
-                          const LSRUse &LU, Instruction *IVIncInsertPos,
-                          Loop *L) {
+// Determine where to insert the transformed IV increment instruction for this
+// fixup. By default this is the default insert position, but if this is a
+// postincrement opportunity then we try to insert it in the same block as the
+// fixup user instruction, as this is needed for a postincrement instruction to
+// be generated.
+static Instruction *getFixupInsertPos(const TargetTransformInfo &TTI,
+                                      const LSRFixup &Fixup, const LSRUse &LU,
+                                      Instruction *IVIncInsertPos,
+                                      DominatorTree &DT) {
+  // Only address uses can be postincremented
   if (LU.Kind != LSRUse::Address)
-    return false;
-
-  // For now this code do the conservative optimization, only work for
-  // the header block. Later we can hoist the IVInc to the block post
-  // dominate all users.
-  BasicBlock *LHeader = L->getHeader();
-  if (IVIncInsertPos->getParent() == LHeader)
-    return false;
-
-  if (!Fixup.OperandValToReplace ||
-      any_of(Fixup.OperandValToReplace->users(), [&LHeader](User *U) {
-        Instruction *UI = cast<Instruction>(U);
-        return UI->getParent() != LHeader;
-      }))
-    return false;
+    return IVIncInsertPos;
 
+  // Don't try to postincrement if it's not legal
   Instruction *I = Fixup.UserInst;
   Type *Ty = I->getType();
-  return (isa<LoadInst>(I) && TTI.isIndexedLoadLegal(TTI.MIM_PostInc, Ty)) ||
-         (isa<StoreInst>(I) && TTI.isIndexedStoreLegal(TTI.MIM_PostInc, Ty));
+  if (!(isa<LoadInst>(I) && TTI.isIndexedLoadLegal(TTI.MIM_PostInc, Ty)) &&
+      !(isa<StoreInst>(I) && TTI.isIndexedStoreLegal(TTI.MIM_PostInc, Ty)))
+    return IVIncInsertPos;
+
+  // It's only legal to hoist to the user block if it dominates the default
+  // insert position.
+  BasicBlock *HoistBlock = I->getParent();
+  BasicBlock *IVIncBlock = IVIncInsertPos->getParent();
+  if (!DT.dominates(I, IVIncBlock))
+    return IVIncInsertPos;
+
+  return HoistBlock->getTerminator();
 }
 
 /// Rewrite all the fixup locations with new values, following the chosen
@@ -6071,9 +6072,7 @@ void LSRInstance::ImplementSolution(
   for (size_t LUIdx = 0, NumUses = Uses.size(); LUIdx != NumUses; ++LUIdx)
     for (const LSRFixup &Fixup : Uses[LUIdx].Fixups) {
       Instruction *InsertPos =
-          canHoistIVInc(TTI, Fixup, Uses[LUIdx], IVIncInsertPos, L)
-              ? L->getHeader()->getTerminator()
-              : IVIncInsertPos;
+          getFixupInsertPos(TTI, Fixup, Uses[LUIdx], IVIncInsertPos, DT);
       Rewriter.setIVIncInsertPos(L, InsertPos);
       Rewrite(Uses[LUIdx], Fixup, *Solution[LUIdx], DeadInsts);
       Changed = true;
diff --git a/llvm/lib/Transforms/Scalar/SROA.cpp b/llvm/lib/Transforms/Scalar/SROA.cpp
index e3dceb7677043..cb44a0d2b4d6b 100644
--- a/llvm/lib/Transforms/Scalar/SROA.cpp
+++ b/llvm/lib/Transforms/Scalar/SROA.cpp
@@ -3150,7 +3150,6 @@ class AllocaSliceRewriter : public InstVisitor<AllocaSliceRewriter, bool> {
     assert(IsSplit || BeginOffset == NewBeginOffset);
     uint64_t Offset = NewBeginOffset - NewAllocaBeginOffset;
 
-#ifndef NDEBUG
     StringRef OldName = OldPtr->getName();
     // Skip through the last '.sroa.' component of the name.
     size_t LastSROAPrefix = OldName.rfind(".sroa.");
@@ -3169,17 +3168,10 @@ class AllocaSliceRewriter : public InstVisitor<AllocaSliceRewriter, bool> {
     }
     // Strip any SROA suffixes as well.
     OldName = OldName.substr(0, OldName.find(".sroa_"));
-#endif
 
     return getAdjustedPtr(IRB, DL, &NewAI,
                           APInt(DL.getIndexTypeSizeInBits(PointerTy), Offset),
-                          PointerTy,
-#ifndef NDEBUG
-                          Twine(OldName) + "."
-#else
-                          Twine()
-#endif
-    );
+                          PointerTy, Twine(OldName) + ".");
   }
 
   /// Compute suitable alignment to access this slice of the *new*
@@ -3280,7 +3272,7 @@ class AllocaSliceRewriter : public InstVisitor<AllocaSliceRewriter, bool> {
       // Copy any metadata that is valid for the new load. This may require
       // conversion to a different kind of metadata, e.g. !nonnull might change
       // to !range or vice versa.
-      copyMetadataForAccess(*NewLI, LI);
+      copyMetadataForLoad(*NewLI, LI);
 
       // Do this after copyMetadataForLoad() to preserve the TBAA shift.
       if (AATags)
diff --git a/llvm/lib/Transforms/Utils/Local.cpp b/llvm/lib/Transforms/Utils/Local.cpp
index 374b48a24f80e..d741a8a8049bc 100644
--- a/llvm/lib/Transforms/Utils/Local.cpp
+++ b/llvm/lib/Transforms/Utils/Local.cpp
@@ -3455,70 +3455,54 @@ void llvm::combineAAMetadata(Instruction *K, const Instruction *J) {
   combineMetadata(K, J, /*DoesKMove=*/true, /*AAOnly=*/true);
 }
 
-void llvm::copyMetadataForAccess(Instruction &DestI, Instruction &SourceI) {
+void llvm::copyMetadataForLoad(LoadInst &Dest, const LoadInst &Source) {
   SmallVector<std::pair<unsigned, MDNode *>, 8> MD;
-  SourceI.getAllMetadata(MD);
-  MDBuilder MDB(DestI.getContext());
-  Type *NewType = DestI.getType();
-
-  // Only needed for range metadata on loads.
-  const DataLayout *DL = nullptr;
-  const LoadInst *LSource = dyn_cast<LoadInst>(&SourceI);
-  if (LSource)
-    DL = &LSource->getDataLayout();
-
+  Source.getAllMetadata(MD);
+  MDBuilder MDB(Dest.getContext());
+  Type *NewType = Dest.getType();
+  const DataLayout &DL = Source.getDataLayout();
   for (const auto &MDPair : MD) {
     unsigned ID = MDPair.first;
     MDNode *N = MDPair.second;
-
+    // Note, essentially every kind of metadata should be preserved here! This
+    // routine is supposed to clone a load instruction changing *only its type*.
+    // The only metadata it makes sense to drop is metadata which is invalidated
+    // when the pointer type changes. This should essentially never be the case
+    // in LLVM, but we explicitly switch over only known metadata to be
+    // conservatively correct. If you are adding metadata to LLVM which pertains
+    // to loads, you almost certainly want to add it here.
     switch (ID) {
-    // Applies to both loads and stores as-is.
     case LLVMContext::MD_dbg:
+    case LLVMContext::MD_tbaa:
     case LLVMContext::MD_prof:
+    case LLVMContext::MD_fpmath:
     case LLVMContext::MD_tbaa_struct:
+    case LLVMContext::MD_invariant_load:
     case LLVMContext::MD_alias_scope:
     case LLVMContext::MD_noalias:
     case LLVMContext::MD_nontemporal:
+    case LLVMContext::MD_mem_parallel_loop_access:
     case LLVMContext::MD_access_group:
     case LLVMContext::MD_noundef:
     case LLVMContext::MD_noalias_addrspace:
-    case LLVMContext::MD_mem_parallel_loop_access:
-      DestI.setMetadata(ID, N);
-      break;
-
-    // Load-only metadata.
-    case LLVMContext::MD_fpmath:
-    case LLVMContext::MD_invariant_load:
-      if (isa<LoadInst>(DestI))
-        DestI.setMetadata(ID, N);
+      // All of these directly apply.
+      Dest.setMetadata(ID, N);
       break;
 
     case LLVMContext::MD_nonnull:
-      if (auto *LDest = dyn_cast<LoadInst>(&DestI)) {
-        if (LSource)
-          copyNonnullMetadata(*LSource, N, *LDest);
-      }
+      copyNonnullMetadata(Source, N, Dest);
       break;
 
     case LLVMContext::MD_align:
     case LLVMContext::MD_dereferenceable:
     case LLVMContext::MD_dereferenceable_or_null:
-      // Applies to both loads and stores only if the new type is also a
-      // pointer.
+      // These only directly apply if the new type is also a pointer.
       if (NewType->isPointerTy())
-        DestI.setMetadata(ID, N);
+        Dest.setMetadata(ID, N);
       break;
 
     case LLVMContext::MD_range:
-      if (auto *LDest = dyn_cast<LoadInst>(&DestI)) {
-        if (LSource && DL)
-          copyRangeMetadata(*DL, *LSource, N, *LDest);
-      }
-      break;
-
-    case LLVMContext::MD_tbaa:
-      if (isa<LoadInst>(DestI))
-        DestI.setMetadata(ID, N);
+      copyRangeMetadata(DL, Source, N, Dest);
       break;
     }
   }
diff --git a/llvm/lib/Transforms/Vectorize/LoadStoreVectorizer.cpp b/llvm/lib/Transforms/Vectorize/LoadStoreVectorizer.cpp
index 114df653bad83..c28314f6ab124 100644
--- a/llvm/lib/Transforms/Vectorize/LoadStoreVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoadStoreVectorizer.cpp
@@ -112,7 +112,6 @@
 #include <optional>
 #include <tuple>
 #include <type_traits>
-#include <unordered_map>
 #include <utility>
 #include <vector>
 
@@ -269,6 +268,11 @@ class Vectorizer {
   /// isGuaranteedToTransferExecutionToSuccessor(I) == true.
   bool runOnPseudoBB(BasicBlock::iterator Begin, BasicBlock::iterator End);
 
+  /// Runs the vectorizer on one equivalence class, i.e. one set of loads/stores
+  /// in the same BB with the same value for getUnderlyingObject() etc.
+  bool runOnEquivalenceClass(const EqClassKey &EqClassKey,
+                             ArrayRef<Instruction *> EqClass);
+
   /// Runs the vectorizer on one chain, i.e. a subset of an equivalence class
   /// where all instructions access a known, constant offset from the first
   /// instruction.
@@ -334,22 +338,12 @@ class Vectorizer {
   EquivalenceClassMap collectEquivalenceClasses(BasicBlock::iterator Begin,
                                                 BasicBlock::iterator End);
 
-  /// Inserts a cast instruction to convert Inst to DstTy.
-  Value *insertCast(Value *Val, Type *DstTy);
-
   /// Partitions Instrs into "chains" where every instruction has a known
   /// constant offset from the first instr in the chain.
   ///
   /// Postcondition: For all i, ret[i][0].second == 0, because the first instr
   /// in the chain is the leader, and an instr touches distance 0 from itself.
   std::vector<Chain> gatherChains(ArrayRef<Instruction *> Instrs);
-
-  // Helpers for chain merging.
-  std::optional<APInt> computeLeaderDelta(Instruction *I1, Instruction *I2);
-  bool chainsOverlapAfterRebase(const Chain &A, const Chain &B,
-                                const APInt &Delta) const;
-  static void rebaseChain(Chain &C, const APInt &Delta);
-  void normalizeChainToType(Chain &C, Type *CastTy);
 };
 
 class LoadStoreVectorizerLegacyPass : public FunctionPass {
@@ -431,20 +425,6 @@ PreservedAnalyses LoadStoreVectorizerPass::run(Function &F,
   return Changed ? PA : PreservedAnalyses::all();
 }
 
-static const Value *getUnderlyingObject(const Value *Ptr) {
-  const Value *ObjPtr = llvm::getUnderlyingObject(Ptr);
-  if (const auto *Sel = dyn_cast<SelectInst>(ObjPtr)) {
-    // The select's themselves are distinct instructions even if they share
-    // the same condition and evaluate to consecutive pointers for true and
-    // false values of the condition. Therefore using the select's themselves
-    // for grouping instructions would put consecutive accesses into different
-    // lists and they won't be even checked for being consecutive, and won't
-    // be vectorized.
-    return Sel->getCondition();
-  }
-  return ObjPtr;
-}
-
 bool Vectorizer::run() {
   bool Changed = false;
   // Break up the BB if there are any instrs which aren't guaranteed to transfer
@@ -488,88 +468,6 @@ bool Vectorizer::run() {
   return Changed;
 }
 
-Value *Vectorizer::insertCast(Value *Val, Type *DstTy) {
-  if (DL.getTypeSizeInBits(Val->getType()) == DL.getTypeSizeInBits(DstTy)) {
-    return Builder.CreateBitOrPointerCast(Val, DstTy, Val->getName() + ".bc");
-  }
-
-  // If the types are of different sizes and both are integers, we can use
-  // zext or sext to cast.
-  if (Val->getType()->isIntegerTy() && DstTy->isIntegerTy()) {
-    if (DL.getTypeSizeInBits(Val->getType()) < DL.getTypeSizeInBits(DstTy)) {
-      return Builder.CreateZExt(Val, DstTy, Val->getName() + ".bc");
-    }
-    return Builder.CreateTrunc(Val, DstTy, Val->getName() + ".bc");
-  }
-
-  return nullptr;
-}
-
-std::optional<APInt> Vectorizer::computeLeaderDelta(Instruction *I1,
-                                                    Instruction *I2) {
-  assert(((isa<LoadInst>(I1) && isa<LoadInst>(I2)) ||
-          (isa<StoreInst>(I1) && isa<StoreInst>(I2))) &&
-         "computeLeaderDelta must be called with two load or two store "
-         "instructions");
-  Instruction *CtxInst = I1->comesBefore(I2) ? I2 : I1;
-  const Value *Ptr1 = getLoadStorePointerOperand(I1);
-  const Value *Ptr2 = getLoadStorePointerOperand(I2);
-  return getConstantOffset(const_cast<Value *>(Ptr1), const_cast<Value *>(Ptr2),
-                           CtxInst);
-}
-
-bool Vectorizer::chainsOverlapAfterRebase(const Chain &A, const Chain &B,
-                                          const APInt &Delta) const {
-  ConstantRange ARange(
-      A.front().OffsetFromLeader,
-      A.back().OffsetFromLeader +
-          DL.getTypeStoreSize(getLoadStoreType(A.back().Inst)));
-  ConstantRange BRange(
-      B.front().OffsetFromLeader + Delta,
-      B.back().OffsetFromLeader + Delta +
-          DL.getTypeStoreSize(getLoadStoreType(B.back().Inst)));
-  return !ARange.intersectWith(BRange).isEmptySet();
-}
-
-void Vectorizer::rebaseChain(Chain &C, const APInt &Delta) {
-  for (ChainElem &E : C)
-    E.OffsetFromLeader += Delta;
-}
-
-void Vectorizer::normalizeChainToType(Chain &C, Type *CastTy) {
-  for (ChainElem &Elem : C) {
-    Instruction *Inst = Elem.Inst;
-    Type *OrigValTy = getLoadStoreType(Inst);
-    if (OrigValTy == CastTy)
-      continue;
-
-    if (auto *LI = dyn_cast<LoadInst>(Inst)) {
-      Builder.SetInsertPoint(LI);
-      LoadInst *NewLoad = Builder.CreateLoad(CastTy, LI->getPointerOperand(),
-                                             LI->getName() + ".mut");
-      copyMetadataForAccess(*NewLoad, *LI);
-      Value *CastBack = insertCast(NewLoad, OrigValTy);
-      if (!CastBack)
-        llvm_unreachable("Failed to insert cast");
-      LI->replaceAllUsesWith(CastBack);
-      ToErase.emplace_back(LI);
-      Elem.Inst = NewLoad;
-    } else if (auto *SI = dyn_cast<StoreInst>(Inst)) {
-      Builder.SetInsertPoint(SI);
-      Value *CastVal = insertCast(SI->getValueOperand(), CastTy);
-      if (!CastVal)
-        llvm_unreachable("Failed to insert cast");
-      StoreInst *NewStore =
-          Builder.CreateStore(CastVal, SI->getPointerOperand());
-      NewStore->setAlignment(SI->getAlign());
-      NewStore->setVolatile(SI->isVolatile());
-      copyMetadataForAccess(*NewStore, *SI);
-      ToErase.emplace_back(SI);
-      Elem.Inst = NewStore;
-    }
-  }
-}
-
 bool Vectorizer::runOnPseudoBB(BasicBlock::iterator Begin,
                                BasicBlock::iterator End) {
   LLVM_DEBUG({
@@ -582,120 +480,49 @@ bool Vectorizer::runOnPseudoBB(BasicBlock::iterator Begin,
   });
 
   bool Changed = false;
-  SmallVector<Chain> ContiguousSubChains;
-
   for (const auto &[EqClassKey, EqClass] :
-       collectEquivalenceClasses(Begin, End)) {
-
-    LLVM_DEBUG({
-      dbgs() << "LSV: Running on equivalence class of size " << EqClass.size()
-             << " keyed on " << EqClassKey << ":\n";
-      for (Instruction *I : EqClass)
-        dbgs() << "  " << *I << "\n";
-    });
-
-    for (Chain &C : gatherChains(EqClass)) {
-
-      // Split up the chain into increasingly smaller chains, until we can
-      // finally vectorize the chains.
-      //
-      // (Don't be scared by the depth of the loop nest here.  These operations
-      // are all at worst O(n lg n) in the number of instructions, and splitting
-      // chains doesn't change the number of instrs.  So the whole loop nest is
-      // O(n lg n).)
-      for (auto &C : splitChainByMayAliasInstrs(C)) {
-        for (auto &C : splitChainByContiguity(C)) {
-          ContiguousSubChains.emplace_back(C);
-        }
-      }
-    }
-  }
-
-  // Merge chains in reverse order, so that the first chain is the largest.
-  for (int I = ContiguousSubChains.size() - 1; I > 0; I--) {
-    Chain &C1 = ContiguousSubChains[I - 1];
-    Chain &C2 = ContiguousSubChains[I];
+       collectEquivalenceClasses(Begin, End))
+    Changed |= runOnEquivalenceClass(EqClassKey, EqClass);
 
-    // If the scalar types of the chains are the same, we can merge them
-    // without inserting any casts.
-    if (getLoadStoreType(C1[0].Inst)->getScalarType() ==
-        getLoadStoreType(C2[0].Inst)->getScalarType())
-      continue;
-
-    const Value *C1Ptr = getLoadStorePointerOperand(C1[0].Inst);
-    const Value *C2Ptr = getLoadStorePointerOperand(C2[0].Inst);
-    unsigned AS1 = C1Ptr->getType()->getPointerAddressSpace();
-    unsigned AS2 = C2Ptr->getType()->getPointerAddressSpace();
-    bool C1IsLoad = isa<LoadInst>(C1[0].Inst);
-    bool C2IsLoad = isa<LoadInst>(C2[0].Inst);
-
-    // If the chains are mapped to different types, have distinct underlying
-    // pointer objects, or include both loads and stores, skip.
-    if (C1IsLoad != C2IsLoad || AS1 != AS2 ||
-        ::getUnderlyingObject(C1Ptr) != ::getUnderlyingObject(C2Ptr))
-      continue;
-
-    // Compute constant offset between chain leaders; if unknown, skip.
-    std::optional<APInt> DeltaOpt = computeLeaderDelta(C1[0].Inst, C2[0].Inst);
-    if (!DeltaOpt)
-      continue;
-
-    // Check that rebasing C2 into C1's coordinate space will not overlap C1.
-    if (chainsOverlapAfterRebase(C1, C2, *DeltaOpt))
-      continue;
-
-    // Determine the common integer cast type for normalization and ensure total
-    // bitwidth matches across all elements of both chains.
-    Type *C1ElemTy = getLoadStoreType(C1[0].Inst);
-    unsigned TotalBits = DL.getTypeSizeInBits(C1ElemTy);
-    auto AllElemsMatchTotalBits = [&](const Chain &C) {
-      return llvm::all_of(C, [&](const ChainElem &E) {
-        return DL.getTypeSizeInBits(getLoadStoreType(E.Inst)) == TotalBits;
-      });
-    };
-    if (!AllElemsMatchTotalBits(C1) || !AllElemsMatchTotalBits(C2))
-      continue;
+  return Changed;
+}
 
-    // Power-of-two span ensures we can form a legal, single vector access
-    // without padding or splitting. Many targets and cost models assume POT
-    // widths, and it guarantees an integral element count for the chosen
-    // VecElemTy.
-    APInt Sz = C2.front().OffsetFromLeader +
-               DL.getTypeStoreSize(getLoadStoreType(C2.front().Inst)) -
-               C1.back().OffsetFromLeader + *DeltaOpt;
-    if (!Sz.isPowerOf2())
-      continue;
+bool Vectorizer::runOnEquivalenceClass(const EqClassKey &EqClassKey,
+                                       ArrayRef<Instruction *> EqClass) {
+  bool Changed = false;
 
-    // Rebase C2's offsets into C1's coordinate space prior to merging and
-    // merge C2 into C1 by appending all elements of C2 to C1, then erase C2
-    // from ContiguousSubChains.
-    rebaseChain(C2, *DeltaOpt);
-    C1.insert(C1.end(), C2.begin(), C2.end());
-    ContiguousSubChains.erase(ContiguousSubChains.begin() + I);
-
-    // Normalize the value operand/result type of each instruction in C1 to
-    // C1CastTy.
-    Type *C1CastTy =
-        Type::getIntNTy(C1ElemTy->getContext(), DL.getTypeSizeInBits(C1ElemTy));
-    normalizeChainToType(C1, C1CastTy);
-  }
+  LLVM_DEBUG({
+    dbgs() << "LSV: Running on equivalence class of size " << EqClass.size()
+           << " keyed on " << EqClassKey << ":\n";
+    for (Instruction *I : EqClass)
+      dbgs() << "  " << *I << "\n";
+  });
 
-  for (auto &C : ContiguousSubChains) {
-    if (C.size() <= 1)
-      continue;
-    for (auto &AlignedSubChain : splitChainByAlignment(C))
-      Changed |= vectorizeChain(AlignedSubChain);
-  }
+  std::vector<Chain> Chains = gatherChains(EqClass);
+  LLVM_DEBUG(dbgs() << "LSV: Got " << Chains.size()
+                    << " nontrivial chains.\n";);
+  for (Chain &C : Chains)
+    Changed |= runOnChain(C);
+  return Changed;
+}
 
-  // Erase all instructions scheduled for deletion in this pseudo-BB.
-  for (Instruction *I : ToErase) {
-    auto *PtrOperand = getLoadStorePointerOperand(I);
-    if (I->use_empty())
-      I->eraseFromParent();
-    RecursivelyDeleteTriviallyDeadInstructions(PtrOperand);
-  }
-  ToErase.clear();
+bool Vectorizer::runOnChain(Chain &C) {
+  LLVM_DEBUG({
+    dbgs() << "LSV: Running on chain with " << C.size() << " instructions:\n";
+    dumpChain(C);
+  });
 
+  // Split up the chain into increasingly smaller chains, until we can finally
+  // vectorize the chains.
+  //
+  // (Don't be scared by the depth of the loop nest here.  These operations are
+  // all at worst O(n lg n) in the number of instructions, and splitting chains
+  // doesn't change the number of instrs.  So the whole loop nest is O(n lg n).)
+  bool Changed = false;
+  for (auto &C : splitChainByMayAliasInstrs(C))
+    for (auto &C : splitChainByContiguity(C))
+      for (auto &C : splitChainByAlignment(C))
+        Changed |= vectorizeChain(C);
   return Changed;
 }
 
@@ -756,7 +583,7 @@ std::vector<Chain> Vectorizer::splitChainByMayAliasInstrs(Chain &C) {
         LLVM_DEBUG(
             dbgs() << "LSV: Found intervening may-alias instrs; cannot merge "
                    << *ChainIt->Inst << " into " << *ChainBegin->Inst << "\n");
-        if (!NewChain.empty()) {
+        if (NewChain.size() > 1) {
           LLVM_DEBUG({
             dbgs() << "LSV: got nontrivial chain without aliasing instrs:\n";
             dumpChain(NewChain);
@@ -768,7 +595,7 @@ std::vector<Chain> Vectorizer::splitChainByMayAliasInstrs(Chain &C) {
         NewChain = SmallVector<ChainElem, 1>({*ChainIt});
       }
     }
-    if (!NewChain.empty()) {
+    if (NewChain.size() > 1) {
       LLVM_DEBUG({
         dbgs() << "LSV: got nontrivial chain without aliasing instrs:\n";
         dumpChain(NewChain);
@@ -833,6 +660,8 @@ std::vector<Chain> Vectorizer::splitChainByContiguity(Chain &C) {
     PrevReadEnd = APIntOps::smax(PrevReadEnd, ReadEnd);
   }
 
+  // Filter out length-1 chains, these are uninteresting.
+  llvm::erase_if(Ret, [](const auto &Chain) { return Chain.size() <= 1; });
   return Ret;
 }
 
@@ -852,7 +681,7 @@ Type *Vectorizer::getChainElemTy(const Chain &C) {
   if (any_of(C, [](const ChainElem &E) {
         return getLoadStoreType(E.Inst)->getScalarType()->isPointerTy();
       })) {
-    return IntegerType::getIntNTy(
+    return Type::getIntNTy(
         F.getContext(),
         DL.getTypeSizeInBits(getLoadStoreType(C[0].Inst)->getScalarType()));
   }
@@ -1640,6 +1469,20 @@ Vectorizer::collectEquivalenceClasses(BasicBlock::iterator Begin,
                                       BasicBlock::iterator End) {
   EquivalenceClassMap Ret;
 
+  auto GetUnderlyingObject = [](const Value *Ptr) -> const Value * {
+    const Value *ObjPtr = llvm::getUnderlyingObject(Ptr);
+    if (const auto *Sel = dyn_cast<SelectInst>(ObjPtr)) {
+      // The select's themselves are distinct instructions even if they share
+      // the same condition and evaluate to consecutive pointers for true and
+      // false values of the condition. Therefore using the select's themselves
+      // for grouping instructions would put consecutive accesses into different
+      // lists and they won't be even checked for being consecutive, and won't
+      // be vectorized.
+      return Sel->getCondition();
+    }
+    return ObjPtr;
+  };
+
   for (Instruction &I : make_range(Begin, End)) {
     auto *LI = dyn_cast<LoadInst>(&I);
     auto *SI = dyn_cast<StoreInst>(&I);
@@ -1687,7 +1530,7 @@ Vectorizer::collectEquivalenceClasses(BasicBlock::iterator Begin,
         (VecTy && TTI.getLoadVectorFactor(VF, TySize, TySize / 8, VecTy) == 0))
       continue;
 
-    Ret[{::getUnderlyingObject(Ptr), AS,
+    Ret[{GetUnderlyingObject(Ptr), AS,
          DL.getTypeSizeInBits(getLoadStoreType(&I)->getScalarType()),
          /*IsLoad=*/LI != nullptr}]
         .emplace_back(&I);
@@ -1782,7 +1625,8 @@ std::vector<Chain> Vectorizer::gatherChains(ArrayRef<Instruction *> Instrs) {
   Ret.reserve(Chains.size());
   // Iterate over MRU rather than Chains so the order is deterministic.
   for (auto &E : MRU)
-    Ret.emplace_back(std::move(E.second));
+    if (E.second.size() > 1)
+      Ret.emplace_back(std::move(E.second));
   return Ret;
 }
 
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 9b727a7998392..9a94d29ba3307 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -5122,8 +5122,18 @@ InstructionCost LoopVectorizationCostModel::expectedCost(ElementCount VF) {
       InstructionCost C = getInstructionCost(&I, VF);
 
       // Check if we should override the cost.
-      if (C.isValid() && ForceTargetInstructionCost.getNumOccurrences() > 0)
-        C = InstructionCost(ForceTargetInstructionCost);
+      if (C.isValid() && ForceTargetInstructionCost.getNumOccurrences() > 0) {
+        // For interleave groups, use ForceTargetInstructionCost once for the
+        // whole group.
+        if (VF.isVector() && getWideningDecision(&I, VF) == CM_Interleave) {
+          if (getInterleavedAccessGroup(&I)->getInsertPos() == &I)
+            C = InstructionCost(ForceTargetInstructionCost);
+          else
+            C = InstructionCost(0);
+        } else {
+          C = InstructionCost(ForceTargetInstructionCost);
+        }
+      }
 
       BlockCost += C;
       LLVM_DEBUG(dbgs() << "LV: Found an estimated cost of " << C << " for VF "
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 6491a2ce6813b..422204ff3f292 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -280,7 +280,6 @@ InstructionCost VPRecipeBase::cost(ElementCount VF, VPCostContext &Ctx) {
   if (UI && Ctx.skipCostComputation(UI, VF.isVector())) {
     RecipeCost = 0;
   } else {
-    RecipeCost = computeCost(VF, Ctx);
     RecipeCost = computeCost(VF, Ctx);
     if (ForceTargetInstructionCost.getNumOccurrences() > 0 &&
         RecipeCost.isValid()) {
diff --git a/llvm/test/Analysis/ScalarEvolution/addrec-may-wrap-udiv-canonicalize.ll b/llvm/test/Analysis/ScalarEvolution/addrec-may-wrap-udiv-canonicalize.ll
index ffae5d38a7d8f..9a9a6a7d45931 100644
--- a/llvm/test/Analysis/ScalarEvolution/addrec-may-wrap-udiv-canonicalize.ll
+++ b/llvm/test/Analysis/ScalarEvolution/addrec-may-wrap-udiv-canonicalize.ll
@@ -167,3 +167,236 @@ loop:
 exit:
   ret void
 }
+
+define void @test_step2_start_outer_add_rec_step_16(i64 %n, i64 %m) {
+; CHECK-LABEL: 'test_step2_start_outer_add_rec_step_16'
+; CHECK-NEXT:  Classifying expressions for: @test_step2_start_outer_add_rec_step_16
+; CHECK-NEXT:    %outer.iv = phi i64 [ 0, %entry ], [ %outer.iv.next, %outer.latch ]
+; CHECK-NEXT:    --> {0,+,16}<%outer.header> U: [0,-15) S: [-9223372036854775808,9223372036854775793) Exits: <<Unknown>> LoopDispositions: { %outer.header: Computable, %loop: Invariant }
+; CHECK-NEXT:    %iv = phi i64 [ %outer.iv, %outer.header ], [ %iv.next, %loop ]
+; CHECK-NEXT:    --> {{\{\{}}0,+,16}<%outer.header>,+,2}<%loop> U: [0,-1) S: [-9223372036854775808,9223372036854775807) Exits: <<Unknown>> LoopDispositions: { %loop: Computable, %outer.header: Variant }
+; CHECK-NEXT:    %div.0 = udiv i64 %iv, 4
+; CHECK-NEXT:    --> ({{\{\{}}0,+,16}<%outer.header>,+,2}<%loop> /u 4) U: [0,4611686018427387904) S: [0,4611686018427387904) Exits: <<Unknown>> LoopDispositions: { %loop: Computable, %outer.header: Variant }
+; CHECK-NEXT:    %iv.1 = add i64 %iv, 1
+; CHECK-NEXT:    --> {{\{\{}}1,+,16}<%outer.header>,+,2}<%loop> U: full-set S: full-set Exits: <<Unknown>> LoopDispositions: { %loop: Computable, %outer.header: Variant }
+; CHECK-NEXT:    %div.1 = udiv i64 %iv.1, 4
+; CHECK-NEXT:    --> ({{\{\{}}1,+,16}<%outer.header>,+,2}<%loop> /u 4) U: [0,4611686018427387904) S: [0,4611686018427387904) Exits: <<Unknown>> LoopDispositions: { %loop: Computable, %outer.header: Variant }
+; CHECK-NEXT:    %iv.2 = add i64 %iv, 2
+; CHECK-NEXT:    --> {{\{\{}}2,+,16}<%outer.header>,+,2}<%loop> U: [0,-1) S: [-9223372036854775808,9223372036854775807) Exits: <<Unknown>> LoopDispositions: { %loop: Computable, %outer.header: Variant }
+; CHECK-NEXT:    %div.2 = udiv i64 %iv.2, 4
+; CHECK-NEXT:    --> ({{\{\{}}2,+,16}<%outer.header>,+,2}<%loop> /u 4) U: [0,4611686018427387904) S: [0,4611686018427387904) Exits: <<Unknown>> LoopDispositions: { %loop: Computable, %outer.header: Variant }
+; CHECK-NEXT:    %iv.3 = add i64 %iv, 3
+; CHECK-NEXT:    --> {{\{\{}}3,+,16}<%outer.header>,+,2}<%loop> U: full-set S: full-set Exits: <<Unknown>> LoopDispositions: { %loop: Computable, %outer.header: Variant }
+; CHECK-NEXT:    %div.3 = udiv i64 %iv.3, 4
+; CHECK-NEXT:    --> ({{\{\{}}3,+,16}<%outer.header>,+,2}<%loop> /u 4) U: [0,4611686018427387904) S: [0,4611686018427387904) Exits: <<Unknown>> LoopDispositions: { %loop: Computable, %outer.header: Variant }
+; CHECK-NEXT:    %iv.4 = add i64 %iv, 4
+; CHECK-NEXT:    --> {{\{\{}}4,+,16}<%outer.header>,+,2}<%loop> U: [0,-1) S: [-9223372036854775808,9223372036854775807) Exits: <<Unknown>> LoopDispositions: { %loop: Computable, %outer.header: Variant }
+; CHECK-NEXT:    %div.4 = udiv i64 %iv.4, 4
+; CHECK-NEXT:    --> ({{\{\{}}4,+,16}<%outer.header>,+,2}<%loop> /u 4) U: [0,4611686018427387904) S: [0,4611686018427387904) Exits: <<Unknown>> LoopDispositions: { %loop: Computable, %outer.header: Variant }
+; CHECK-NEXT:    %iv.5 = add i64 %iv, 5
+; CHECK-NEXT:    --> {{\{\{}}5,+,16}<%outer.header>,+,2}<%loop> U: full-set S: full-set Exits: <<Unknown>> LoopDispositions: { %loop: Computable, %outer.header: Variant }
+; CHECK-NEXT:    %div.5 = udiv i64 %iv.5, 4
+; CHECK-NEXT:    --> ({{\{\{}}5,+,16}<%outer.header>,+,2}<%loop> /u 4) U: [0,4611686018427387904) S: [0,4611686018427387904) Exits: <<Unknown>> LoopDispositions: { %loop: Computable, %outer.header: Variant }
+; CHECK-NEXT:    %iv.neg.1 = add i64 %iv, -1
+; CHECK-NEXT:    --> {{\{\{}}-1,+,16}<%outer.header>,+,2}<%loop> U: full-set S: full-set Exits: <<Unknown>> LoopDispositions: { %loop: Computable, %outer.header: Variant }
+; CHECK-NEXT:    %div.neg.1 = udiv i64 %iv.neg.1, 4
+; CHECK-NEXT:    --> ({{\{\{}}-1,+,16}<%outer.header>,+,2}<%loop> /u 4) U: [0,4611686018427387904) S: [0,4611686018427387904) Exits: <<Unknown>> LoopDispositions: { %loop: Computable, %outer.header: Variant }
+; CHECK-NEXT:    %div3.0 = udiv i64 %iv, 3
+; CHECK-NEXT:    --> ({{\{\{}}0,+,16}<%outer.header>,+,2}<%loop> /u 3) U: [0,6148914691236517205) S: [0,6148914691236517206) Exits: <<Unknown>> LoopDispositions: { %loop: Computable, %outer.header: Variant }
+; CHECK-NEXT:    %div3.1 = udiv i64 %iv.1, 3
+; CHECK-NEXT:    --> ({{\{\{}}1,+,16}<%outer.header>,+,2}<%loop> /u 3) U: [0,6148914691236517206) S: [0,6148914691236517206) Exits: <<Unknown>> LoopDispositions: { %loop: Computable, %outer.header: Variant }
+; CHECK-NEXT:    %div3.2 = udiv i64 %iv.2, 3
+; CHECK-NEXT:    --> ({{\{\{}}2,+,16}<%outer.header>,+,2}<%loop> /u 3) U: [0,6148914691236517205) S: [0,6148914691236517206) Exits: <<Unknown>> LoopDispositions: { %loop: Computable, %outer.header: Variant }
+; CHECK-NEXT:    %div3.4 = udiv i64 %iv.4, 3
+; CHECK-NEXT:    --> ({{\{\{}}4,+,16}<%outer.header>,+,2}<%loop> /u 3) U: [0,6148914691236517205) S: [0,6148914691236517206) Exits: <<Unknown>> LoopDispositions: { %loop: Computable, %outer.header: Variant }
+; CHECK-NEXT:    %div3.5 = udiv i64 %iv.5, 3
+; CHECK-NEXT:    --> ({{\{\{}}5,+,16}<%outer.header>,+,2}<%loop> /u 3) U: [0,6148914691236517206) S: [0,6148914691236517206) Exits: <<Unknown>> LoopDispositions: { %loop: Computable, %outer.header: Variant }
+; CHECK-NEXT:    %iv.next = add i64 %iv, 2
+; CHECK-NEXT:    --> {{\{\{}}2,+,16}<%outer.header>,+,2}<%loop> U: [0,-1) S: [-9223372036854775808,9223372036854775807) Exits: <<Unknown>> LoopDispositions: { %loop: Computable, %outer.header: Variant }
+; CHECK-NEXT:    %outer.iv.next = add i64 %outer.iv, 16
+; CHECK-NEXT:    --> {16,+,16}<%outer.header> U: [0,-15) S: [-9223372036854775808,9223372036854775793) Exits: <<Unknown>> LoopDispositions: { %outer.header: Computable, %loop: Invariant }
+; CHECK-NEXT:  Determining loop execution counts for: @test_step2_start_outer_add_rec_step_16
+; CHECK-NEXT:  Loop %loop: Unpredictable backedge-taken count.
+; CHECK-NEXT:  Loop %loop: Unpredictable constant max backedge-taken count.
+; CHECK-NEXT:  Loop %loop: Unpredictable symbolic max backedge-taken count.
+; CHECK-NEXT:  Loop %outer.header: Unpredictable backedge-taken count.
+; CHECK-NEXT:  Loop %outer.header: Unpredictable constant max backedge-taken count.
+; CHECK-NEXT:  Loop %outer.header: Unpredictable symbolic max backedge-taken count.
+; CHECK-NEXT:  Loop %outer.header: Predicated backedge-taken count is (%m /u 16)
+; CHECK-NEXT:   Predicates:
+; CHECK-NEXT:      Equal predicate: (zext i4 (trunc i64 %m to i4) to i64) == 0
+; CHECK-NEXT:  Loop %outer.header: Predicated constant max backedge-taken count is i64 1152921504606846975
+; CHECK-NEXT:   Predicates:
+; CHECK-NEXT:      Equal predicate: (zext i4 (trunc i64 %m to i4) to i64) == 0
+; CHECK-NEXT:  Loop %outer.header: Predicated symbolic max backedge-taken count is (%m /u 16)
+; CHECK-NEXT:   Predicates:
+; CHECK-NEXT:      Equal predicate: (zext i4 (trunc i64 %m to i4) to i64) == 0
+;
+entry:
+  br label %outer.header
+
+outer.header:
+  %outer.iv = phi i64 [ 0, %entry ], [ %outer.iv.next, %outer.latch ]
+  br label %loop
+
+loop:
+  %iv = phi i64 [ %outer.iv, %outer.header ], [ %iv.next, %loop ]
+  %div.0 = udiv i64 %iv, 4
+  call void @use(i64 %div.0)
+  %iv.1 = add i64 %iv, 1
+  %div.1 = udiv i64 %iv.1, 4
+  call void @use(i64 %div.1)
+  %iv.2 = add i64 %iv, 2
+  %div.2 = udiv i64 %iv.2, 4
+  call void @use(i64 %div.2)
+  %iv.3 = add i64 %iv, 3
+  %div.3 = udiv i64 %iv.3, 4
+  call void @use(i64 %div.3)
+  %iv.4 = add i64 %iv, 4
+  %div.4 = udiv i64 %iv.4, 4
+  call void @use(i64 %div.4)
+  %iv.5 = add i64 %iv, 5
+  %div.5 = udiv i64 %iv.5, 4
+  call void @use(i64 %div.5)
+  %iv.neg.1 = add i64 %iv, -1
+  %div.neg.1 = udiv i64 %iv.neg.1, 4
+  call void @use(i64 %div.neg.1)
+  %div3.0 = udiv i64 %iv, 3
+  call void @use(i64 %div3.0)
+  %div3.1 = udiv i64 %iv.1,3
+  call void @use(i64 %div3.1)
+  %div3.2 = udiv i64 %iv.2, 3
+  call void @use(i64 %div3.2)
+  %div3.4 = udiv i64 %iv.4, 3
+  call void @use(i64 %div3.4)
+  %div3.5 = udiv i64 %iv.5, 3
+  call void @use(i64 %div3.5)
+  %iv.next = add i64 %iv, 2
+  %cond = icmp slt i64 %iv, %n
+  br i1 %cond, label %loop, label %outer.latch
+
+outer.latch:
+  %outer.iv.next = add i64 %outer.iv, 16
+  %outer.ec = icmp eq i64 %outer.iv, %m
+  br i1 %outer.ec, label %exit, label %outer.header
+
+exit:
+  ret void
+}
+
+define void @test_step2_div4_start_outer_add_rec_step_2(i64 %n, i64 %m) {
+; CHECK-LABEL: 'test_step2_div4_start_outer_add_rec_step_2'
+; CHECK-NEXT:  Classifying expressions for: @test_step2_div4_start_outer_add_rec_step_2
+; CHECK-NEXT:    %outer.iv = phi i64 [ 0, %entry ], [ %outer.iv.next, %outer.latch ]
+; CHECK-NEXT:    --> {0,+,2}<%outer.header> U: [0,-1) S: [-9223372036854775808,9223372036854775807) Exits: <<Unknown>> LoopDispositions: { %outer.header: Computable, %loop: Invariant }
+; CHECK-NEXT:    %iv = phi i64 [ %outer.iv, %outer.header ], [ %iv.next, %loop ]
+; CHECK-NEXT:    --> {{\{\{}}0,+,2}<%outer.header>,+,2}<%loop> U: [0,-1) S: [-9223372036854775808,9223372036854775807) Exits: <<Unknown>> LoopDispositions: { %loop: Computable, %outer.header: Variant }
+; CHECK-NEXT:    %div.0 = udiv i64 %iv, 4
+; CHECK-NEXT:    --> ({{\{\{}}0,+,2}<%outer.header>,+,2}<%loop> /u 4) U: [0,4611686018427387904) S: [0,4611686018427387904) Exits: <<Unknown>> LoopDispositions: { %loop: Computable, %outer.header: Variant }
+; CHECK-NEXT:    %iv.1 = add i64 %iv, 1
+; CHECK-NEXT:    --> {{\{\{}}1,+,2}<%outer.header>,+,2}<%loop> U: full-set S: full-set Exits: <<Unknown>> LoopDispositions: { %loop: Computable, %outer.header: Variant }
+; CHECK-NEXT:    %div.1 = udiv i64 %iv.1, 4
+; CHECK-NEXT:    --> ({{\{\{}}1,+,2}<%outer.header>,+,2}<%loop> /u 4) U: [0,4611686018427387904) S: [0,4611686018427387904) Exits: <<Unknown>> LoopDispositions: { %loop: Computable, %outer.header: Variant }
+; CHECK-NEXT:    %iv.2 = add i64 %iv, 2
+; CHECK-NEXT:    --> {{\{\{}}2,+,2}<%outer.header>,+,2}<%loop> U: [0,-1) S: [-9223372036854775808,9223372036854775807) Exits: <<Unknown>> LoopDispositions: { %loop: Computable, %outer.header: Variant }
+; CHECK-NEXT:    %div.2 = udiv i64 %iv.2, 4
+; CHECK-NEXT:    --> ({{\{\{}}2,+,2}<%outer.header>,+,2}<%loop> /u 4) U: [0,4611686018427387904) S: [0,4611686018427387904) Exits: <<Unknown>> LoopDispositions: { %loop: Computable, %outer.header: Variant }
+; CHECK-NEXT:    %iv.3 = add i64 %iv, 3
+; CHECK-NEXT:    --> {{\{\{}}3,+,2}<%outer.header>,+,2}<%loop> U: full-set S: full-set Exits: <<Unknown>> LoopDispositions: { %loop: Computable, %outer.header: Variant }
+; CHECK-NEXT:    %div.3 = udiv i64 %iv.3, 4
+; CHECK-NEXT:    --> ({{\{\{}}3,+,2}<%outer.header>,+,2}<%loop> /u 4) U: [0,4611686018427387904) S: [0,4611686018427387904) Exits: <<Unknown>> LoopDispositions: { %loop: Computable, %outer.header: Variant }
+; CHECK-NEXT:    %iv.4 = add i64 %iv, 4
+; CHECK-NEXT:    --> {{\{\{}}4,+,2}<%outer.header>,+,2}<%loop> U: [0,-1) S: [-9223372036854775808,9223372036854775807) Exits: <<Unknown>> LoopDispositions: { %loop: Computable, %outer.header: Variant }
+; CHECK-NEXT:    %div.4 = udiv i64 %iv.4, 4
+; CHECK-NEXT:    --> ({{\{\{}}4,+,2}<%outer.header>,+,2}<%loop> /u 4) U: [0,4611686018427387904) S: [0,4611686018427387904) Exits: <<Unknown>> LoopDispositions: { %loop: Computable, %outer.header: Variant }
+; CHECK-NEXT:    %iv.5 = add i64 %iv, 5
+; CHECK-NEXT:    --> {{\{\{}}5,+,2}<%outer.header>,+,2}<%loop> U: full-set S: full-set Exits: <<Unknown>> LoopDispositions: { %loop: Computable, %outer.header: Variant }
+; CHECK-NEXT:    %div.5 = udiv i64 %iv.5, 4
+; CHECK-NEXT:    --> ({{\{\{}}5,+,2}<%outer.header>,+,2}<%loop> /u 4) U: [0,4611686018427387904) S: [0,4611686018427387904) Exits: <<Unknown>> LoopDispositions: { %loop: Computable, %outer.header: Variant }
+; CHECK-NEXT:    %iv.neg.1 = add i64 %iv, -1
+; CHECK-NEXT:    --> {{\{\{}}-1,+,2}<%outer.header>,+,2}<%loop> U: full-set S: full-set Exits: <<Unknown>> LoopDispositions: { %loop: Computable, %outer.header: Variant }
+; CHECK-NEXT:    %div.neg.1 = udiv i64 %iv.neg.1, 4
+; CHECK-NEXT:    --> ({{\{\{}}-1,+,2}<%outer.header>,+,2}<%loop> /u 4) U: [0,4611686018427387904) S: [0,4611686018427387904) Exits: <<Unknown>> LoopDispositions: { %loop: Computable, %outer.header: Variant }
+; CHECK-NEXT:    %div3.0 = udiv i64 %iv, 3
+; CHECK-NEXT:    --> ({{\{\{}}0,+,2}<%outer.header>,+,2}<%loop> /u 3) U: [0,6148914691236517205) S: [0,6148914691236517206) Exits: <<Unknown>> LoopDispositions: { %loop: Computable, %outer.header: Variant }
+; CHECK-NEXT:    %div3.1 = udiv i64 %iv.1, 3
+; CHECK-NEXT:    --> ({{\{\{}}1,+,2}<%outer.header>,+,2}<%loop> /u 3) U: [0,6148914691236517206) S: [0,6148914691236517206) Exits: <<Unknown>> LoopDispositions: { %loop: Computable, %outer.header: Variant }
+; CHECK-NEXT:    %div3.2 = udiv i64 %iv.2, 3
+; CHECK-NEXT:    --> ({{\{\{}}2,+,2}<%outer.header>,+,2}<%loop> /u 3) U: [0,6148914691236517205) S: [0,6148914691236517206) Exits: <<Unknown>> LoopDispositions: { %loop: Computable, %outer.header: Variant }
+; CHECK-NEXT:    %div3.4 = udiv i64 %iv.4, 3
+; CHECK-NEXT:    --> ({{\{\{}}4,+,2}<%outer.header>,+,2}<%loop> /u 3) U: [0,6148914691236517205) S: [0,6148914691236517206) Exits: <<Unknown>> LoopDispositions: { %loop: Computable, %outer.header: Variant }
+; CHECK-NEXT:    %div3.5 = udiv i64 %iv.5, 3
+; CHECK-NEXT:    --> ({{\{\{}}5,+,2}<%outer.header>,+,2}<%loop> /u 3) U: [0,6148914691236517206) S: [0,6148914691236517206) Exits: <<Unknown>> LoopDispositions: { %loop: Computable, %outer.header: Variant }
+; CHECK-NEXT:    %iv.next = add i64 %iv, 2
+; CHECK-NEXT:    --> {{\{\{}}2,+,2}<%outer.header>,+,2}<%loop> U: [0,-1) S: [-9223372036854775808,9223372036854775807) Exits: <<Unknown>> LoopDispositions: { %loop: Computable, %outer.header: Variant }
+; CHECK-NEXT:    %outer.iv.next = add i64 %outer.iv, 2
+; CHECK-NEXT:    --> {2,+,2}<%outer.header> U: [0,-1) S: [-9223372036854775808,9223372036854775807) Exits: <<Unknown>> LoopDispositions: { %outer.header: Computable, %loop: Invariant }
+; CHECK-NEXT:  Determining loop execution counts for: @test_step2_div4_start_outer_add_rec_step_2
+; CHECK-NEXT:  Loop %loop: Unpredictable backedge-taken count.
+; CHECK-NEXT:  Loop %loop: Unpredictable constant max backedge-taken count.
+; CHECK-NEXT:  Loop %loop: Unpredictable symbolic max backedge-taken count.
+; CHECK-NEXT:  Loop %outer.header: Unpredictable backedge-taken count.
+; CHECK-NEXT:  Loop %outer.header: Unpredictable constant max backedge-taken count.
+; CHECK-NEXT:  Loop %outer.header: Unpredictable symbolic max backedge-taken count.
+; CHECK-NEXT:  Loop %outer.header: Predicated backedge-taken count is (%m /u 2)
+; CHECK-NEXT:   Predicates:
+; CHECK-NEXT:      Equal predicate: (zext i1 (trunc i64 %m to i1) to i64) == 0
+; CHECK-NEXT:  Loop %outer.header: Predicated constant max backedge-taken count is i64 9223372036854775807
+; CHECK-NEXT:   Predicates:
+; CHECK-NEXT:      Equal predicate: (zext i1 (trunc i64 %m to i1) to i64) == 0
+; CHECK-NEXT:  Loop %outer.header: Predicated symbolic max backedge-taken count is (%m /u 2)
+; CHECK-NEXT:   Predicates:
+; CHECK-NEXT:      Equal predicate: (zext i1 (trunc i64 %m to i1) to i64) == 0
+;
+entry:
+  br label %outer.header
+
+outer.header:
+  %outer.iv = phi i64 [ 0, %entry ], [ %outer.iv.next, %outer.latch ]
+  br label %loop
+
+loop:
+  %iv = phi i64 [ %outer.iv, %outer.header ], [ %iv.next, %loop ]
+  %div.0 = udiv i64 %iv, 4
+  call void @use(i64 %div.0)
+  %iv.1 = add i64 %iv, 1
+  %div.1 = udiv i64 %iv.1, 4
+  call void @use(i64 %div.1)
+  %iv.2 = add i64 %iv, 2
+  %div.2 = udiv i64 %iv.2, 4
+  call void @use(i64 %div.2)
+  %iv.3 = add i64 %iv, 3
+  %div.3 = udiv i64 %iv.3, 4
+  call void @use(i64 %div.3)
+  %iv.4 = add i64 %iv, 4
+  %div.4 = udiv i64 %iv.4, 4
+  call void @use(i64 %div.4)
+  %iv.5 = add i64 %iv, 5
+  %div.5 = udiv i64 %iv.5, 4
+  call void @use(i64 %div.5)
+  %iv.neg.1 = add i64 %iv, -1
+  %div.neg.1 = udiv i64 %iv.neg.1, 4
+  call void @use(i64 %div.neg.1)
+  %div3.0 = udiv i64 %iv, 3
+  call void @use(i64 %div3.0)
+  %div3.1 = udiv i64 %iv.1,3
+  call void @use(i64 %div3.1)
+  %div3.2 = udiv i64 %iv.2, 3
+  call void @use(i64 %div3.2)
+  %div3.4 = udiv i64 %iv.4, 3
+  call void @use(i64 %div3.4)
+  %div3.5 = udiv i64 %iv.5, 3
+  call void @use(i64 %div3.5)
+  call void @use(i64 %div.neg.1)
+  %iv.next = add i64 %iv, 2
+  %cond = icmp slt i64 %iv, %n
+  br i1 %cond, label %loop, label %outer.latch
+
+outer.latch:
+  %outer.iv.next = add i64 %outer.iv, 2
+  %outer.ec = icmp eq i64 %outer.iv, %m
+  br i1 %outer.ec, label %exit, label %outer.header
+
+exit:
+  ret void
+}
diff --git a/llvm/test/CodeGen/AArch64/sve-int-mulh-pred.ll b/llvm/test/CodeGen/AArch64/sve-int-mulh-pred.ll
index 32760caa524ec..146720febf486 100644
--- a/llvm/test/CodeGen/AArch64/sve-int-mulh-pred.ll
+++ b/llvm/test/CodeGen/AArch64/sve-int-mulh-pred.ll
@@ -1,11 +1,11 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=aarch64-linux-gnu < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s | FileCheck %s
 
 ;
 ; SMULH
 ;
 
-define <vscale x 16 x i8> @smulh_i8(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b) #0 {
+define <vscale x 16 x i8> @smulh_i8(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b) {
 ; CHECK-LABEL: smulh_i8:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptrue p0.b
@@ -19,7 +19,7 @@ define <vscale x 16 x i8> @smulh_i8(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b
   ret <vscale x 16 x i8> %tr
 }
 
-define <vscale x 8 x i16> @smulh_i16(<vscale x 8 x i16> %a, <vscale x 8 x i16> %b) #0 {
+define <vscale x 8 x i16> @smulh_i16(<vscale x 8 x i16> %a, <vscale x 8 x i16> %b) {
 ; CHECK-LABEL: smulh_i16:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptrue p0.h
@@ -33,7 +33,7 @@ define <vscale x 8 x i16> @smulh_i16(<vscale x 8 x i16> %a, <vscale x 8 x i16> %
   ret <vscale x 8 x i16> %tr
 }
 
-define <vscale x 4 x i32> @smulh_i32(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b) #0 {
+define <vscale x 4 x i32> @smulh_i32(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
 ; CHECK-LABEL: smulh_i32:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptrue p0.s
@@ -47,7 +47,7 @@ define <vscale x 4 x i32> @smulh_i32(<vscale x 4 x i32> %a, <vscale x 4 x i32> %
   ret <vscale x 4 x i32> %tr
 }
 
-define <vscale x 2 x i64> @smulh_i64(<vscale x 2 x i64> %a, <vscale x 2 x i64> %b) #0 {
+define <vscale x 2 x i64> @smulh_i64(<vscale x 2 x i64> %a, <vscale x 2 x i64> %b) {
 ; CHECK-LABEL: smulh_i64:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptrue p0.d
@@ -65,7 +65,7 @@ define <vscale x 2 x i64> @smulh_i64(<vscale x 2 x i64> %a, <vscale x 2 x i64> %
 ; UMULH
 ;
 
-define <vscale x 16 x i8> @umulh_i8(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b) #0 {
+define <vscale x 16 x i8> @umulh_i8(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b) {
 ; CHECK-LABEL: umulh_i8:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptrue p0.b
@@ -79,7 +79,7 @@ define <vscale x 16 x i8> @umulh_i8(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b
   ret <vscale x 16 x i8> %tr
 }
 
-define <vscale x 8 x i16> @umulh_i16(<vscale x 8 x i16> %a, <vscale x 8 x i16> %b) #0 {
+define <vscale x 8 x i16> @umulh_i16(<vscale x 8 x i16> %a, <vscale x 8 x i16> %b) {
 ; CHECK-LABEL: umulh_i16:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptrue p0.h
@@ -93,7 +93,7 @@ define <vscale x 8 x i16> @umulh_i16(<vscale x 8 x i16> %a, <vscale x 8 x i16> %
   ret <vscale x 8 x i16> %tr
 }
 
-define <vscale x 4 x i32> @umulh_i32(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b) #0 {
+define <vscale x 4 x i32> @umulh_i32(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
 ; CHECK-LABEL: umulh_i32:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptrue p0.s
@@ -107,7 +107,7 @@ define <vscale x 4 x i32> @umulh_i32(<vscale x 4 x i32> %a, <vscale x 4 x i32> %
   ret <vscale x 4 x i32> %tr
 }
 
-define <vscale x 2 x i64> @umulh_i64(<vscale x 2 x i64> %a, <vscale x 2 x i64> %b) #0 {
+define <vscale x 2 x i64> @umulh_i64(<vscale x 2 x i64> %a, <vscale x 2 x i64> %b) {
 ; CHECK-LABEL: umulh_i64:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptrue p0.d
@@ -121,4 +121,262 @@ define <vscale x 2 x i64> @umulh_i64(<vscale x 2 x i64> %a, <vscale x 2 x i64> %
   ret <vscale x 2 x i64> %tr
 }
 
-attributes #0 = { "target-features"="+sve" }
+
+; Fixed-length 128bits
+
+define <16 x i8> @smulh_v16i8(<16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: smulh_v16i8:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    smull2 v2.8h, v0.16b, v1.16b
+; CHECK-NEXT:    smull v0.8h, v0.8b, v1.8b
+; CHECK-NEXT:    uzp2 v0.16b, v0.16b, v2.16b
+; CHECK-NEXT:    ret
+  %1 = sext <16 x i8> %a to <16 x i16>
+  %2 = sext <16 x i8> %b to <16 x i16>
+  %mul = mul <16 x i16> %1, %2
+  %shr = lshr <16 x i16> %mul, splat(i16 8)
+  %tr = trunc <16 x i16> %shr to <16 x i8>
+  ret <16 x i8> %tr
+}
+
+define <8 x i16> @smulh_v8i16(<8 x i16> %a, <8 x i16> %b) {
+; CHECK-LABEL: smulh_v8i16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    smull2 v2.4s, v0.8h, v1.8h
+; CHECK-NEXT:    smull v0.4s, v0.4h, v1.4h
+; CHECK-NEXT:    uzp2 v0.8h, v0.8h, v2.8h
+; CHECK-NEXT:    ret
+  %1 = sext <8 x i16> %a to <8 x i32>
+  %2 = sext <8 x i16> %b to <8 x i32>
+  %mul = mul <8 x i32> %1, %2
+  %shr = lshr <8 x i32> %mul, splat(i32 16)
+  %tr = trunc <8 x i32> %shr to <8 x i16>
+  ret <8 x i16> %tr
+}
+
+define <4 x i32> @smulh_v4i32(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: smulh_v4i32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    smull2 v2.2d, v0.4s, v1.4s
+; CHECK-NEXT:    smull v0.2d, v0.2s, v1.2s
+; CHECK-NEXT:    uzp2 v0.4s, v0.4s, v2.4s
+; CHECK-NEXT:    ret
+  %1 = sext <4 x i32> %a to <4 x i64>
+  %2 = sext <4 x i32> %b to <4 x i64>
+  %mul = mul <4 x i64> %1, %2
+  %shr = lshr <4 x i64> %mul, splat(i64 32)
+  %tr = trunc <4 x i64> %shr to <4 x i32>
+  ret <4 x i32> %tr
+}
+
+define <2 x i64> @smulh_v2i64(<2 x i64> %a, <2 x i64> %b) {
+; CHECK-LABEL: smulh_v2i64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov x8, v0.d[1]
+; CHECK-NEXT:    mov x9, v1.d[1]
+; CHECK-NEXT:    fmov x10, d0
+; CHECK-NEXT:    fmov x11, d1
+; CHECK-NEXT:    smulh x10, x10, x11
+; CHECK-NEXT:    smulh x8, x8, x9
+; CHECK-NEXT:    fmov d0, x10
+; CHECK-NEXT:    fmov d1, x8
+; CHECK-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-NEXT:    ret
+  %1 = sext <2 x i64> %a to <2 x i128>
+  %2 = sext <2 x i64> %b to <2 x i128>
+  %mul = mul <2 x i128> %1, %2
+  %shr = lshr <2 x i128> %mul, splat(i128 64)
+  %tr = trunc <2 x i128> %shr to <2 x i64>
+  ret <2 x i64> %tr
+}
+
+define <16 x i8> @umulh_v16i8(<16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: umulh_v16i8:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    umull2 v2.8h, v0.16b, v1.16b
+; CHECK-NEXT:    umull v0.8h, v0.8b, v1.8b
+; CHECK-NEXT:    uzp2 v0.16b, v0.16b, v2.16b
+; CHECK-NEXT:    ret
+  %1 = zext <16 x i8> %a to <16 x i16>
+  %2 = zext <16 x i8> %b to <16 x i16>
+  %mul = mul <16 x i16> %1, %2
+  %shr = lshr <16 x i16> %mul, splat(i16 8)
+  %tr = trunc <16 x i16> %shr to <16 x i8>
+  ret <16 x i8> %tr
+}
+
+define <8 x i16> @umulh_v8i16(<8 x i16> %a, <8 x i16> %b) {
+; CHECK-LABEL: umulh_v8i16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    umull2 v2.4s, v0.8h, v1.8h
+; CHECK-NEXT:    umull v0.4s, v0.4h, v1.4h
+; CHECK-NEXT:    uzp2 v0.8h, v0.8h, v2.8h
+; CHECK-NEXT:    ret
+  %1 = zext <8 x i16> %a to <8 x i32>
+  %2 = zext <8 x i16> %b to <8 x i32>
+  %mul = mul <8 x i32> %1, %2
+  %shr = lshr <8 x i32> %mul, splat(i32 16)
+  %tr = trunc <8 x i32> %shr to <8 x i16>
+  ret <8 x i16> %tr
+}
+
+define <4 x i32> @umulh_v4i32(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: umulh_v4i32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    umull2 v2.2d, v0.4s, v1.4s
+; CHECK-NEXT:    umull v0.2d, v0.2s, v1.2s
+; CHECK-NEXT:    uzp2 v0.4s, v0.4s, v2.4s
+; CHECK-NEXT:    ret
+  %1 = zext <4 x i32> %a to <4 x i64>
+  %2 = zext <4 x i32> %b to <4 x i64>
+  %mul = mul <4 x i64> %1, %2
+  %shr = lshr <4 x i64> %mul, splat(i64 32)
+  %tr = trunc <4 x i64> %shr to <4 x i32>
+  ret <4 x i32> %tr
+}
+
+define <2 x i64> @umulh_v2i64(<2 x i64> %a, <2 x i64> %b) {
+; CHECK-LABEL: umulh_v2i64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov x8, v0.d[1]
+; CHECK-NEXT:    mov x9, v1.d[1]
+; CHECK-NEXT:    fmov x10, d0
+; CHECK-NEXT:    fmov x11, d1
+; CHECK-NEXT:    umulh x10, x10, x11
+; CHECK-NEXT:    umulh x8, x8, x9
+; CHECK-NEXT:    fmov d0, x10
+; CHECK-NEXT:    fmov d1, x8
+; CHECK-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-NEXT:    ret
+  %1 = zext <2 x i64> %a to <2 x i128>
+  %2 = zext <2 x i64> %b to <2 x i128>
+  %mul = mul <2 x i128> %1, %2
+  %shr = lshr <2 x i128> %mul, splat(i128 64)
+  %tr = trunc <2 x i128> %shr to <2 x i64>
+  ret <2 x i64> %tr
+}
+
+
+
+; Fixed-length 64bits
+
+define <8 x i8> @smulh_v8i8(<8 x i8> %a, <8 x i8> %b) {
+; CHECK-LABEL: smulh_v8i8:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    smull v0.8h, v0.8b, v1.8b
+; CHECK-NEXT:    shrn v0.8b, v0.8h, #8
+; CHECK-NEXT:    ret
+  %1 = sext <8 x i8> %a to <8 x i16>
+  %2 = sext <8 x i8> %b to <8 x i16>
+  %mul = mul <8 x i16> %1, %2
+  %shr = lshr <8 x i16> %mul, splat(i16 8)
+  %tr = trunc <8 x i16> %shr to <8 x i8>
+  ret <8 x i8> %tr
+}
+
+define <4 x i16> @smulh_v4i16(<4 x i16> %a, <4 x i16> %b) {
+; CHECK-LABEL: smulh_v4i16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    smull v0.4s, v0.4h, v1.4h
+; CHECK-NEXT:    shrn v0.4h, v0.4s, #16
+; CHECK-NEXT:    ret
+  %1 = sext <4 x i16> %a to <4 x i32>
+  %2 = sext <4 x i16> %b to <4 x i32>
+  %mul = mul <4 x i32> %1, %2
+  %shr = lshr <4 x i32> %mul, splat(i32 16)
+  %tr = trunc <4 x i32> %shr to <4 x i16>
+  ret <4 x i16> %tr
+}
+
+define <2 x i32> @smulh_v2i32(<2 x i32> %a, <2 x i32> %b) {
+; CHECK-LABEL: smulh_v2i32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    smull v0.2d, v0.2s, v1.2s
+; CHECK-NEXT:    shrn v0.2s, v0.2d, #32
+; CHECK-NEXT:    ret
+  %1 = sext <2 x i32> %a to <2 x i64>
+  %2 = sext <2 x i32> %b to <2 x i64>
+  %mul = mul <2 x i64> %1, %2
+  %shr = lshr <2 x i64> %mul, splat(i64 32)
+  %tr = trunc <2 x i64> %shr to <2 x i32>
+  ret <2 x i32> %tr
+}
+
+define <1 x i64> @smulh_v1i64(<1 x i64> %a, <1 x i64> %b) {
+; CHECK-LABEL: smulh_v1i64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $d1 killed $d1 def $q1
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT:    fmov x8, d0
+; CHECK-NEXT:    fmov x9, d1
+; CHECK-NEXT:    smulh x8, x8, x9
+; CHECK-NEXT:    fmov d0, x8
+; CHECK-NEXT:    ret
+  %1 = sext <1 x i64> %a to <1 x i128>
+  %2 = sext <1 x i64> %b to <1 x i128>
+  %mul = mul <1 x i128> %1, %2
+  %shr = lshr <1 x i128> %mul, splat(i128 64)
+  %tr = trunc <1 x i128> %shr to <1 x i64>
+  ret <1 x i64> %tr
+}
+
+define <8 x i8> @umulh_v8i8(<8 x i8> %a, <8 x i8> %b) {
+; CHECK-LABEL: umulh_v8i8:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    umull v0.8h, v0.8b, v1.8b
+; CHECK-NEXT:    shrn v0.8b, v0.8h, #8
+; CHECK-NEXT:    ret
+  %1 = zext <8 x i8> %a to <8 x i16>
+  %2 = zext <8 x i8> %b to <8 x i16>
+  %mul = mul <8 x i16> %1, %2
+  %shr = lshr <8 x i16> %mul, splat(i16 8)
+  %tr = trunc <8 x i16> %shr to <8 x i8>
+  ret <8 x i8> %tr
+}
+
+define <4 x i16> @umulh_v4i16(<4 x i16> %a, <4 x i16> %b) {
+; CHECK-LABEL: umulh_v4i16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    umull v0.4s, v0.4h, v1.4h
+; CHECK-NEXT:    shrn v0.4h, v0.4s, #16
+; CHECK-NEXT:    ret
+  %1 = zext <4 x i16> %a to <4 x i32>
+  %2 = zext <4 x i16> %b to <4 x i32>
+  %mul = mul <4 x i32> %1, %2
+  %shr = lshr <4 x i32> %mul, splat(i32 16)
+  %tr = trunc <4 x i32> %shr to <4 x i16>
+  ret <4 x i16> %tr
+}
+
+define <2 x i32> @umulh_v2i32(<2 x i32> %a, <2 x i32> %b) {
+; CHECK-LABEL: umulh_v2i32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    umull v0.2d, v0.2s, v1.2s
+; CHECK-NEXT:    shrn v0.2s, v0.2d, #32
+; CHECK-NEXT:    ret
+  %1 = zext <2 x i32> %a to <2 x i64>
+  %2 = zext <2 x i32> %b to <2 x i64>
+  %mul = mul <2 x i64> %1, %2
+  %shr = lshr <2 x i64> %mul, splat(i64 32)
+  %tr = trunc <2 x i64> %shr to <2 x i32>
+  ret <2 x i32> %tr
+}
+
+define <1 x i64> @umulh_v1i64(<1 x i64> %a, <1 x i64> %b) {
+; CHECK-LABEL: umulh_v1i64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $d1 killed $d1 def $q1
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT:    fmov x8, d0
+; CHECK-NEXT:    fmov x9, d1
+; CHECK-NEXT:    umulh x8, x8, x9
+; CHECK-NEXT:    fmov d0, x8
+; CHECK-NEXT:    ret
+  %1 = zext <1 x i64> %a to <1 x i128>
+  %2 = zext <1 x i64> %b to <1 x i128>
+  %mul = mul <1 x i128> %1, %2
+  %shr = lshr <1 x i128> %mul, splat(i128 64)
+  %tr = trunc <1 x i128> %shr to <1 x i64>
+  ret <1 x i64> %tr
+}
+
diff --git a/llvm/test/CodeGen/AArch64/sve2-int-mulh.ll b/llvm/test/CodeGen/AArch64/sve2-int-mulh.ll
index bcf76d5b13d62..d7534712b53a0 100644
--- a/llvm/test/CodeGen/AArch64/sve2-int-mulh.ll
+++ b/llvm/test/CodeGen/AArch64/sve2-int-mulh.ll
@@ -1,11 +1,11 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=aarch64-linux-gnu < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve2 < %s | FileCheck %s
 
 ;
 ; SMULH
 ;
 
-define <vscale x 16 x i8> @smulh_i8(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b) #0 {
+define <vscale x 16 x i8> @smulh_i8(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b) {
 ; CHECK-LABEL: smulh_i8:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    smulh z0.b, z0.b, z1.b
@@ -18,7 +18,7 @@ define <vscale x 16 x i8> @smulh_i8(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b
   ret <vscale x 16 x i8> %tr
 }
 
-define <vscale x 8 x i16> @smulh_i16(<vscale x 8 x i16> %a, <vscale x 8 x i16> %b) #0 {
+define <vscale x 8 x i16> @smulh_i16(<vscale x 8 x i16> %a, <vscale x 8 x i16> %b) {
 ; CHECK-LABEL: smulh_i16:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    smulh z0.h, z0.h, z1.h
@@ -31,7 +31,7 @@ define <vscale x 8 x i16> @smulh_i16(<vscale x 8 x i16> %a, <vscale x 8 x i16> %
   ret <vscale x 8 x i16> %tr
 }
 
-define <vscale x 4 x i32> @smulh_i32(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b) #0 {
+define <vscale x 4 x i32> @smulh_i32(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
 ; CHECK-LABEL: smulh_i32:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    smulh z0.s, z0.s, z1.s
@@ -44,7 +44,7 @@ define <vscale x 4 x i32> @smulh_i32(<vscale x 4 x i32> %a, <vscale x 4 x i32> %
   ret <vscale x 4 x i32> %tr
 }
 
-define <vscale x 2 x i64> @smulh_i64(<vscale x 2 x i64> %a, <vscale x 2 x i64> %b) #0 {
+define <vscale x 2 x i64> @smulh_i64(<vscale x 2 x i64> %a, <vscale x 2 x i64> %b) {
 ; CHECK-LABEL: smulh_i64:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    smulh z0.d, z0.d, z1.d
@@ -61,7 +61,7 @@ define <vscale x 2 x i64> @smulh_i64(<vscale x 2 x i64> %a, <vscale x 2 x i64> %
 ; UMULH
 ;
 
-define <vscale x 16 x i8> @umulh_i8(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b) #0 {
+define <vscale x 16 x i8> @umulh_i8(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b) {
 ; CHECK-LABEL: umulh_i8:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    umulh z0.b, z0.b, z1.b
@@ -74,7 +74,7 @@ define <vscale x 16 x i8> @umulh_i8(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b
   ret <vscale x 16 x i8> %tr
 }
 
-define <vscale x 8 x i16> @umulh_i16(<vscale x 8 x i16> %a, <vscale x 8 x i16> %b) #0 {
+define <vscale x 8 x i16> @umulh_i16(<vscale x 8 x i16> %a, <vscale x 8 x i16> %b) {
 ; CHECK-LABEL: umulh_i16:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    umulh z0.h, z0.h, z1.h
@@ -87,7 +87,7 @@ define <vscale x 8 x i16> @umulh_i16(<vscale x 8 x i16> %a, <vscale x 8 x i16> %
   ret <vscale x 8 x i16> %tr
 }
 
-define <vscale x 4 x i32> @umulh_i32(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b) #0 {
+define <vscale x 4 x i32> @umulh_i32(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
 ; CHECK-LABEL: umulh_i32:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    umulh z0.s, z0.s, z1.s
@@ -100,7 +100,7 @@ define <vscale x 4 x i32> @umulh_i32(<vscale x 4 x i32> %a, <vscale x 4 x i32> %
   ret <vscale x 4 x i32> %tr
 }
 
-define <vscale x 2 x i64> @umulh_i64(<vscale x 2 x i64> %a, <vscale x 2 x i64> %b) #0 {
+define <vscale x 2 x i64> @umulh_i64(<vscale x 2 x i64> %a, <vscale x 2 x i64> %b) {
 ; CHECK-LABEL: umulh_i64:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    umulh z0.d, z0.d, z1.d
@@ -113,4 +113,261 @@ define <vscale x 2 x i64> @umulh_i64(<vscale x 2 x i64> %a, <vscale x 2 x i64> %
   ret <vscale x 2 x i64> %tr
 }
 
-attributes #0 = { "target-features"="+sve2" }
+
+; Fixed-length 128bits
+
+define <16 x i8> @smulh_v16i8(<16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: smulh_v16i8:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    smull2 v2.8h, v0.16b, v1.16b
+; CHECK-NEXT:    smull v0.8h, v0.8b, v1.8b
+; CHECK-NEXT:    uzp2 v0.16b, v0.16b, v2.16b
+; CHECK-NEXT:    ret
+  %1 = sext <16 x i8> %a to <16 x i16>
+  %2 = sext <16 x i8> %b to <16 x i16>
+  %mul = mul <16 x i16> %1, %2
+  %shr = lshr <16 x i16> %mul, splat(i16 8)
+  %tr = trunc <16 x i16> %shr to <16 x i8>
+  ret <16 x i8> %tr
+}
+
+define <8 x i16> @smulh_v8i16(<8 x i16> %a, <8 x i16> %b) {
+; CHECK-LABEL: smulh_v8i16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    smull2 v2.4s, v0.8h, v1.8h
+; CHECK-NEXT:    smull v0.4s, v0.4h, v1.4h
+; CHECK-NEXT:    uzp2 v0.8h, v0.8h, v2.8h
+; CHECK-NEXT:    ret
+  %1 = sext <8 x i16> %a to <8 x i32>
+  %2 = sext <8 x i16> %b to <8 x i32>
+  %mul = mul <8 x i32> %1, %2
+  %shr = lshr <8 x i32> %mul, splat(i32 16)
+  %tr = trunc <8 x i32> %shr to <8 x i16>
+  ret <8 x i16> %tr
+}
+
+define <4 x i32> @smulh_v4i32(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: smulh_v4i32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    smull2 v2.2d, v0.4s, v1.4s
+; CHECK-NEXT:    smull v0.2d, v0.2s, v1.2s
+; CHECK-NEXT:    uzp2 v0.4s, v0.4s, v2.4s
+; CHECK-NEXT:    ret
+  %1 = sext <4 x i32> %a to <4 x i64>
+  %2 = sext <4 x i32> %b to <4 x i64>
+  %mul = mul <4 x i64> %1, %2
+  %shr = lshr <4 x i64> %mul, splat(i64 32)
+  %tr = trunc <4 x i64> %shr to <4 x i32>
+  ret <4 x i32> %tr
+}
+
+define <2 x i64> @smulh_v2i64(<2 x i64> %a, <2 x i64> %b) {
+; CHECK-LABEL: smulh_v2i64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov x8, v0.d[1]
+; CHECK-NEXT:    mov x9, v1.d[1]
+; CHECK-NEXT:    fmov x10, d0
+; CHECK-NEXT:    fmov x11, d1
+; CHECK-NEXT:    smulh x10, x10, x11
+; CHECK-NEXT:    smulh x8, x8, x9
+; CHECK-NEXT:    fmov d0, x10
+; CHECK-NEXT:    fmov d1, x8
+; CHECK-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-NEXT:    ret
+  %1 = sext <2 x i64> %a to <2 x i128>
+  %2 = sext <2 x i64> %b to <2 x i128>
+  %mul = mul <2 x i128> %1, %2
+  %shr = lshr <2 x i128> %mul, splat(i128 64)
+  %tr = trunc <2 x i128> %shr to <2 x i64>
+  ret <2 x i64> %tr
+}
+
+define <16 x i8> @umulh_v16i8(<16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: umulh_v16i8:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    umull2 v2.8h, v0.16b, v1.16b
+; CHECK-NEXT:    umull v0.8h, v0.8b, v1.8b
+; CHECK-NEXT:    uzp2 v0.16b, v0.16b, v2.16b
+; CHECK-NEXT:    ret
+  %1 = zext <16 x i8> %a to <16 x i16>
+  %2 = zext <16 x i8> %b to <16 x i16>
+  %mul = mul <16 x i16> %1, %2
+  %shr = lshr <16 x i16> %mul, splat(i16 8)
+  %tr = trunc <16 x i16> %shr to <16 x i8>
+  ret <16 x i8> %tr
+}
+
+define <8 x i16> @umulh_v8i16(<8 x i16> %a, <8 x i16> %b) {
+; CHECK-LABEL: umulh_v8i16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    umull2 v2.4s, v0.8h, v1.8h
+; CHECK-NEXT:    umull v0.4s, v0.4h, v1.4h
+; CHECK-NEXT:    uzp2 v0.8h, v0.8h, v2.8h
+; CHECK-NEXT:    ret
+  %1 = zext <8 x i16> %a to <8 x i32>
+  %2 = zext <8 x i16> %b to <8 x i32>
+  %mul = mul <8 x i32> %1, %2
+  %shr = lshr <8 x i32> %mul, splat(i32 16)
+  %tr = trunc <8 x i32> %shr to <8 x i16>
+  ret <8 x i16> %tr
+}
+
+define <4 x i32> @umulh_v4i32(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: umulh_v4i32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    umull2 v2.2d, v0.4s, v1.4s
+; CHECK-NEXT:    umull v0.2d, v0.2s, v1.2s
+; CHECK-NEXT:    uzp2 v0.4s, v0.4s, v2.4s
+; CHECK-NEXT:    ret
+  %1 = zext <4 x i32> %a to <4 x i64>
+  %2 = zext <4 x i32> %b to <4 x i64>
+  %mul = mul <4 x i64> %1, %2
+  %shr = lshr <4 x i64> %mul, splat(i64 32)
+  %tr = trunc <4 x i64> %shr to <4 x i32>
+  ret <4 x i32> %tr
+}
+
+define <2 x i64> @umulh_v2i64(<2 x i64> %a, <2 x i64> %b) {
+; CHECK-LABEL: umulh_v2i64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov x8, v0.d[1]
+; CHECK-NEXT:    mov x9, v1.d[1]
+; CHECK-NEXT:    fmov x10, d0
+; CHECK-NEXT:    fmov x11, d1
+; CHECK-NEXT:    umulh x10, x10, x11
+; CHECK-NEXT:    umulh x8, x8, x9
+; CHECK-NEXT:    fmov d0, x10
+; CHECK-NEXT:    fmov d1, x8
+; CHECK-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-NEXT:    ret
+  %1 = zext <2 x i64> %a to <2 x i128>
+  %2 = zext <2 x i64> %b to <2 x i128>
+  %mul = mul <2 x i128> %1, %2
+  %shr = lshr <2 x i128> %mul, splat(i128 64)
+  %tr = trunc <2 x i128> %shr to <2 x i64>
+  ret <2 x i64> %tr
+}
+
+
+
+; Fixed-length 64bits
+
+define <8 x i8> @smulh_v8i8(<8 x i8> %a, <8 x i8> %b) {
+; CHECK-LABEL: smulh_v8i8:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    smull v0.8h, v0.8b, v1.8b
+; CHECK-NEXT:    shrn v0.8b, v0.8h, #8
+; CHECK-NEXT:    ret
+  %1 = sext <8 x i8> %a to <8 x i16>
+  %2 = sext <8 x i8> %b to <8 x i16>
+  %mul = mul <8 x i16> %1, %2
+  %shr = lshr <8 x i16> %mul, splat(i16 8)
+  %tr = trunc <8 x i16> %shr to <8 x i8>
+  ret <8 x i8> %tr
+}
+
+define <4 x i16> @smulh_v4i16(<4 x i16> %a, <4 x i16> %b) {
+; CHECK-LABEL: smulh_v4i16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    smull v0.4s, v0.4h, v1.4h
+; CHECK-NEXT:    shrn v0.4h, v0.4s, #16
+; CHECK-NEXT:    ret
+  %1 = sext <4 x i16> %a to <4 x i32>
+  %2 = sext <4 x i16> %b to <4 x i32>
+  %mul = mul <4 x i32> %1, %2
+  %shr = lshr <4 x i32> %mul, splat(i32 16)
+  %tr = trunc <4 x i32> %shr to <4 x i16>
+  ret <4 x i16> %tr
+}
+
+define <2 x i32> @smulh_v2i32(<2 x i32> %a, <2 x i32> %b) {
+; CHECK-LABEL: smulh_v2i32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    smull v0.2d, v0.2s, v1.2s
+; CHECK-NEXT:    shrn v0.2s, v0.2d, #32
+; CHECK-NEXT:    ret
+  %1 = sext <2 x i32> %a to <2 x i64>
+  %2 = sext <2 x i32> %b to <2 x i64>
+  %mul = mul <2 x i64> %1, %2
+  %shr = lshr <2 x i64> %mul, splat(i64 32)
+  %tr = trunc <2 x i64> %shr to <2 x i32>
+  ret <2 x i32> %tr
+}
+
+define <1 x i64> @smulh_v1i64(<1 x i64> %a, <1 x i64> %b) {
+; CHECK-LABEL: smulh_v1i64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $d1 killed $d1 def $q1
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT:    fmov x8, d0
+; CHECK-NEXT:    fmov x9, d1
+; CHECK-NEXT:    smulh x8, x8, x9
+; CHECK-NEXT:    fmov d0, x8
+; CHECK-NEXT:    ret
+  %1 = sext <1 x i64> %a to <1 x i128>
+  %2 = sext <1 x i64> %b to <1 x i128>
+  %mul = mul <1 x i128> %1, %2
+  %shr = lshr <1 x i128> %mul, splat(i128 64)
+  %tr = trunc <1 x i128> %shr to <1 x i64>
+  ret <1 x i64> %tr
+}
+
+define <8 x i8> @umulh_v8i8(<8 x i8> %a, <8 x i8> %b) {
+; CHECK-LABEL: umulh_v8i8:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    umull v0.8h, v0.8b, v1.8b
+; CHECK-NEXT:    shrn v0.8b, v0.8h, #8
+; CHECK-NEXT:    ret
+  %1 = zext <8 x i8> %a to <8 x i16>
+  %2 = zext <8 x i8> %b to <8 x i16>
+  %mul = mul <8 x i16> %1, %2
+  %shr = lshr <8 x i16> %mul, splat(i16 8)
+  %tr = trunc <8 x i16> %shr to <8 x i8>
+  ret <8 x i8> %tr
+}
+
+define <4 x i16> @umulh_v4i16(<4 x i16> %a, <4 x i16> %b) {
+; CHECK-LABEL: umulh_v4i16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    umull v0.4s, v0.4h, v1.4h
+; CHECK-NEXT:    shrn v0.4h, v0.4s, #16
+; CHECK-NEXT:    ret
+  %1 = zext <4 x i16> %a to <4 x i32>
+  %2 = zext <4 x i16> %b to <4 x i32>
+  %mul = mul <4 x i32> %1, %2
+  %shr = lshr <4 x i32> %mul, splat(i32 16)
+  %tr = trunc <4 x i32> %shr to <4 x i16>
+  ret <4 x i16> %tr
+}
+
+define <2 x i32> @umulh_v2i32(<2 x i32> %a, <2 x i32> %b) {
+; CHECK-LABEL: umulh_v2i32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    umull v0.2d, v0.2s, v1.2s
+; CHECK-NEXT:    shrn v0.2s, v0.2d, #32
+; CHECK-NEXT:    ret
+  %1 = zext <2 x i32> %a to <2 x i64>
+  %2 = zext <2 x i32> %b to <2 x i64>
+  %mul = mul <2 x i64> %1, %2
+  %shr = lshr <2 x i64> %mul, splat(i64 32)
+  %tr = trunc <2 x i64> %shr to <2 x i32>
+  ret <2 x i32> %tr
+}
+
+define <1 x i64> @umulh_v1i64(<1 x i64> %a, <1 x i64> %b) {
+; CHECK-LABEL: umulh_v1i64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $d1 killed $d1 def $q1
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT:    fmov x8, d0
+; CHECK-NEXT:    fmov x9, d1
+; CHECK-NEXT:    umulh x8, x8, x9
+; CHECK-NEXT:    fmov d0, x8
+; CHECK-NEXT:    ret
+  %1 = zext <1 x i64> %a to <1 x i128>
+  %2 = zext <1 x i64> %b to <1 x i128>
+  %mul = mul <1 x i128> %1, %2
+  %shr = lshr <1 x i128> %mul, splat(i128 64)
+  %tr = trunc <1 x i128> %shr to <1 x i64>
+  ret <1 x i64> %tr
+}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/amdgpu-irtranslator.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/amdgpu-irtranslator.ll
index 7dd907e3c143f..fc236147f1238 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/amdgpu-irtranslator.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/amdgpu-irtranslator.ll
@@ -20,5 +20,3 @@ define void @addi32(i32 %arg1, i32 %arg2) {
   store i32 %res, ptr addrspace(1) poison
   ret void
 }
-;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-; CHECK: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/extractelement.i128.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/extractelement.i128.ll
index 405861d791169..9dfd0a47d1e1e 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/extractelement.i128.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/extractelement.i128.ll
@@ -10,41 +10,75 @@ define amdgpu_ps i128 @extractelement_sgpr_v4i128_sgpr_idx(ptr addrspace(4) inre
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_and_b32 s0, s4, 3
 ; GFX9-NEXT:    s_lshl_b32 s0, s0, 4
-; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[2:3], s0 offset:0x0
-; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v0, s0
+; GFX9-NEXT:    global_load_dwordx4 v[0:3], v0, s[2:3]
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX9-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX9-NEXT:    v_readfirstlane_b32 s2, v2
+; GFX9-NEXT:    v_readfirstlane_b32 s3, v3
 ; GFX9-NEXT:    ; return to shader part epilog
 ;
 ; GFX8-LABEL: extractelement_sgpr_v4i128_sgpr_idx:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_and_b32 s0, s4, 3
 ; GFX8-NEXT:    s_lshl_b32 s0, s0, 4
-; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[2:3], s0
-; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    s_add_u32 s0, s2, s0
+; GFX8-NEXT:    s_addc_u32 s1, s3, 0
+; GFX8-NEXT:    v_mov_b32_e32 v0, s0
+; GFX8-NEXT:    v_mov_b32_e32 v1, s1
+; GFX8-NEXT:    flat_load_dwordx4 v[0:3], v[0:1]
+; GFX8-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX8-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX8-NEXT:    v_readfirstlane_b32 s2, v2
+; GFX8-NEXT:    v_readfirstlane_b32 s3, v3
 ; GFX8-NEXT:    ; return to shader part epilog
 ;
 ; GFX7-LABEL: extractelement_sgpr_v4i128_sgpr_idx:
 ; GFX7:       ; %bb.0:
-; GFX7-NEXT:    s_and_b32 s0, s4, 3
-; GFX7-NEXT:    s_lshl_b32 s0, s0, 4
-; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[2:3], s0
-; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_mov_b32 s0, s2
+; GFX7-NEXT:    s_and_b32 s2, s4, 3
+; GFX7-NEXT:    s_lshl_b32 s4, s2, 4
+; GFX7-NEXT:    s_mov_b32 s5, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    s_mov_b32 s1, s3
+; GFX7-NEXT:    s_mov_b32 s3, 0xf000
+; GFX7-NEXT:    s_mov_b32 s2, s5
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    buffer_load_dwordx4 v[0:3], v[0:1], s[0:3], 0 addr64
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX7-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX7-NEXT:    v_readfirstlane_b32 s2, v2
+; GFX7-NEXT:    v_readfirstlane_b32 s3, v3
 ; GFX7-NEXT:    ; return to shader part epilog
 ;
 ; GFX10-LABEL: extractelement_sgpr_v4i128_sgpr_idx:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_and_b32 s0, s4, 3
 ; GFX10-NEXT:    s_lshl_b32 s0, s0, 4
-; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[2:3], s0 offset:0x0
-; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-NEXT:    global_load_dwordx4 v[0:3], v0, s[2:3]
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX10-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX10-NEXT:    v_readfirstlane_b32 s2, v2
+; GFX10-NEXT:    v_readfirstlane_b32 s3, v3
 ; GFX10-NEXT:    ; return to shader part epilog
 ;
 ; GFX11-LABEL: extractelement_sgpr_v4i128_sgpr_idx:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_and_b32 s0, s4, 3
-; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
 ; GFX11-NEXT:    s_lshl_b32 s0, s0, 4
-; GFX11-NEXT:    s_load_b128 s[0:3], s[2:3], s0 offset:0x0
-; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_mov_b32_e32 v0, s0
+; GFX11-NEXT:    global_load_b128 v[0:3], v0, s[2:3]
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX11-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX11-NEXT:    v_readfirstlane_b32 s2, v2
+; GFX11-NEXT:    v_readfirstlane_b32 s3, v3
 ; GFX11-NEXT:    ; return to shader part epilog
   %vector = load <4 x i128>, ptr addrspace(4) %ptr
   %element = extractelement <4 x i128> %vector, i32 %idx
@@ -281,22 +315,63 @@ define amdgpu_ps i128 @extractelement_sgpr_v4i128_vgpr_idx(ptr addrspace(4) inre
 }
 
 define amdgpu_ps i128 @extractelement_sgpr_v4i128_idx0(ptr addrspace(4) inreg %ptr) {
-; GCN-LABEL: extractelement_sgpr_v4i128_idx0:
-; GCN:       ; %bb.0:
-; GCN-NEXT:    s_load_dwordx4 s[0:3], s[2:3], 0x0
-; GCN-NEXT:    s_waitcnt lgkmcnt(0)
-; GCN-NEXT:    ; return to shader part epilog
+; GFX9-LABEL: extractelement_sgpr_v4i128_idx0:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    v_mov_b32_e32 v0, 0
+; GFX9-NEXT:    global_load_dwordx4 v[0:3], v0, s[2:3]
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX9-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX9-NEXT:    v_readfirstlane_b32 s2, v2
+; GFX9-NEXT:    v_readfirstlane_b32 s3, v3
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: extractelement_sgpr_v4i128_idx0:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    v_mov_b32_e32 v0, s2
+; GFX8-NEXT:    v_mov_b32_e32 v1, s3
+; GFX8-NEXT:    flat_load_dwordx4 v[0:3], v[0:1]
+; GFX8-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX8-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX8-NEXT:    v_readfirstlane_b32 s2, v2
+; GFX8-NEXT:    v_readfirstlane_b32 s3, v3
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX7-LABEL: extractelement_sgpr_v4i128_idx0:
+; GFX7:       ; %bb.0:
+; GFX7-NEXT:    s_mov_b32 s0, s2
+; GFX7-NEXT:    s_mov_b32 s1, s3
+; GFX7-NEXT:    s_mov_b32 s2, -1
+; GFX7-NEXT:    s_mov_b32 s3, 0xf000
+; GFX7-NEXT:    buffer_load_dwordx4 v[0:3], off, s[0:3], 0
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX7-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX7-NEXT:    v_readfirstlane_b32 s2, v2
+; GFX7-NEXT:    v_readfirstlane_b32 s3, v3
+; GFX7-NEXT:    ; return to shader part epilog
 ;
 ; GFX10-LABEL: extractelement_sgpr_v4i128_idx0:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[2:3], 0x0
-; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-NEXT:    global_load_dwordx4 v[0:3], v0, s[2:3]
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX10-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX10-NEXT:    v_readfirstlane_b32 s2, v2
+; GFX10-NEXT:    v_readfirstlane_b32 s3, v3
 ; GFX10-NEXT:    ; return to shader part epilog
 ;
 ; GFX11-LABEL: extractelement_sgpr_v4i128_idx0:
 ; GFX11:       ; %bb.0:
-; GFX11-NEXT:    s_load_b128 s[0:3], s[2:3], 0x0
-; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_mov_b32_e32 v0, 0
+; GFX11-NEXT:    global_load_b128 v[0:3], v0, s[2:3]
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX11-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX11-NEXT:    v_readfirstlane_b32 s2, v2
+; GFX11-NEXT:    v_readfirstlane_b32 s3, v3
 ; GFX11-NEXT:    ; return to shader part epilog
   %vector = load <4 x i128>, ptr addrspace(4) %ptr
   %element = extractelement <4 x i128> %vector, i32 0
@@ -306,32 +381,63 @@ define amdgpu_ps i128 @extractelement_sgpr_v4i128_idx0(ptr addrspace(4) inreg %p
 define amdgpu_ps i128 @extractelement_sgpr_v4i128_idx1(ptr addrspace(4) inreg %ptr) {
 ; GFX9-LABEL: extractelement_sgpr_v4i128_idx1:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[2:3], 0x10
-; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v0, 0
+; GFX9-NEXT:    global_load_dwordx4 v[0:3], v0, s[2:3] offset:16
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX9-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX9-NEXT:    v_readfirstlane_b32 s2, v2
+; GFX9-NEXT:    v_readfirstlane_b32 s3, v3
 ; GFX9-NEXT:    ; return to shader part epilog
 ;
 ; GFX8-LABEL: extractelement_sgpr_v4i128_idx1:
 ; GFX8:       ; %bb.0:
-; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[2:3], 0x10
-; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    s_add_u32 s0, s2, 16
+; GFX8-NEXT:    s_addc_u32 s1, s3, 0
+; GFX8-NEXT:    v_mov_b32_e32 v0, s0
+; GFX8-NEXT:    v_mov_b32_e32 v1, s1
+; GFX8-NEXT:    flat_load_dwordx4 v[0:3], v[0:1]
+; GFX8-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX8-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX8-NEXT:    v_readfirstlane_b32 s2, v2
+; GFX8-NEXT:    v_readfirstlane_b32 s3, v3
 ; GFX8-NEXT:    ; return to shader part epilog
 ;
 ; GFX7-LABEL: extractelement_sgpr_v4i128_idx1:
 ; GFX7:       ; %bb.0:
-; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[2:3], 0x4
-; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_mov_b32 s0, s2
+; GFX7-NEXT:    s_mov_b32 s1, s3
+; GFX7-NEXT:    s_mov_b32 s2, -1
+; GFX7-NEXT:    s_mov_b32 s3, 0xf000
+; GFX7-NEXT:    buffer_load_dwordx4 v[0:3], off, s[0:3], 0 offset:16
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX7-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX7-NEXT:    v_readfirstlane_b32 s2, v2
+; GFX7-NEXT:    v_readfirstlane_b32 s3, v3
 ; GFX7-NEXT:    ; return to shader part epilog
 ;
 ; GFX10-LABEL: extractelement_sgpr_v4i128_idx1:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[2:3], 0x10
-; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-NEXT:    global_load_dwordx4 v[0:3], v0, s[2:3] offset:16
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX10-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX10-NEXT:    v_readfirstlane_b32 s2, v2
+; GFX10-NEXT:    v_readfirstlane_b32 s3, v3
 ; GFX10-NEXT:    ; return to shader part epilog
 ;
 ; GFX11-LABEL: extractelement_sgpr_v4i128_idx1:
 ; GFX11:       ; %bb.0:
-; GFX11-NEXT:    s_load_b128 s[0:3], s[2:3], 0x10
-; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_mov_b32_e32 v0, 0
+; GFX11-NEXT:    global_load_b128 v[0:3], v0, s[2:3] offset:16
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX11-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX11-NEXT:    v_readfirstlane_b32 s2, v2
+; GFX11-NEXT:    v_readfirstlane_b32 s3, v3
 ; GFX11-NEXT:    ; return to shader part epilog
   %vector = load <4 x i128>, ptr addrspace(4) %ptr
   %element = extractelement <4 x i128> %vector, i32 1
@@ -341,32 +447,63 @@ define amdgpu_ps i128 @extractelement_sgpr_v4i128_idx1(ptr addrspace(4) inreg %p
 define amdgpu_ps i128 @extractelement_sgpr_v4i128_idx2(ptr addrspace(4) inreg %ptr) {
 ; GFX9-LABEL: extractelement_sgpr_v4i128_idx2:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[2:3], 0x20
-; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v0, 0
+; GFX9-NEXT:    global_load_dwordx4 v[0:3], v0, s[2:3] offset:32
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX9-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX9-NEXT:    v_readfirstlane_b32 s2, v2
+; GFX9-NEXT:    v_readfirstlane_b32 s3, v3
 ; GFX9-NEXT:    ; return to shader part epilog
 ;
 ; GFX8-LABEL: extractelement_sgpr_v4i128_idx2:
 ; GFX8:       ; %bb.0:
-; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[2:3], 0x20
-; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    s_add_u32 s0, s2, 32
+; GFX8-NEXT:    s_addc_u32 s1, s3, 0
+; GFX8-NEXT:    v_mov_b32_e32 v0, s0
+; GFX8-NEXT:    v_mov_b32_e32 v1, s1
+; GFX8-NEXT:    flat_load_dwordx4 v[0:3], v[0:1]
+; GFX8-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX8-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX8-NEXT:    v_readfirstlane_b32 s2, v2
+; GFX8-NEXT:    v_readfirstlane_b32 s3, v3
 ; GFX8-NEXT:    ; return to shader part epilog
 ;
 ; GFX7-LABEL: extractelement_sgpr_v4i128_idx2:
 ; GFX7:       ; %bb.0:
-; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[2:3], 0x8
-; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_mov_b32 s0, s2
+; GFX7-NEXT:    s_mov_b32 s1, s3
+; GFX7-NEXT:    s_mov_b32 s2, -1
+; GFX7-NEXT:    s_mov_b32 s3, 0xf000
+; GFX7-NEXT:    buffer_load_dwordx4 v[0:3], off, s[0:3], 0 offset:32
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX7-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX7-NEXT:    v_readfirstlane_b32 s2, v2
+; GFX7-NEXT:    v_readfirstlane_b32 s3, v3
 ; GFX7-NEXT:    ; return to shader part epilog
 ;
 ; GFX10-LABEL: extractelement_sgpr_v4i128_idx2:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[2:3], 0x20
-; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-NEXT:    global_load_dwordx4 v[0:3], v0, s[2:3] offset:32
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX10-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX10-NEXT:    v_readfirstlane_b32 s2, v2
+; GFX10-NEXT:    v_readfirstlane_b32 s3, v3
 ; GFX10-NEXT:    ; return to shader part epilog
 ;
 ; GFX11-LABEL: extractelement_sgpr_v4i128_idx2:
 ; GFX11:       ; %bb.0:
-; GFX11-NEXT:    s_load_b128 s[0:3], s[2:3], 0x20
-; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_mov_b32_e32 v0, 0
+; GFX11-NEXT:    global_load_b128 v[0:3], v0, s[2:3] offset:32
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX11-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX11-NEXT:    v_readfirstlane_b32 s2, v2
+; GFX11-NEXT:    v_readfirstlane_b32 s3, v3
 ; GFX11-NEXT:    ; return to shader part epilog
   %vector = load <4 x i128>, ptr addrspace(4) %ptr
   %element = extractelement <4 x i128> %vector, i32 2
@@ -376,32 +513,63 @@ define amdgpu_ps i128 @extractelement_sgpr_v4i128_idx2(ptr addrspace(4) inreg %p
 define amdgpu_ps i128 @extractelement_sgpr_v4i128_idx3(ptr addrspace(4) inreg %ptr) {
 ; GFX9-LABEL: extractelement_sgpr_v4i128_idx3:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[2:3], 0x30
-; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v0, 0
+; GFX9-NEXT:    global_load_dwordx4 v[0:3], v0, s[2:3] offset:48
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX9-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX9-NEXT:    v_readfirstlane_b32 s2, v2
+; GFX9-NEXT:    v_readfirstlane_b32 s3, v3
 ; GFX9-NEXT:    ; return to shader part epilog
 ;
 ; GFX8-LABEL: extractelement_sgpr_v4i128_idx3:
 ; GFX8:       ; %bb.0:
-; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[2:3], 0x30
-; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    s_add_u32 s0, s2, 48
+; GFX8-NEXT:    s_addc_u32 s1, s3, 0
+; GFX8-NEXT:    v_mov_b32_e32 v0, s0
+; GFX8-NEXT:    v_mov_b32_e32 v1, s1
+; GFX8-NEXT:    flat_load_dwordx4 v[0:3], v[0:1]
+; GFX8-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX8-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX8-NEXT:    v_readfirstlane_b32 s2, v2
+; GFX8-NEXT:    v_readfirstlane_b32 s3, v3
 ; GFX8-NEXT:    ; return to shader part epilog
 ;
 ; GFX7-LABEL: extractelement_sgpr_v4i128_idx3:
 ; GFX7:       ; %bb.0:
-; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[2:3], 0xc
-; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_mov_b32 s0, s2
+; GFX7-NEXT:    s_mov_b32 s1, s3
+; GFX7-NEXT:    s_mov_b32 s2, -1
+; GFX7-NEXT:    s_mov_b32 s3, 0xf000
+; GFX7-NEXT:    buffer_load_dwordx4 v[0:3], off, s[0:3], 0 offset:48
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX7-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX7-NEXT:    v_readfirstlane_b32 s2, v2
+; GFX7-NEXT:    v_readfirstlane_b32 s3, v3
 ; GFX7-NEXT:    ; return to shader part epilog
 ;
 ; GFX10-LABEL: extractelement_sgpr_v4i128_idx3:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[2:3], 0x30
-; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-NEXT:    global_load_dwordx4 v[0:3], v0, s[2:3] offset:48
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX10-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX10-NEXT:    v_readfirstlane_b32 s2, v2
+; GFX10-NEXT:    v_readfirstlane_b32 s3, v3
 ; GFX10-NEXT:    ; return to shader part epilog
 ;
 ; GFX11-LABEL: extractelement_sgpr_v4i128_idx3:
 ; GFX11:       ; %bb.0:
-; GFX11-NEXT:    s_load_b128 s[0:3], s[2:3], 0x30
-; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_mov_b32_e32 v0, 0
+; GFX11-NEXT:    global_load_b128 v[0:3], v0, s[2:3] offset:48
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX11-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX11-NEXT:    v_readfirstlane_b32 s2, v2
+; GFX11-NEXT:    v_readfirstlane_b32 s3, v3
 ; GFX11-NEXT:    ; return to shader part epilog
   %vector = load <4 x i128>, ptr addrspace(4) %ptr
   %element = extractelement <4 x i128> %vector, i32 3
@@ -585,3 +753,5 @@ define i128 @extractelement_vgpr_v4i128_idx3(ptr addrspace(1) %ptr) {
   %element = extractelement <4 x i128> %vector, i32 3
   ret i128 %element
 }
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; GCN: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/implicit-kernarg-backend-usage-global-isel.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/implicit-kernarg-backend-usage-global-isel.ll
index 9539ec465e02f..91ee7642790fc 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/implicit-kernarg-backend-usage-global-isel.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/implicit-kernarg-backend-usage-global-isel.ll
@@ -11,28 +11,40 @@ define amdgpu_kernel void @addrspacecast(ptr addrspace(5) %ptr.private, ptr addr
 ; GFX8V4-LABEL: addrspacecast:
 ; GFX8V4:       ; %bb.0:
 ; GFX8V4-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
-; GFX8V4-NEXT:    s_load_dwordx2 s[2:3], s[6:7], 0x40
 ; GFX8V4-NEXT:    s_add_i32 s12, s12, s17
 ; GFX8V4-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
-; GFX8V4-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX8V4-NEXT:    s_add_u32 s2, s6, 0x44
+; GFX8V4-NEXT:    s_addc_u32 s3, s7, 0
+; GFX8V4-NEXT:    v_mov_b32_e32 v0, s2
 ; GFX8V4-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8V4-NEXT:    s_mov_b32 s4, s0
-; GFX8V4-NEXT:    s_mov_b32 s5, s3
 ; GFX8V4-NEXT:    s_cmp_lg_u32 s0, -1
-; GFX8V4-NEXT:    s_cselect_b64 s[4:5], s[4:5], 0
-; GFX8V4-NEXT:    s_mov_b32 s6, s1
-; GFX8V4-NEXT:    s_mov_b32 s7, s2
+; GFX8V4-NEXT:    v_mov_b32_e32 v1, s3
+; GFX8V4-NEXT:    s_cselect_b32 s2, 1, 0
+; GFX8V4-NEXT:    s_and_b32 s4, 1, s2
+; GFX8V4-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; GFX8V4-NEXT:    s_add_u32 s2, s6, 64
+; GFX8V4-NEXT:    flat_load_dword v3, v[0:1]
+; GFX8V4-NEXT:    s_addc_u32 s3, s7, 0
+; GFX8V4-NEXT:    v_mov_b32_e32 v0, s2
+; GFX8V4-NEXT:    v_mov_b32_e32 v1, s3
+; GFX8V4-NEXT:    flat_load_dword v4, v[0:1]
 ; GFX8V4-NEXT:    s_cmp_lg_u32 s1, -1
-; GFX8V4-NEXT:    v_mov_b32_e32 v0, s4
-; GFX8V4-NEXT:    s_cselect_b64 s[0:1], s[6:7], 0
-; GFX8V4-NEXT:    v_mov_b32_e32 v2, 1
-; GFX8V4-NEXT:    v_mov_b32_e32 v1, s5
-; GFX8V4-NEXT:    flat_store_dword v[0:1], v2
-; GFX8V4-NEXT:    s_waitcnt vmcnt(0)
 ; GFX8V4-NEXT:    v_mov_b32_e32 v0, s0
-; GFX8V4-NEXT:    v_mov_b32_e32 v2, 2
+; GFX8V4-NEXT:    s_cselect_b32 s0, 1, 0
+; GFX8V4-NEXT:    s_and_b32 s0, 1, s0
 ; GFX8V4-NEXT:    v_mov_b32_e32 v1, s1
-; GFX8V4-NEXT:    flat_store_dword v[0:1], v2
+; GFX8V4-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s4
+; GFX8V4-NEXT:    v_cmp_ne_u32_e64 s[0:1], 0, s0
+; GFX8V4-NEXT:    v_mov_b32_e32 v5, 1
+; GFX8V4-NEXT:    v_cndmask_b32_e32 v0, 0, v0, vcc
+; GFX8V4-NEXT:    v_cndmask_b32_e64 v2, 0, v1, s[0:1]
+; GFX8V4-NEXT:    s_waitcnt vmcnt(1)
+; GFX8V4-NEXT:    v_cndmask_b32_e32 v1, 0, v3, vcc
+; GFX8V4-NEXT:    flat_store_dword v[0:1], v5
+; GFX8V4-NEXT:    s_waitcnt vmcnt(0)
+; GFX8V4-NEXT:    v_mov_b32_e32 v0, 2
+; GFX8V4-NEXT:    v_cndmask_b32_e64 v3, 0, v4, s[0:1]
+; GFX8V4-NEXT:    flat_store_dword v[2:3], v0
 ; GFX8V4-NEXT:    s_waitcnt vmcnt(0)
 ; GFX8V4-NEXT:    s_endpgm
 ;
@@ -124,13 +136,15 @@ define amdgpu_kernel void @addrspacecast(ptr addrspace(5) %ptr.private, ptr addr
 define amdgpu_kernel void @llvm_amdgcn_is_shared(ptr %ptr) #0 {
 ; GFX8V4-LABEL: llvm_amdgcn_is_shared:
 ; GFX8V4:       ; %bb.0:
-; GFX8V4-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
-; GFX8V4-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8V4-NEXT:    s_load_dword s0, s[6:7], 0x40
-; GFX8V4-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8V4-NEXT:    s_cmp_eq_u32 s1, s0
-; GFX8V4-NEXT:    s_cselect_b32 s0, 1, 0
+; GFX8V4-NEXT:    s_add_u32 s0, s6, 64
+; GFX8V4-NEXT:    s_addc_u32 s1, s7, 0
 ; GFX8V4-NEXT:    v_mov_b32_e32 v0, s0
+; GFX8V4-NEXT:    v_mov_b32_e32 v1, s1
+; GFX8V4-NEXT:    flat_load_dword v0, v[0:1]
+; GFX8V4-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX8V4-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8V4-NEXT:    v_cmp_eq_u32_e32 vcc, s1, v0
+; GFX8V4-NEXT:    v_cndmask_b32_e64 v0, 0, 1, vcc
 ; GFX8V4-NEXT:    flat_store_dword v[0:1], v0
 ; GFX8V4-NEXT:    s_waitcnt vmcnt(0)
 ; GFX8V4-NEXT:    s_endpgm
@@ -180,13 +194,15 @@ define amdgpu_kernel void @llvm_amdgcn_is_shared(ptr %ptr) #0 {
 define amdgpu_kernel void @llvm_amdgcn_is_private(ptr %ptr) #0 {
 ; GFX8V4-LABEL: llvm_amdgcn_is_private:
 ; GFX8V4:       ; %bb.0:
-; GFX8V4-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
-; GFX8V4-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8V4-NEXT:    s_load_dword s0, s[6:7], 0x44
-; GFX8V4-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8V4-NEXT:    s_cmp_eq_u32 s1, s0
-; GFX8V4-NEXT:    s_cselect_b32 s0, 1, 0
+; GFX8V4-NEXT:    s_add_u32 s0, s6, 0x44
+; GFX8V4-NEXT:    s_addc_u32 s1, s7, 0
 ; GFX8V4-NEXT:    v_mov_b32_e32 v0, s0
+; GFX8V4-NEXT:    v_mov_b32_e32 v1, s1
+; GFX8V4-NEXT:    flat_load_dword v0, v[0:1]
+; GFX8V4-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX8V4-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8V4-NEXT:    v_cmp_eq_u32_e32 vcc, s1, v0
+; GFX8V4-NEXT:    v_cndmask_b32_e64 v0, 0, 1, vcc
 ; GFX8V4-NEXT:    flat_store_dword v[0:1], v0
 ; GFX8V4-NEXT:    s_waitcnt vmcnt(0)
 ; GFX8V4-NEXT:    s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/agpr-copy-no-free-registers.ll b/llvm/test/CodeGen/AMDGPU/agpr-copy-no-free-registers.ll
index ab01eeec44234..a21db73cf3714 100644
--- a/llvm/test/CodeGen/AMDGPU/agpr-copy-no-free-registers.ll
+++ b/llvm/test/CodeGen/AMDGPU/agpr-copy-no-free-registers.ll
@@ -511,55 +511,53 @@ define amdgpu_kernel void @introduced_copy_to_sgpr(i64 %arg, i32 %arg1, i32 %arg
 ; GFX908-LABEL: introduced_copy_to_sgpr:
 ; GFX908:       ; %bb.0: ; %bb
 ; GFX908-NEXT:    global_load_ushort v16, v[0:1], off glc
-; GFX908-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x8
+; GFX908-NEXT:    s_load_dwordx4 s[4:7], s[8:9], 0x0
+; GFX908-NEXT:    s_load_dwordx2 s[10:11], s[8:9], 0x10
+; GFX908-NEXT:    s_load_dword s0, s[8:9], 0x18
+; GFX908-NEXT:    s_mov_b32 s12, 0
+; GFX908-NEXT:    s_mov_b32 s9, s12
 ; GFX908-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX908-NEXT:    s_load_dwordx2 s[2:3], s[8:9], 0x0
-; GFX908-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x10
-; GFX908-NEXT:    s_load_dword s5, s[8:9], 0x18
-; GFX908-NEXT:    s_mov_b32 s4, 0
-; GFX908-NEXT:    s_mov_b32 s9, s4
-; GFX908-NEXT:    v_cvt_f32_u32_e32 v0, s1
-; GFX908-NEXT:    s_sub_i32 s8, 0, s1
-; GFX908-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX908-NEXT:    v_cvt_f32_f16_e32 v18, s5
+; GFX908-NEXT:    v_cvt_f32_u32_e32 v0, s7
+; GFX908-NEXT:    s_sub_i32 s1, 0, s7
+; GFX908-NEXT:    v_cvt_f32_f16_e32 v18, s0
 ; GFX908-NEXT:    v_mov_b32_e32 v17, 0
 ; GFX908-NEXT:    v_rcp_iflag_f32_e32 v0, v0
 ; GFX908-NEXT:    v_mul_f32_e32 v0, 0x4f7ffffe, v0
 ; GFX908-NEXT:    v_cvt_u32_f32_e32 v0, v0
-; GFX908-NEXT:    v_readfirstlane_b32 s10, v0
-; GFX908-NEXT:    s_mul_i32 s8, s8, s10
-; GFX908-NEXT:    s_mul_hi_u32 s8, s10, s8
-; GFX908-NEXT:    s_add_i32 s10, s10, s8
-; GFX908-NEXT:    s_mul_hi_u32 s8, s0, s10
-; GFX908-NEXT:    s_mul_i32 s10, s8, s1
-; GFX908-NEXT:    s_sub_i32 s0, s0, s10
-; GFX908-NEXT:    s_add_i32 s11, s8, 1
-; GFX908-NEXT:    s_sub_i32 s10, s0, s1
-; GFX908-NEXT:    s_cmp_ge_u32 s0, s1
-; GFX908-NEXT:    s_cselect_b32 s8, s11, s8
-; GFX908-NEXT:    s_cselect_b32 s0, s10, s0
-; GFX908-NEXT:    s_add_i32 s10, s8, 1
-; GFX908-NEXT:    s_cmp_ge_u32 s0, s1
-; GFX908-NEXT:    s_cselect_b32 s8, s10, s8
-; GFX908-NEXT:    s_lshr_b32 s5, s5, 16
-; GFX908-NEXT:    v_cvt_f32_f16_e32 v19, s5
-; GFX908-NEXT:    s_lshl_b64 s[10:11], s[2:3], 5
-; GFX908-NEXT:    s_lshl_b64 s[14:15], s[8:9], 5
-; GFX908-NEXT:    s_lshl_b64 s[12:13], s[6:7], 5
+; GFX908-NEXT:    v_readfirstlane_b32 s2, v0
+; GFX908-NEXT:    s_mul_i32 s1, s1, s2
+; GFX908-NEXT:    s_mul_hi_u32 s1, s2, s1
+; GFX908-NEXT:    s_add_i32 s2, s2, s1
+; GFX908-NEXT:    s_mul_hi_u32 s1, s6, s2
+; GFX908-NEXT:    s_mul_i32 s2, s1, s7
+; GFX908-NEXT:    s_sub_i32 s2, s6, s2
+; GFX908-NEXT:    s_add_i32 s3, s1, 1
+; GFX908-NEXT:    s_sub_i32 s6, s2, s7
+; GFX908-NEXT:    s_cmp_ge_u32 s2, s7
+; GFX908-NEXT:    s_cselect_b32 s1, s3, s1
+; GFX908-NEXT:    s_cselect_b32 s2, s6, s2
+; GFX908-NEXT:    s_add_i32 s3, s1, 1
+; GFX908-NEXT:    s_cmp_ge_u32 s2, s7
+; GFX908-NEXT:    s_cselect_b32 s8, s3, s1
+; GFX908-NEXT:    s_lshr_b32 s2, s0, 16
+; GFX908-NEXT:    v_cvt_f32_f16_e32 v19, s2
+; GFX908-NEXT:    s_lshl_b64 s[6:7], s[4:5], 5
+; GFX908-NEXT:    s_lshl_b64 s[14:15], s[10:11], 5
 ; GFX908-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX908-NEXT:    s_and_b64 s[0:1], exec, s[0:1]
-; GFX908-NEXT:    s_or_b32 s12, s12, 28
+; GFX908-NEXT:    s_or_b32 s14, s14, 28
+; GFX908-NEXT:    s_lshl_b64 s[16:17], s[8:9], 5
 ; GFX908-NEXT:    v_mov_b32_e32 v1, 0
 ; GFX908-NEXT:    s_waitcnt vmcnt(0)
-; GFX908-NEXT:    v_readfirstlane_b32 s5, v16
-; GFX908-NEXT:    s_and_b32 s5, 0xffff, s5
-; GFX908-NEXT:    s_mul_i32 s3, s3, s5
-; GFX908-NEXT:    s_mul_hi_u32 s9, s2, s5
-; GFX908-NEXT:    s_mul_i32 s2, s2, s5
-; GFX908-NEXT:    s_add_i32 s3, s9, s3
-; GFX908-NEXT:    s_lshl_b64 s[16:17], s[2:3], 5
+; GFX908-NEXT:    v_readfirstlane_b32 s2, v16
+; GFX908-NEXT:    s_and_b32 s2, 0xffff, s2
+; GFX908-NEXT:    s_mul_i32 s3, s5, s2
+; GFX908-NEXT:    s_mul_hi_u32 s5, s4, s2
+; GFX908-NEXT:    s_mul_i32 s2, s4, s2
+; GFX908-NEXT:    s_add_i32 s3, s5, s3
+; GFX908-NEXT:    s_lshl_b64 s[4:5], s[2:3], 5
 ; GFX908-NEXT:    s_branch .LBB3_2
-; GFX908-NEXT:  .LBB3_1: ; %Flow21
+; GFX908-NEXT:  .LBB3_1: ; %Flow20
 ; GFX908-NEXT:    ; in Loop: Header=BB3_2 Depth=1
 ; GFX908-NEXT:    s_andn2_b64 vcc, exec, s[2:3]
 ; GFX908-NEXT:    s_cbranch_vccz .LBB3_12
@@ -572,47 +570,47 @@ define amdgpu_kernel void @introduced_copy_to_sgpr(i64 %arg, i32 %arg1, i32 %arg
 ; GFX908-NEXT:  ; %bb.3: ; %bb14
 ; GFX908-NEXT:    ; in Loop: Header=BB3_2 Depth=1
 ; GFX908-NEXT:    global_load_dwordx2 v[2:3], v[0:1], off
-; GFX908-NEXT:    v_cmp_gt_i64_e64 s[2:3], s[6:7], -1
-; GFX908-NEXT:    s_mov_b32 s5, s4
+; GFX908-NEXT:    v_cmp_gt_i64_e64 s[2:3], s[10:11], -1
+; GFX908-NEXT:    s_mov_b32 s13, s12
 ; GFX908-NEXT:    v_cndmask_b32_e64 v6, 0, 1, s[2:3]
-; GFX908-NEXT:    v_mov_b32_e32 v4, s4
+; GFX908-NEXT:    v_mov_b32_e32 v4, s12
 ; GFX908-NEXT:    v_cmp_ne_u32_e64 s[2:3], 1, v6
-; GFX908-NEXT:    v_mov_b32_e32 v7, s5
-; GFX908-NEXT:    v_mov_b32_e32 v9, s5
-; GFX908-NEXT:    v_mov_b32_e32 v5, s5
-; GFX908-NEXT:    v_mov_b32_e32 v6, s4
-; GFX908-NEXT:    v_mov_b32_e32 v8, s4
-; GFX908-NEXT:    v_cmp_lt_i64_e64 s[18:19], s[6:7], 0
+; GFX908-NEXT:    v_mov_b32_e32 v6, s12
+; GFX908-NEXT:    v_mov_b32_e32 v8, s12
+; GFX908-NEXT:    v_mov_b32_e32 v5, s13
+; GFX908-NEXT:    v_mov_b32_e32 v7, s13
+; GFX908-NEXT:    v_mov_b32_e32 v9, s13
+; GFX908-NEXT:    v_cmp_lt_i64_e64 s[18:19], s[10:11], 0
 ; GFX908-NEXT:    v_mov_b32_e32 v11, v5
-; GFX908-NEXT:    s_mov_b64 s[20:21], s[12:13]
+; GFX908-NEXT:    s_mov_b64 s[20:21], s[14:15]
 ; GFX908-NEXT:    v_mov_b32_e32 v10, v4
 ; GFX908-NEXT:    s_waitcnt vmcnt(0)
-; GFX908-NEXT:    v_readfirstlane_b32 s5, v2
-; GFX908-NEXT:    v_readfirstlane_b32 s9, v3
-; GFX908-NEXT:    s_add_u32 s5, s5, 1
-; GFX908-NEXT:    s_addc_u32 s9, s9, 0
-; GFX908-NEXT:    s_mul_hi_u32 s22, s10, s5
-; GFX908-NEXT:    s_mul_i32 s9, s10, s9
-; GFX908-NEXT:    s_mul_i32 s23, s11, s5
-; GFX908-NEXT:    s_add_i32 s9, s22, s9
-; GFX908-NEXT:    s_mul_i32 s5, s10, s5
-; GFX908-NEXT:    s_add_i32 s9, s9, s23
+; GFX908-NEXT:    v_readfirstlane_b32 s9, v2
+; GFX908-NEXT:    v_readfirstlane_b32 s13, v3
+; GFX908-NEXT:    s_add_u32 s9, s9, 1
+; GFX908-NEXT:    s_addc_u32 s13, s13, 0
+; GFX908-NEXT:    s_mul_hi_u32 s22, s6, s9
+; GFX908-NEXT:    s_mul_i32 s13, s6, s13
+; GFX908-NEXT:    s_mul_i32 s23, s7, s9
+; GFX908-NEXT:    s_add_i32 s13, s22, s13
+; GFX908-NEXT:    s_mul_i32 s9, s6, s9
+; GFX908-NEXT:    s_add_i32 s13, s13, s23
 ; GFX908-NEXT:    s_branch .LBB3_5
 ; GFX908-NEXT:  .LBB3_4: ; %bb58
 ; GFX908-NEXT:    ; in Loop: Header=BB3_5 Depth=2
 ; GFX908-NEXT:    v_add_co_u32_sdwa v2, vcc, v2, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
 ; GFX908-NEXT:    v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
-; GFX908-NEXT:    s_add_u32 s20, s20, s16
+; GFX908-NEXT:    s_add_u32 s20, s20, s4
 ; GFX908-NEXT:    v_cmp_lt_i64_e64 s[24:25], -1, v[2:3]
-; GFX908-NEXT:    s_addc_u32 s21, s21, s17
+; GFX908-NEXT:    s_addc_u32 s21, s21, s5
 ; GFX908-NEXT:    s_mov_b64 s[22:23], 0
 ; GFX908-NEXT:    s_andn2_b64 vcc, exec, s[24:25]
 ; GFX908-NEXT:    s_cbranch_vccz .LBB3_9
 ; GFX908-NEXT:  .LBB3_5: ; %bb16
 ; GFX908-NEXT:    ; Parent Loop BB3_2 Depth=1
 ; GFX908-NEXT:    ; => This Inner Loop Header: Depth=2
-; GFX908-NEXT:    s_add_u32 s22, s20, s5
-; GFX908-NEXT:    s_addc_u32 s23, s21, s9
+; GFX908-NEXT:    s_add_u32 s22, s20, s9
+; GFX908-NEXT:    s_addc_u32 s23, s21, s13
 ; GFX908-NEXT:    global_load_dword v21, v17, s[22:23] offset:-12 glc
 ; GFX908-NEXT:    s_waitcnt vmcnt(0)
 ; GFX908-NEXT:    global_load_dword v20, v17, s[22:23] offset:-8 glc
@@ -660,17 +658,17 @@ define amdgpu_kernel void @introduced_copy_to_sgpr(i64 %arg, i32 %arg1, i32 %arg
 ; GFX908-NEXT:  .LBB3_9: ; %loop.exit.guard
 ; GFX908-NEXT:    ; in Loop: Header=BB3_2 Depth=1
 ; GFX908-NEXT:    s_xor_b64 s[18:19], s[22:23], -1
-; GFX908-NEXT:  .LBB3_10: ; %Flow20
+; GFX908-NEXT:  .LBB3_10: ; %Flow19
 ; GFX908-NEXT:    ; in Loop: Header=BB3_2 Depth=1
 ; GFX908-NEXT:    s_mov_b64 s[2:3], -1
 ; GFX908-NEXT:    s_and_b64 vcc, exec, s[18:19]
 ; GFX908-NEXT:    s_cbranch_vccz .LBB3_1
 ; GFX908-NEXT:  ; %bb.11: ; %bb12
 ; GFX908-NEXT:    ; in Loop: Header=BB3_2 Depth=1
-; GFX908-NEXT:    s_add_u32 s6, s6, s8
-; GFX908-NEXT:    s_addc_u32 s7, s7, 0
-; GFX908-NEXT:    s_add_u32 s12, s12, s14
-; GFX908-NEXT:    s_addc_u32 s13, s13, s15
+; GFX908-NEXT:    s_add_u32 s10, s10, s8
+; GFX908-NEXT:    s_addc_u32 s11, s11, 0
+; GFX908-NEXT:    s_add_u32 s14, s14, s16
+; GFX908-NEXT:    s_addc_u32 s15, s15, s17
 ; GFX908-NEXT:    s_mov_b64 s[2:3], 0
 ; GFX908-NEXT:    s_branch .LBB3_1
 ; GFX908-NEXT:  .LBB3_12: ; %DummyReturnBlock
@@ -679,54 +677,52 @@ define amdgpu_kernel void @introduced_copy_to_sgpr(i64 %arg, i32 %arg1, i32 %arg
 ; GFX90A-LABEL: introduced_copy_to_sgpr:
 ; GFX90A:       ; %bb.0: ; %bb
 ; GFX90A-NEXT:    global_load_ushort v18, v[0:1], off glc
-; GFX90A-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x8
+; GFX90A-NEXT:    s_load_dwordx4 s[4:7], s[8:9], 0x0
+; GFX90A-NEXT:    s_load_dwordx2 s[10:11], s[8:9], 0x10
+; GFX90A-NEXT:    s_load_dword s0, s[8:9], 0x18
+; GFX90A-NEXT:    s_mov_b32 s12, 0
+; GFX90A-NEXT:    s_mov_b32 s9, s12
 ; GFX90A-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT:    s_load_dwordx2 s[2:3], s[8:9], 0x0
-; GFX90A-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x10
-; GFX90A-NEXT:    s_load_dword s5, s[8:9], 0x18
-; GFX90A-NEXT:    s_mov_b32 s4, 0
-; GFX90A-NEXT:    s_mov_b32 s9, s4
-; GFX90A-NEXT:    v_cvt_f32_u32_e32 v0, s1
-; GFX90A-NEXT:    s_sub_i32 s8, 0, s1
+; GFX90A-NEXT:    v_cvt_f32_u32_e32 v0, s7
+; GFX90A-NEXT:    s_sub_i32 s1, 0, s7
 ; GFX90A-NEXT:    v_mov_b32_e32 v19, 0
 ; GFX90A-NEXT:    v_pk_mov_b32 v[2:3], 0, 0
 ; GFX90A-NEXT:    v_rcp_iflag_f32_e32 v0, v0
 ; GFX90A-NEXT:    v_mul_f32_e32 v0, 0x4f7ffffe, v0
 ; GFX90A-NEXT:    v_cvt_u32_f32_e32 v1, v0
-; GFX90A-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT:    v_cvt_f32_f16_e32 v0, s5
-; GFX90A-NEXT:    v_readfirstlane_b32 s10, v1
-; GFX90A-NEXT:    s_mul_i32 s8, s8, s10
-; GFX90A-NEXT:    s_mul_hi_u32 s8, s10, s8
-; GFX90A-NEXT:    s_add_i32 s10, s10, s8
-; GFX90A-NEXT:    s_mul_hi_u32 s8, s0, s10
-; GFX90A-NEXT:    s_mul_i32 s10, s8, s1
-; GFX90A-NEXT:    s_sub_i32 s0, s0, s10
-; GFX90A-NEXT:    s_add_i32 s11, s8, 1
-; GFX90A-NEXT:    s_sub_i32 s10, s0, s1
-; GFX90A-NEXT:    s_cmp_ge_u32 s0, s1
-; GFX90A-NEXT:    s_cselect_b32 s8, s11, s8
-; GFX90A-NEXT:    s_cselect_b32 s0, s10, s0
-; GFX90A-NEXT:    s_add_i32 s10, s8, 1
-; GFX90A-NEXT:    s_cmp_ge_u32 s0, s1
-; GFX90A-NEXT:    s_cselect_b32 s8, s10, s8
-; GFX90A-NEXT:    s_lshr_b32 s5, s5, 16
-; GFX90A-NEXT:    v_cvt_f32_f16_e32 v1, s5
-; GFX90A-NEXT:    s_lshl_b64 s[10:11], s[2:3], 5
-; GFX90A-NEXT:    s_lshl_b64 s[14:15], s[8:9], 5
-; GFX90A-NEXT:    s_lshl_b64 s[12:13], s[6:7], 5
+; GFX90A-NEXT:    v_cvt_f32_f16_e32 v0, s0
+; GFX90A-NEXT:    v_readfirstlane_b32 s2, v1
+; GFX90A-NEXT:    s_mul_i32 s1, s1, s2
+; GFX90A-NEXT:    s_mul_hi_u32 s1, s2, s1
+; GFX90A-NEXT:    s_add_i32 s2, s2, s1
+; GFX90A-NEXT:    s_mul_hi_u32 s1, s6, s2
+; GFX90A-NEXT:    s_mul_i32 s2, s1, s7
+; GFX90A-NEXT:    s_sub_i32 s2, s6, s2
+; GFX90A-NEXT:    s_add_i32 s3, s1, 1
+; GFX90A-NEXT:    s_sub_i32 s6, s2, s7
+; GFX90A-NEXT:    s_cmp_ge_u32 s2, s7
+; GFX90A-NEXT:    s_cselect_b32 s1, s3, s1
+; GFX90A-NEXT:    s_cselect_b32 s2, s6, s2
+; GFX90A-NEXT:    s_add_i32 s3, s1, 1
+; GFX90A-NEXT:    s_cmp_ge_u32 s2, s7
+; GFX90A-NEXT:    s_cselect_b32 s8, s3, s1
+; GFX90A-NEXT:    s_lshr_b32 s2, s0, 16
+; GFX90A-NEXT:    v_cvt_f32_f16_e32 v1, s2
+; GFX90A-NEXT:    s_lshl_b64 s[6:7], s[4:5], 5
+; GFX90A-NEXT:    s_lshl_b64 s[14:15], s[10:11], 5
 ; GFX90A-NEXT:    s_and_b64 s[0:1], exec, s[0:1]
-; GFX90A-NEXT:    s_or_b32 s12, s12, 28
+; GFX90A-NEXT:    s_or_b32 s14, s14, 28
+; GFX90A-NEXT:    s_lshl_b64 s[16:17], s[8:9], 5
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
-; GFX90A-NEXT:    v_readfirstlane_b32 s5, v18
-; GFX90A-NEXT:    s_and_b32 s5, 0xffff, s5
-; GFX90A-NEXT:    s_mul_i32 s3, s3, s5
-; GFX90A-NEXT:    s_mul_hi_u32 s9, s2, s5
-; GFX90A-NEXT:    s_mul_i32 s2, s2, s5
-; GFX90A-NEXT:    s_add_i32 s3, s9, s3
-; GFX90A-NEXT:    s_lshl_b64 s[16:17], s[2:3], 5
+; GFX90A-NEXT:    v_readfirstlane_b32 s2, v18
+; GFX90A-NEXT:    s_and_b32 s2, 0xffff, s2
+; GFX90A-NEXT:    s_mul_i32 s3, s5, s2
+; GFX90A-NEXT:    s_mul_hi_u32 s5, s4, s2
+; GFX90A-NEXT:    s_mul_i32 s2, s4, s2
+; GFX90A-NEXT:    s_add_i32 s3, s5, s3
+; GFX90A-NEXT:    s_lshl_b64 s[4:5], s[2:3], 5
 ; GFX90A-NEXT:    s_branch .LBB3_2
-; GFX90A-NEXT:  .LBB3_1: ; %Flow21
+; GFX90A-NEXT:  .LBB3_1: ; %Flow20
 ; GFX90A-NEXT:    ; in Loop: Header=BB3_2 Depth=1
 ; GFX90A-NEXT:    s_andn2_b64 vcc, exec, s[2:3]
 ; GFX90A-NEXT:    s_cbranch_vccz .LBB3_12
@@ -739,34 +735,34 @@ define amdgpu_kernel void @introduced_copy_to_sgpr(i64 %arg, i32 %arg1, i32 %arg
 ; GFX90A-NEXT:  ; %bb.3: ; %bb14
 ; GFX90A-NEXT:    ; in Loop: Header=BB3_2 Depth=1
 ; GFX90A-NEXT:    global_load_dwordx2 v[4:5], v[2:3], off
-; GFX90A-NEXT:    v_cmp_gt_i64_e64 s[2:3], s[6:7], -1
-; GFX90A-NEXT:    s_mov_b32 s5, s4
+; GFX90A-NEXT:    v_cmp_gt_i64_e64 s[2:3], s[10:11], -1
+; GFX90A-NEXT:    s_mov_b32 s13, s12
 ; GFX90A-NEXT:    v_cndmask_b32_e64 v8, 0, 1, s[2:3]
-; GFX90A-NEXT:    v_pk_mov_b32 v[6:7], s[4:5], s[4:5] op_sel:[0,1]
+; GFX90A-NEXT:    v_pk_mov_b32 v[6:7], s[12:13], s[12:13] op_sel:[0,1]
 ; GFX90A-NEXT:    v_cmp_ne_u32_e64 s[2:3], 1, v8
-; GFX90A-NEXT:    v_pk_mov_b32 v[8:9], s[4:5], s[4:5] op_sel:[0,1]
-; GFX90A-NEXT:    v_pk_mov_b32 v[10:11], s[4:5], s[4:5] op_sel:[0,1]
-; GFX90A-NEXT:    v_cmp_lt_i64_e64 s[18:19], s[6:7], 0
-; GFX90A-NEXT:    s_mov_b64 s[20:21], s[12:13]
+; GFX90A-NEXT:    v_pk_mov_b32 v[8:9], s[12:13], s[12:13] op_sel:[0,1]
+; GFX90A-NEXT:    v_pk_mov_b32 v[10:11], s[12:13], s[12:13] op_sel:[0,1]
+; GFX90A-NEXT:    v_cmp_lt_i64_e64 s[18:19], s[10:11], 0
+; GFX90A-NEXT:    s_mov_b64 s[20:21], s[14:15]
 ; GFX90A-NEXT:    v_pk_mov_b32 v[12:13], v[6:7], v[6:7] op_sel:[0,1]
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
-; GFX90A-NEXT:    v_readfirstlane_b32 s5, v4
-; GFX90A-NEXT:    v_readfirstlane_b32 s9, v5
-; GFX90A-NEXT:    s_add_u32 s5, s5, 1
-; GFX90A-NEXT:    s_addc_u32 s9, s9, 0
-; GFX90A-NEXT:    s_mul_hi_u32 s22, s10, s5
-; GFX90A-NEXT:    s_mul_i32 s9, s10, s9
-; GFX90A-NEXT:    s_mul_i32 s23, s11, s5
-; GFX90A-NEXT:    s_add_i32 s9, s22, s9
-; GFX90A-NEXT:    s_mul_i32 s5, s10, s5
-; GFX90A-NEXT:    s_add_i32 s9, s9, s23
+; GFX90A-NEXT:    v_readfirstlane_b32 s9, v4
+; GFX90A-NEXT:    v_readfirstlane_b32 s13, v5
+; GFX90A-NEXT:    s_add_u32 s9, s9, 1
+; GFX90A-NEXT:    s_addc_u32 s13, s13, 0
+; GFX90A-NEXT:    s_mul_hi_u32 s22, s6, s9
+; GFX90A-NEXT:    s_mul_i32 s13, s6, s13
+; GFX90A-NEXT:    s_mul_i32 s23, s7, s9
+; GFX90A-NEXT:    s_add_i32 s13, s22, s13
+; GFX90A-NEXT:    s_mul_i32 s9, s6, s9
+; GFX90A-NEXT:    s_add_i32 s13, s13, s23
 ; GFX90A-NEXT:    s_branch .LBB3_5
 ; GFX90A-NEXT:  .LBB3_4: ; %bb58
 ; GFX90A-NEXT:    ; in Loop: Header=BB3_5 Depth=2
 ; GFX90A-NEXT:    v_add_co_u32_sdwa v4, vcc, v4, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
 ; GFX90A-NEXT:    v_addc_co_u32_e32 v5, vcc, 0, v5, vcc
-; GFX90A-NEXT:    s_add_u32 s20, s20, s16
-; GFX90A-NEXT:    s_addc_u32 s21, s21, s17
+; GFX90A-NEXT:    s_add_u32 s20, s20, s4
+; GFX90A-NEXT:    s_addc_u32 s21, s21, s5
 ; GFX90A-NEXT:    v_cmp_lt_i64_e64 s[24:25], -1, v[4:5]
 ; GFX90A-NEXT:    s_mov_b64 s[22:23], 0
 ; GFX90A-NEXT:    s_andn2_b64 vcc, exec, s[24:25]
@@ -774,8 +770,8 @@ define amdgpu_kernel void @introduced_copy_to_sgpr(i64 %arg, i32 %arg1, i32 %arg
 ; GFX90A-NEXT:  .LBB3_5: ; %bb16
 ; GFX90A-NEXT:    ; Parent Loop BB3_2 Depth=1
 ; GFX90A-NEXT:    ; => This Inner Loop Header: Depth=2
-; GFX90A-NEXT:    s_add_u32 s22, s20, s5
-; GFX90A-NEXT:    s_addc_u32 s23, s21, s9
+; GFX90A-NEXT:    s_add_u32 s22, s20, s9
+; GFX90A-NEXT:    s_addc_u32 s23, s21, s13
 ; GFX90A-NEXT:    global_load_dword v21, v19, s[22:23] offset:-12 glc
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    global_load_dword v20, v19, s[22:23] offset:-8 glc
@@ -816,17 +812,17 @@ define amdgpu_kernel void @introduced_copy_to_sgpr(i64 %arg, i32 %arg1, i32 %arg
 ; GFX90A-NEXT:  .LBB3_9: ; %loop.exit.guard
 ; GFX90A-NEXT:    ; in Loop: Header=BB3_2 Depth=1
 ; GFX90A-NEXT:    s_xor_b64 s[18:19], s[22:23], -1
-; GFX90A-NEXT:  .LBB3_10: ; %Flow20
+; GFX90A-NEXT:  .LBB3_10: ; %Flow19
 ; GFX90A-NEXT:    ; in Loop: Header=BB3_2 Depth=1
 ; GFX90A-NEXT:    s_mov_b64 s[2:3], -1
 ; GFX90A-NEXT:    s_and_b64 vcc, exec, s[18:19]
 ; GFX90A-NEXT:    s_cbranch_vccz .LBB3_1
 ; GFX90A-NEXT:  ; %bb.11: ; %bb12
 ; GFX90A-NEXT:    ; in Loop: Header=BB3_2 Depth=1
-; GFX90A-NEXT:    s_add_u32 s6, s6, s8
-; GFX90A-NEXT:    s_addc_u32 s7, s7, 0
-; GFX90A-NEXT:    s_add_u32 s12, s12, s14
-; GFX90A-NEXT:    s_addc_u32 s13, s13, s15
+; GFX90A-NEXT:    s_add_u32 s10, s10, s8
+; GFX90A-NEXT:    s_addc_u32 s11, s11, 0
+; GFX90A-NEXT:    s_add_u32 s14, s14, s16
+; GFX90A-NEXT:    s_addc_u32 s15, s15, s17
 ; GFX90A-NEXT:    s_mov_b64 s[2:3], 0
 ; GFX90A-NEXT:    s_branch .LBB3_1
 ; GFX90A-NEXT:  .LBB3_12: ; %DummyReturnBlock
diff --git a/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-idiv.ll b/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-idiv.ll
index 08ce28c12118b..df77e7de43bf6 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-idiv.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-idiv.ll
@@ -2543,45 +2543,44 @@ define amdgpu_kernel void @udiv_v4i16(ptr addrspace(1) %out, <4 x i16> %x, <4 x
 ;
 ; GFX6-LABEL: udiv_v4i16:
 ; GFX6:       ; %bb.0:
-; GFX6-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0xd
-; GFX6-NEXT:    s_load_dwordx4 s[4:7], s[4:5], 0x9
+; GFX6-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0xb
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x9
 ; GFX6-NEXT:    s_mov_b32 s3, 0xf000
 ; GFX6-NEXT:    s_mov_b32 s2, -1
 ; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX6-NEXT:    s_and_b32 s0, s8, 0xffff
-; GFX6-NEXT:    v_cvt_f32_u32_e32 v0, s0
-; GFX6-NEXT:    s_mov_b32 s0, s4
-; GFX6-NEXT:    s_and_b32 s1, s6, 0xffff
+; GFX6-NEXT:    s_and_b32 s5, s10, 0xffff
+; GFX6-NEXT:    v_cvt_f32_u32_e32 v0, s5
+; GFX6-NEXT:    s_lshr_b32 s5, s10, 16
+; GFX6-NEXT:    s_and_b32 s4, s8, 0xffff
+; GFX6-NEXT:    v_cvt_f32_u32_e32 v2, s5
+; GFX6-NEXT:    v_cvt_f32_u32_e32 v1, s4
+; GFX6-NEXT:    v_rcp_iflag_f32_e32 v3, v0
 ; GFX6-NEXT:    s_lshr_b32 s4, s8, 16
-; GFX6-NEXT:    v_cvt_f32_u32_e32 v1, s1
-; GFX6-NEXT:    v_rcp_iflag_f32_e32 v2, v0
-; GFX6-NEXT:    v_cvt_f32_u32_e32 v3, s4
-; GFX6-NEXT:    s_lshr_b32 s4, s6, 16
 ; GFX6-NEXT:    v_cvt_f32_u32_e32 v4, s4
-; GFX6-NEXT:    v_mul_f32_e32 v2, v1, v2
-; GFX6-NEXT:    v_rcp_iflag_f32_e32 v5, v3
-; GFX6-NEXT:    v_trunc_f32_e32 v2, v2
-; GFX6-NEXT:    v_mad_f32 v1, -v2, v0, v1
-; GFX6-NEXT:    v_cvt_u32_f32_e32 v2, v2
+; GFX6-NEXT:    v_rcp_iflag_f32_e32 v5, v2
+; GFX6-NEXT:    v_mul_f32_e32 v3, v1, v3
+; GFX6-NEXT:    v_trunc_f32_e32 v3, v3
+; GFX6-NEXT:    v_mad_f32 v1, -v3, v0, v1
 ; GFX6-NEXT:    v_cmp_ge_f32_e64 vcc, |v1|, v0
 ; GFX6-NEXT:    v_mul_f32_e32 v1, v4, v5
 ; GFX6-NEXT:    v_trunc_f32_e32 v1, v1
-; GFX6-NEXT:    s_and_b32 s4, s9, 0xffff
-; GFX6-NEXT:    v_addc_u32_e32 v0, vcc, 0, v2, vcc
-; GFX6-NEXT:    v_mad_f32 v2, -v1, v3, v4
+; GFX6-NEXT:    s_and_b32 s4, s11, 0xffff
+; GFX6-NEXT:    v_cvt_u32_f32_e32 v6, v3
+; GFX6-NEXT:    v_mad_f32 v3, -v1, v2, v4
 ; GFX6-NEXT:    v_cvt_f32_u32_e32 v4, s4
-; GFX6-NEXT:    s_and_b32 s4, s7, 0xffff
+; GFX6-NEXT:    s_and_b32 s4, s9, 0xffff
+; GFX6-NEXT:    v_addc_u32_e32 v0, vcc, 0, v6, vcc
 ; GFX6-NEXT:    v_cvt_u32_f32_e32 v1, v1
 ; GFX6-NEXT:    v_cvt_f32_u32_e32 v5, s4
 ; GFX6-NEXT:    v_rcp_iflag_f32_e32 v6, v4
-; GFX6-NEXT:    v_cmp_ge_f32_e64 vcc, |v2|, v3
+; GFX6-NEXT:    v_cmp_ge_f32_e64 vcc, |v3|, v2
 ; GFX6-NEXT:    v_addc_u32_e32 v2, vcc, 0, v1, vcc
 ; GFX6-NEXT:    v_mul_f32_e32 v1, v5, v6
 ; GFX6-NEXT:    v_trunc_f32_e32 v1, v1
-; GFX6-NEXT:    s_lshr_b32 s4, s9, 16
+; GFX6-NEXT:    s_lshr_b32 s4, s11, 16
 ; GFX6-NEXT:    v_mad_f32 v3, -v1, v4, v5
 ; GFX6-NEXT:    v_cvt_f32_u32_e32 v5, s4
-; GFX6-NEXT:    s_lshr_b32 s4, s7, 16
+; GFX6-NEXT:    s_lshr_b32 s4, s9, 16
 ; GFX6-NEXT:    v_cvt_f32_u32_e32 v6, s4
 ; GFX6-NEXT:    v_cvt_u32_f32_e32 v1, v1
 ; GFX6-NEXT:    v_rcp_iflag_f32_e32 v7, v5
@@ -2597,7 +2596,6 @@ define amdgpu_kernel void @udiv_v4i16(ptr addrspace(1) %out, <4 x i16> %x, <4 x
 ; GFX6-NEXT:    v_addc_u32_e32 v3, vcc, 0, v4, vcc
 ; GFX6-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
 ; GFX6-NEXT:    v_and_b32_e32 v0, 0xffff, v0
-; GFX6-NEXT:    s_mov_b32 s1, s5
 ; GFX6-NEXT:    v_or_b32_e32 v1, v1, v3
 ; GFX6-NEXT:    v_or_b32_e32 v0, v0, v2
 ; GFX6-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
@@ -2605,43 +2603,42 @@ define amdgpu_kernel void @udiv_v4i16(ptr addrspace(1) %out, <4 x i16> %x, <4 x
 ;
 ; GFX9-LABEL: udiv_v4i16:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX9-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x2c
 ; GFX9-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    s_and_b32 s4, s2, 0xffff
-; GFX9-NEXT:    s_and_b32 s5, s6, 0xffff
-; GFX9-NEXT:    v_cvt_f32_u32_e32 v0, s5
-; GFX9-NEXT:    s_lshr_b32 s6, s6, 16
-; GFX9-NEXT:    v_cvt_f32_u32_e32 v2, s4
-; GFX9-NEXT:    v_cvt_f32_u32_e32 v1, s6
-; GFX9-NEXT:    v_rcp_iflag_f32_e32 v4, v0
+; GFX9-NEXT:    s_and_b32 s7, s2, 0xffff
+; GFX9-NEXT:    v_cvt_f32_u32_e32 v0, s7
+; GFX9-NEXT:    s_and_b32 s6, s0, 0xffff
 ; GFX9-NEXT:    s_lshr_b32 s2, s2, 16
-; GFX9-NEXT:    v_cvt_f32_u32_e32 v3, s2
-; GFX9-NEXT:    v_rcp_iflag_f32_e32 v5, v1
+; GFX9-NEXT:    v_cvt_f32_u32_e32 v2, s6
+; GFX9-NEXT:    v_rcp_iflag_f32_e32 v4, v0
+; GFX9-NEXT:    v_cvt_f32_u32_e32 v1, s2
+; GFX9-NEXT:    s_lshr_b32 s0, s0, 16
+; GFX9-NEXT:    v_cvt_f32_u32_e32 v3, s0
 ; GFX9-NEXT:    v_mul_f32_e32 v4, v2, v4
+; GFX9-NEXT:    v_rcp_iflag_f32_e32 v5, v1
 ; GFX9-NEXT:    v_trunc_f32_e32 v4, v4
-; GFX9-NEXT:    s_and_b32 s2, s7, 0xffff
+; GFX9-NEXT:    s_and_b32 s0, s3, 0xffff
 ; GFX9-NEXT:    v_cvt_u32_f32_e32 v7, v4
 ; GFX9-NEXT:    v_mad_f32 v2, -v4, v0, v2
-; GFX9-NEXT:    v_cvt_f32_u32_e32 v4, s2
+; GFX9-NEXT:    v_cvt_f32_u32_e32 v4, s0
 ; GFX9-NEXT:    v_mul_f32_e32 v5, v3, v5
 ; GFX9-NEXT:    v_cmp_ge_f32_e64 vcc, |v2|, v0
-; GFX9-NEXT:    s_and_b32 s2, s3, 0xffff
+; GFX9-NEXT:    s_and_b32 s0, s1, 0xffff
 ; GFX9-NEXT:    v_addc_co_u32_e32 v0, vcc, 0, v7, vcc
 ; GFX9-NEXT:    v_trunc_f32_e32 v2, v5
-; GFX9-NEXT:    v_cvt_f32_u32_e32 v5, s2
+; GFX9-NEXT:    v_cvt_f32_u32_e32 v5, s0
 ; GFX9-NEXT:    v_rcp_iflag_f32_e32 v7, v4
 ; GFX9-NEXT:    v_mad_f32 v3, -v2, v1, v3
 ; GFX9-NEXT:    v_cmp_ge_f32_e64 vcc, |v3|, v1
-; GFX9-NEXT:    s_lshr_b32 s2, s7, 16
+; GFX9-NEXT:    s_lshr_b32 s0, s3, 16
 ; GFX9-NEXT:    v_mul_f32_e32 v1, v5, v7
 ; GFX9-NEXT:    v_trunc_f32_e32 v1, v1
 ; GFX9-NEXT:    v_mad_f32 v3, -v1, v4, v5
-; GFX9-NEXT:    v_cvt_f32_u32_e32 v5, s2
+; GFX9-NEXT:    v_cvt_f32_u32_e32 v5, s0
 ; GFX9-NEXT:    v_cvt_u32_f32_e32 v2, v2
-; GFX9-NEXT:    s_lshr_b32 s2, s3, 16
-; GFX9-NEXT:    v_cvt_f32_u32_e32 v7, s2
+; GFX9-NEXT:    s_lshr_b32 s0, s1, 16
+; GFX9-NEXT:    v_cvt_f32_u32_e32 v7, s0
 ; GFX9-NEXT:    v_rcp_iflag_f32_e32 v8, v5
 ; GFX9-NEXT:    v_addc_co_u32_e32 v2, vcc, 0, v2, vcc
 ; GFX9-NEXT:    v_cvt_u32_f32_e32 v1, v1
@@ -2649,6 +2646,7 @@ define amdgpu_kernel void @udiv_v4i16(ptr addrspace(1) %out, <4 x i16> %x, <4 x
 ; GFX9-NEXT:    v_mul_f32_e32 v3, v7, v8
 ; GFX9-NEXT:    v_trunc_f32_e32 v3, v3
 ; GFX9-NEXT:    v_cvt_u32_f32_e32 v4, v3
+; GFX9-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x24
 ; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
 ; GFX9-NEXT:    v_mad_f32 v3, -v3, v5, v7
 ; GFX9-NEXT:    v_cmp_ge_f32_e64 vcc, |v3|, v5
@@ -2657,7 +2655,8 @@ define amdgpu_kernel void @udiv_v4i16(ptr addrspace(1) %out, <4 x i16> %x, <4 x
 ; GFX9-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX9-NEXT:    v_lshl_or_b32 v1, v3, 16, v1
 ; GFX9-NEXT:    v_lshl_or_b32 v0, v2, 16, v0
-; GFX9-NEXT:    global_store_dwordx2 v6, v[0:1], s[0:1]
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    global_store_dwordx2 v6, v[0:1], s[6:7]
 ; GFX9-NEXT:    s_endpgm
   %r = udiv <4 x i16> %x, %y
   store <4 x i16> %r, ptr addrspace(1) %out
@@ -2759,51 +2758,49 @@ define amdgpu_kernel void @urem_v4i16(ptr addrspace(1) %out, <4 x i16> %x, <4 x
 ;
 ; GFX6-LABEL: urem_v4i16:
 ; GFX6:       ; %bb.0:
-; GFX6-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0xd
-; GFX6-NEXT:    s_load_dwordx4 s[4:7], s[4:5], 0x9
+; GFX6-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0xb
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x9
 ; GFX6-NEXT:    s_mov_b32 s3, 0xf000
 ; GFX6-NEXT:    s_mov_b32 s2, -1
 ; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX6-NEXT:    s_and_b32 s0, s8, 0xffff
-; GFX6-NEXT:    v_cvt_f32_u32_e32 v0, s0
-; GFX6-NEXT:    s_mov_b32 s0, s4
-; GFX6-NEXT:    s_and_b32 s1, s6, 0xffff
+; GFX6-NEXT:    s_and_b32 s5, s10, 0xffff
+; GFX6-NEXT:    v_cvt_f32_u32_e32 v0, s5
+; GFX6-NEXT:    s_lshr_b32 s5, s10, 16
+; GFX6-NEXT:    s_and_b32 s4, s8, 0xffff
+; GFX6-NEXT:    v_cvt_f32_u32_e32 v2, s5
+; GFX6-NEXT:    v_cvt_f32_u32_e32 v1, s4
+; GFX6-NEXT:    v_rcp_iflag_f32_e32 v3, v0
 ; GFX6-NEXT:    s_lshr_b32 s4, s8, 16
-; GFX6-NEXT:    v_cvt_f32_u32_e32 v1, s1
-; GFX6-NEXT:    v_rcp_iflag_f32_e32 v2, v0
-; GFX6-NEXT:    v_cvt_f32_u32_e32 v3, s4
-; GFX6-NEXT:    s_mov_b32 s1, s5
-; GFX6-NEXT:    s_lshr_b32 s5, s6, 16
-; GFX6-NEXT:    v_mul_f32_e32 v2, v1, v2
-; GFX6-NEXT:    v_cvt_f32_u32_e32 v4, s5
-; GFX6-NEXT:    v_rcp_iflag_f32_e32 v5, v3
-; GFX6-NEXT:    v_trunc_f32_e32 v2, v2
-; GFX6-NEXT:    v_mad_f32 v1, -v2, v0, v1
-; GFX6-NEXT:    v_cvt_u32_f32_e32 v2, v2
+; GFX6-NEXT:    v_cvt_f32_u32_e32 v4, s4
+; GFX6-NEXT:    v_rcp_iflag_f32_e32 v5, v2
+; GFX6-NEXT:    v_mul_f32_e32 v3, v1, v3
+; GFX6-NEXT:    v_trunc_f32_e32 v3, v3
+; GFX6-NEXT:    v_cvt_u32_f32_e32 v6, v3
+; GFX6-NEXT:    v_mad_f32 v1, -v3, v0, v1
 ; GFX6-NEXT:    v_cmp_ge_f32_e64 vcc, |v1|, v0
 ; GFX6-NEXT:    v_mul_f32_e32 v1, v4, v5
 ; GFX6-NEXT:    v_trunc_f32_e32 v1, v1
-; GFX6-NEXT:    v_addc_u32_e32 v0, vcc, 0, v2, vcc
-; GFX6-NEXT:    v_cvt_u32_f32_e32 v2, v1
-; GFX6-NEXT:    v_mad_f32 v1, -v1, v3, v4
-; GFX6-NEXT:    v_mul_lo_u32 v0, v0, s8
-; GFX6-NEXT:    v_cmp_ge_f32_e64 vcc, |v1|, v3
-; GFX6-NEXT:    s_and_b32 s8, s9, 0xffff
-; GFX6-NEXT:    v_addc_u32_e32 v1, vcc, 0, v2, vcc
-; GFX6-NEXT:    v_cvt_f32_u32_e32 v2, s8
-; GFX6-NEXT:    v_mul_lo_u32 v1, v1, s4
-; GFX6-NEXT:    s_and_b32 s4, s7, 0xffff
-; GFX6-NEXT:    v_cvt_f32_u32_e32 v3, s4
+; GFX6-NEXT:    v_cvt_u32_f32_e32 v3, v1
+; GFX6-NEXT:    v_addc_u32_e32 v0, vcc, 0, v6, vcc
+; GFX6-NEXT:    v_mad_f32 v1, -v1, v2, v4
+; GFX6-NEXT:    s_and_b32 s6, s11, 0xffff
+; GFX6-NEXT:    v_cmp_ge_f32_e64 vcc, |v1|, v2
+; GFX6-NEXT:    v_cvt_f32_u32_e32 v2, s6
+; GFX6-NEXT:    v_addc_u32_e32 v1, vcc, 0, v3, vcc
+; GFX6-NEXT:    v_mul_lo_u32 v1, v1, s5
+; GFX6-NEXT:    s_and_b32 s5, s9, 0xffff
+; GFX6-NEXT:    v_cvt_f32_u32_e32 v3, s5
 ; GFX6-NEXT:    v_rcp_iflag_f32_e32 v4, v2
-; GFX6-NEXT:    s_lshr_b32 s4, s9, 16
-; GFX6-NEXT:    v_sub_i32_e32 v5, vcc, s5, v1
+; GFX6-NEXT:    v_sub_i32_e32 v5, vcc, s4, v1
+; GFX6-NEXT:    s_lshr_b32 s4, s11, 16
 ; GFX6-NEXT:    v_mul_f32_e32 v1, v3, v4
 ; GFX6-NEXT:    v_cvt_f32_u32_e32 v4, s4
-; GFX6-NEXT:    s_lshr_b32 s5, s7, 16
+; GFX6-NEXT:    v_mul_lo_u32 v0, v0, s10
+; GFX6-NEXT:    s_lshr_b32 s5, s9, 16
 ; GFX6-NEXT:    v_cvt_f32_u32_e32 v6, s5
-; GFX6-NEXT:    v_trunc_f32_e32 v1, v1
 ; GFX6-NEXT:    v_rcp_iflag_f32_e32 v7, v4
-; GFX6-NEXT:    v_sub_i32_e32 v0, vcc, s6, v0
+; GFX6-NEXT:    v_trunc_f32_e32 v1, v1
+; GFX6-NEXT:    v_sub_i32_e32 v0, vcc, s8, v0
 ; GFX6-NEXT:    v_mad_f32 v3, -v1, v2, v3
 ; GFX6-NEXT:    v_cvt_u32_f32_e32 v1, v1
 ; GFX6-NEXT:    v_cmp_ge_f32_e64 vcc, |v3|, v2
@@ -2814,10 +2811,10 @@ define amdgpu_kernel void @urem_v4i16(ptr addrspace(1) %out, <4 x i16> %x, <4 x
 ; GFX6-NEXT:    v_mad_f32 v2, -v2, v4, v6
 ; GFX6-NEXT:    v_cmp_ge_f32_e64 vcc, |v2|, v4
 ; GFX6-NEXT:    v_addc_u32_e32 v2, vcc, 0, v3, vcc
-; GFX6-NEXT:    v_mul_lo_u32 v1, v1, s9
+; GFX6-NEXT:    v_mul_lo_u32 v1, v1, s11
 ; GFX6-NEXT:    v_mul_lo_u32 v2, v2, s4
 ; GFX6-NEXT:    v_and_b32_e32 v0, 0xffff, v0
-; GFX6-NEXT:    v_sub_i32_e32 v1, vcc, s7, v1
+; GFX6-NEXT:    v_sub_i32_e32 v1, vcc, s9, v1
 ; GFX6-NEXT:    v_sub_i32_e32 v2, vcc, s5, v2
 ; GFX6-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
 ; GFX6-NEXT:    v_and_b32_e32 v1, 0xffff, v1
@@ -2829,67 +2826,67 @@ define amdgpu_kernel void @urem_v4i16(ptr addrspace(1) %out, <4 x i16> %x, <4 x
 ;
 ; GFX9-LABEL: urem_v4i16:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX9-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x2c
+; GFX9-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x24
 ; GFX9-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    s_and_b32 s4, s2, 0xffff
-; GFX9-NEXT:    s_and_b32 s5, s6, 0xffff
-; GFX9-NEXT:    v_cvt_f32_u32_e32 v0, s5
-; GFX9-NEXT:    v_cvt_f32_u32_e32 v2, s4
-; GFX9-NEXT:    s_lshr_b32 s6, s6, 16
-; GFX9-NEXT:    v_cvt_f32_u32_e32 v1, s6
-; GFX9-NEXT:    v_rcp_iflag_f32_e32 v4, v0
+; GFX9-NEXT:    s_and_b32 s9, s2, 0xffff
+; GFX9-NEXT:    v_cvt_f32_u32_e32 v0, s9
+; GFX9-NEXT:    s_and_b32 s8, s0, 0xffff
 ; GFX9-NEXT:    s_lshr_b32 s2, s2, 16
-; GFX9-NEXT:    v_cvt_f32_u32_e32 v3, s2
+; GFX9-NEXT:    v_cvt_f32_u32_e32 v1, s2
+; GFX9-NEXT:    v_cvt_f32_u32_e32 v2, s8
+; GFX9-NEXT:    v_rcp_iflag_f32_e32 v4, v0
+; GFX9-NEXT:    s_lshr_b32 s0, s0, 16
+; GFX9-NEXT:    v_cvt_f32_u32_e32 v3, s0
 ; GFX9-NEXT:    v_rcp_iflag_f32_e32 v5, v1
 ; GFX9-NEXT:    v_mul_f32_e32 v4, v2, v4
 ; GFX9-NEXT:    v_trunc_f32_e32 v4, v4
+; GFX9-NEXT:    s_and_b32 s4, s3, 0xffff
 ; GFX9-NEXT:    v_cvt_u32_f32_e32 v7, v4
 ; GFX9-NEXT:    v_mad_f32 v2, -v4, v0, v2
-; GFX9-NEXT:    v_cmp_ge_f32_e64 vcc, |v2|, v0
+; GFX9-NEXT:    v_cvt_f32_u32_e32 v4, s4
 ; GFX9-NEXT:    v_mul_f32_e32 v5, v3, v5
-; GFX9-NEXT:    v_addc_co_u32_e32 v0, vcc, 0, v7, vcc
-; GFX9-NEXT:    v_mul_lo_u32 v0, v0, s5
-; GFX9-NEXT:    s_and_b32 s5, s7, 0xffff
-; GFX9-NEXT:    v_cvt_f32_u32_e32 v4, s5
+; GFX9-NEXT:    v_cmp_ge_f32_e64 vcc, |v2|, v0
 ; GFX9-NEXT:    v_trunc_f32_e32 v2, v5
-; GFX9-NEXT:    s_and_b32 s8, s3, 0xffff
+; GFX9-NEXT:    s_and_b32 s5, s1, 0xffff
+; GFX9-NEXT:    v_addc_co_u32_e32 v0, vcc, 0, v7, vcc
 ; GFX9-NEXT:    v_mad_f32 v3, -v2, v1, v3
 ; GFX9-NEXT:    v_cvt_u32_f32_e32 v2, v2
-; GFX9-NEXT:    v_cvt_f32_u32_e32 v5, s8
+; GFX9-NEXT:    v_cvt_f32_u32_e32 v5, s5
 ; GFX9-NEXT:    v_rcp_iflag_f32_e32 v7, v4
 ; GFX9-NEXT:    v_cmp_ge_f32_e64 vcc, |v3|, v1
 ; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v2, vcc
 ; GFX9-NEXT:    v_mul_f32_e32 v2, v5, v7
-; GFX9-NEXT:    v_mul_lo_u32 v1, v1, s6
+; GFX9-NEXT:    v_mul_lo_u32 v1, v1, s2
 ; GFX9-NEXT:    v_trunc_f32_e32 v2, v2
-; GFX9-NEXT:    s_lshr_b32 s6, s7, 16
+; GFX9-NEXT:    s_lshr_b32 s2, s3, 16
 ; GFX9-NEXT:    v_mad_f32 v3, -v2, v4, v5
-; GFX9-NEXT:    v_cvt_f32_u32_e32 v5, s6
-; GFX9-NEXT:    s_lshr_b32 s3, s3, 16
-; GFX9-NEXT:    v_cvt_f32_u32_e32 v7, s3
+; GFX9-NEXT:    v_cvt_f32_u32_e32 v5, s2
+; GFX9-NEXT:    s_lshr_b32 s1, s1, 16
+; GFX9-NEXT:    v_cvt_f32_u32_e32 v7, s1
 ; GFX9-NEXT:    v_cvt_u32_f32_e32 v2, v2
 ; GFX9-NEXT:    v_rcp_iflag_f32_e32 v8, v5
 ; GFX9-NEXT:    v_cmp_ge_f32_e64 vcc, |v3|, v4
-; GFX9-NEXT:    v_sub_u32_e32 v0, s4, v0
+; GFX9-NEXT:    v_mul_lo_u32 v0, v0, s9
 ; GFX9-NEXT:    v_addc_co_u32_e32 v2, vcc, 0, v2, vcc
 ; GFX9-NEXT:    v_mul_f32_e32 v3, v7, v8
 ; GFX9-NEXT:    v_trunc_f32_e32 v3, v3
 ; GFX9-NEXT:    v_cvt_u32_f32_e32 v4, v3
 ; GFX9-NEXT:    v_mad_f32 v3, -v3, v5, v7
 ; GFX9-NEXT:    v_cmp_ge_f32_e64 vcc, |v3|, v5
-; GFX9-NEXT:    v_mul_lo_u32 v2, v2, s5
+; GFX9-NEXT:    v_mul_lo_u32 v2, v2, s4
 ; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, 0, v4, vcc
-; GFX9-NEXT:    v_mul_lo_u32 v3, v3, s6
-; GFX9-NEXT:    v_sub_u32_e32 v4, s2, v1
-; GFX9-NEXT:    v_sub_u32_e32 v1, s8, v2
+; GFX9-NEXT:    v_mul_lo_u32 v3, v3, s2
+; GFX9-NEXT:    v_sub_u32_e32 v0, s8, v0
+; GFX9-NEXT:    v_sub_u32_e32 v4, s0, v1
+; GFX9-NEXT:    v_sub_u32_e32 v1, s5, v2
+; GFX9-NEXT:    v_sub_u32_e32 v2, s1, v3
 ; GFX9-NEXT:    v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NEXT:    v_sub_u32_e32 v2, s3, v3
 ; GFX9-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX9-NEXT:    v_lshl_or_b32 v1, v2, 16, v1
 ; GFX9-NEXT:    v_lshl_or_b32 v0, v4, 16, v0
-; GFX9-NEXT:    global_store_dwordx2 v6, v[0:1], s[0:1]
+; GFX9-NEXT:    global_store_dwordx2 v6, v[0:1], s[6:7]
 ; GFX9-NEXT:    s_endpgm
   %r = urem <4 x i16> %x, %y
   store <4 x i16> %r, ptr addrspace(1) %out
@@ -2999,64 +2996,62 @@ define amdgpu_kernel void @sdiv_v4i16(ptr addrspace(1) %out, <4 x i16> %x, <4 x
 ;
 ; GFX6-LABEL: sdiv_v4i16:
 ; GFX6:       ; %bb.0:
-; GFX6-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x9
-; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xd
+; GFX6-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0xb
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x9
 ; GFX6-NEXT:    s_mov_b32 s3, 0xf000
 ; GFX6-NEXT:    s_mov_b32 s2, -1
 ; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX6-NEXT:    s_sext_i32_i16 s7, s10
-; GFX6-NEXT:    s_sext_i32_i16 s6, s4
-; GFX6-NEXT:    v_cvt_f32_i32_e32 v0, s6
-; GFX6-NEXT:    v_cvt_f32_i32_e32 v1, s7
-; GFX6-NEXT:    s_xor_b32 s6, s7, s6
-; GFX6-NEXT:    s_ashr_i32 s6, s6, 30
+; GFX6-NEXT:    s_sext_i32_i16 s4, s10
+; GFX6-NEXT:    v_cvt_f32_i32_e32 v0, s4
+; GFX6-NEXT:    s_sext_i32_i16 s5, s8
+; GFX6-NEXT:    v_cvt_f32_i32_e32 v1, s5
+; GFX6-NEXT:    s_xor_b32 s4, s5, s4
 ; GFX6-NEXT:    v_rcp_iflag_f32_e32 v2, v0
-; GFX6-NEXT:    s_mov_b32 s0, s8
-; GFX6-NEXT:    s_or_b32 s8, s6, 1
-; GFX6-NEXT:    s_mov_b32 s1, s9
+; GFX6-NEXT:    s_ashr_i32 s4, s4, 30
+; GFX6-NEXT:    s_or_b32 s6, s4, 1
 ; GFX6-NEXT:    v_mul_f32_e32 v2, v1, v2
 ; GFX6-NEXT:    v_trunc_f32_e32 v2, v2
 ; GFX6-NEXT:    v_mad_f32 v1, -v2, v0, v1
-; GFX6-NEXT:    v_cmp_ge_f32_e64 s[6:7], |v1|, |v0|
-; GFX6-NEXT:    s_and_b64 s[6:7], s[6:7], exec
+; GFX6-NEXT:    v_cmp_ge_f32_e64 s[4:5], |v1|, |v0|
+; GFX6-NEXT:    s_and_b64 s[4:5], s[4:5], exec
 ; GFX6-NEXT:    v_cvt_i32_f32_e32 v2, v2
-; GFX6-NEXT:    s_cselect_b32 s6, s8, 0
-; GFX6-NEXT:    s_ashr_i32 s4, s4, 16
-; GFX6-NEXT:    v_cvt_f32_i32_e32 v0, s4
-; GFX6-NEXT:    v_add_i32_e32 v2, vcc, s6, v2
-; GFX6-NEXT:    s_ashr_i32 s6, s10, 16
-; GFX6-NEXT:    v_cvt_f32_i32_e32 v1, s6
+; GFX6-NEXT:    s_cselect_b32 s4, s6, 0
+; GFX6-NEXT:    s_ashr_i32 s5, s10, 16
+; GFX6-NEXT:    v_cvt_f32_i32_e32 v0, s5
+; GFX6-NEXT:    v_add_i32_e32 v2, vcc, s4, v2
+; GFX6-NEXT:    s_ashr_i32 s4, s8, 16
+; GFX6-NEXT:    v_cvt_f32_i32_e32 v1, s4
 ; GFX6-NEXT:    v_rcp_iflag_f32_e32 v3, v0
-; GFX6-NEXT:    s_xor_b32 s4, s6, s4
+; GFX6-NEXT:    s_xor_b32 s4, s4, s5
 ; GFX6-NEXT:    s_ashr_i32 s4, s4, 30
-; GFX6-NEXT:    s_or_b32 s4, s4, 1
+; GFX6-NEXT:    s_or_b32 s6, s4, 1
 ; GFX6-NEXT:    v_mul_f32_e32 v3, v1, v3
 ; GFX6-NEXT:    v_trunc_f32_e32 v3, v3
 ; GFX6-NEXT:    v_mad_f32 v1, -v3, v0, v1
-; GFX6-NEXT:    v_cmp_ge_f32_e64 s[6:7], |v1|, |v0|
-; GFX6-NEXT:    s_and_b64 s[6:7], s[6:7], exec
+; GFX6-NEXT:    v_cmp_ge_f32_e64 s[4:5], |v1|, |v0|
+; GFX6-NEXT:    s_and_b64 s[4:5], s[4:5], exec
 ; GFX6-NEXT:    v_cvt_i32_f32_e32 v3, v3
-; GFX6-NEXT:    s_sext_i32_i16 s6, s5
-; GFX6-NEXT:    v_cvt_f32_i32_e32 v0, s6
-; GFX6-NEXT:    s_cselect_b32 s4, s4, 0
+; GFX6-NEXT:    s_sext_i32_i16 s5, s11
+; GFX6-NEXT:    v_cvt_f32_i32_e32 v0, s5
+; GFX6-NEXT:    s_cselect_b32 s4, s6, 0
 ; GFX6-NEXT:    v_add_i32_e32 v3, vcc, s4, v3
-; GFX6-NEXT:    s_sext_i32_i16 s4, s11
+; GFX6-NEXT:    s_sext_i32_i16 s4, s9
 ; GFX6-NEXT:    v_cvt_f32_i32_e32 v1, s4
 ; GFX6-NEXT:    v_rcp_iflag_f32_e32 v4, v0
-; GFX6-NEXT:    s_xor_b32 s4, s4, s6
+; GFX6-NEXT:    s_xor_b32 s4, s4, s5
 ; GFX6-NEXT:    s_ashr_i32 s4, s4, 30
-; GFX6-NEXT:    s_or_b32 s4, s4, 1
+; GFX6-NEXT:    s_or_b32 s6, s4, 1
 ; GFX6-NEXT:    v_mul_f32_e32 v4, v1, v4
 ; GFX6-NEXT:    v_trunc_f32_e32 v4, v4
 ; GFX6-NEXT:    v_mad_f32 v1, -v4, v0, v1
-; GFX6-NEXT:    v_cmp_ge_f32_e64 s[6:7], |v1|, |v0|
-; GFX6-NEXT:    s_and_b64 s[6:7], s[6:7], exec
+; GFX6-NEXT:    v_cmp_ge_f32_e64 s[4:5], |v1|, |v0|
+; GFX6-NEXT:    s_and_b64 s[4:5], s[4:5], exec
 ; GFX6-NEXT:    v_cvt_i32_f32_e32 v4, v4
-; GFX6-NEXT:    s_cselect_b32 s4, s4, 0
-; GFX6-NEXT:    s_ashr_i32 s5, s5, 16
+; GFX6-NEXT:    s_cselect_b32 s4, s6, 0
+; GFX6-NEXT:    s_ashr_i32 s5, s11, 16
 ; GFX6-NEXT:    v_cvt_f32_i32_e32 v0, s5
 ; GFX6-NEXT:    v_add_i32_e32 v1, vcc, s4, v4
-; GFX6-NEXT:    s_ashr_i32 s4, s11, 16
+; GFX6-NEXT:    s_ashr_i32 s4, s9, 16
 ; GFX6-NEXT:    v_cvt_f32_i32_e32 v4, s4
 ; GFX6-NEXT:    v_rcp_iflag_f32_e32 v5, v0
 ; GFX6-NEXT:    s_xor_b32 s4, s4, s5
@@ -3081,13 +3076,13 @@ define amdgpu_kernel void @sdiv_v4i16(ptr addrspace(1) %out, <4 x i16> %x, <4 x
 ;
 ; GFX9-LABEL: sdiv_v4i16:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
-; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x2c
+; GFX9-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x24
 ; GFX9-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    s_sext_i32_i16 s4, s6
+; GFX9-NEXT:    s_sext_i32_i16 s4, s2
 ; GFX9-NEXT:    v_cvt_f32_i32_e32 v0, s4
-; GFX9-NEXT:    s_sext_i32_i16 s5, s2
+; GFX9-NEXT:    s_sext_i32_i16 s5, s0
 ; GFX9-NEXT:    v_cvt_f32_i32_e32 v1, s5
 ; GFX9-NEXT:    s_xor_b32 s4, s5, s4
 ; GFX9-NEXT:    v_rcp_iflag_f32_e32 v3, v0
@@ -3099,61 +3094,61 @@ define amdgpu_kernel void @sdiv_v4i16(ptr addrspace(1) %out, <4 x i16> %x, <4 x
 ; GFX9-NEXT:    v_cmp_ge_f32_e64 s[4:5], |v1|, |v0|
 ; GFX9-NEXT:    s_and_b64 s[4:5], s[4:5], exec
 ; GFX9-NEXT:    s_cselect_b32 s4, s8, 0
-; GFX9-NEXT:    s_ashr_i32 s5, s6, 16
-; GFX9-NEXT:    v_cvt_f32_i32_e32 v0, s5
 ; GFX9-NEXT:    s_ashr_i32 s2, s2, 16
-; GFX9-NEXT:    v_cvt_f32_i32_e32 v1, s2
+; GFX9-NEXT:    v_cvt_f32_i32_e32 v0, s2
+; GFX9-NEXT:    s_ashr_i32 s0, s0, 16
+; GFX9-NEXT:    v_cvt_f32_i32_e32 v1, s0
 ; GFX9-NEXT:    v_cvt_i32_f32_e32 v3, v3
 ; GFX9-NEXT:    v_rcp_iflag_f32_e32 v4, v0
-; GFX9-NEXT:    s_xor_b32 s2, s2, s5
-; GFX9-NEXT:    s_ashr_i32 s2, s2, 30
-; GFX9-NEXT:    v_add_u32_e32 v3, s4, v3
+; GFX9-NEXT:    s_xor_b32 s0, s0, s2
+; GFX9-NEXT:    s_ashr_i32 s0, s0, 30
+; GFX9-NEXT:    s_sext_i32_i16 s2, s3
 ; GFX9-NEXT:    v_mul_f32_e32 v4, v1, v4
 ; GFX9-NEXT:    v_trunc_f32_e32 v4, v4
 ; GFX9-NEXT:    v_mad_f32 v1, -v4, v0, v1
-; GFX9-NEXT:    s_or_b32 s2, s2, 1
+; GFX9-NEXT:    v_cvt_i32_f32_e32 v4, v4
+; GFX9-NEXT:    v_add_u32_e32 v3, s4, v3
+; GFX9-NEXT:    s_or_b32 s0, s0, 1
 ; GFX9-NEXT:    v_cmp_ge_f32_e64 s[4:5], |v1|, |v0|
+; GFX9-NEXT:    v_cvt_f32_i32_e32 v0, s2
 ; GFX9-NEXT:    s_and_b64 s[4:5], s[4:5], exec
-; GFX9-NEXT:    v_cvt_i32_f32_e32 v4, v4
-; GFX9-NEXT:    s_sext_i32_i16 s4, s7
-; GFX9-NEXT:    v_cvt_f32_i32_e32 v0, s4
-; GFX9-NEXT:    s_cselect_b32 s2, s2, 0
-; GFX9-NEXT:    v_add_u32_e32 v4, s2, v4
-; GFX9-NEXT:    s_sext_i32_i16 s2, s3
-; GFX9-NEXT:    v_cvt_f32_i32_e32 v1, s2
+; GFX9-NEXT:    s_cselect_b32 s0, s0, 0
+; GFX9-NEXT:    v_add_u32_e32 v4, s0, v4
+; GFX9-NEXT:    s_sext_i32_i16 s0, s1
+; GFX9-NEXT:    v_cvt_f32_i32_e32 v1, s0
 ; GFX9-NEXT:    v_rcp_iflag_f32_e32 v5, v0
-; GFX9-NEXT:    s_xor_b32 s2, s2, s4
-; GFX9-NEXT:    s_ashr_i32 s2, s2, 30
-; GFX9-NEXT:    s_or_b32 s2, s2, 1
+; GFX9-NEXT:    s_xor_b32 s0, s0, s2
+; GFX9-NEXT:    s_ashr_i32 s0, s0, 30
+; GFX9-NEXT:    s_or_b32 s0, s0, 1
 ; GFX9-NEXT:    v_mul_f32_e32 v5, v1, v5
 ; GFX9-NEXT:    v_trunc_f32_e32 v5, v5
 ; GFX9-NEXT:    v_mad_f32 v1, -v5, v0, v1
 ; GFX9-NEXT:    v_cmp_ge_f32_e64 s[4:5], |v1|, |v0|
 ; GFX9-NEXT:    s_and_b64 s[4:5], s[4:5], exec
 ; GFX9-NEXT:    v_cvt_i32_f32_e32 v5, v5
-; GFX9-NEXT:    s_cselect_b32 s2, s2, 0
-; GFX9-NEXT:    s_ashr_i32 s4, s7, 16
-; GFX9-NEXT:    v_cvt_f32_i32_e32 v0, s4
-; GFX9-NEXT:    v_add_u32_e32 v1, s2, v5
+; GFX9-NEXT:    s_cselect_b32 s0, s0, 0
 ; GFX9-NEXT:    s_ashr_i32 s2, s3, 16
-; GFX9-NEXT:    v_cvt_f32_i32_e32 v5, s2
+; GFX9-NEXT:    v_cvt_f32_i32_e32 v0, s2
+; GFX9-NEXT:    v_add_u32_e32 v1, s0, v5
+; GFX9-NEXT:    s_ashr_i32 s0, s1, 16
+; GFX9-NEXT:    v_cvt_f32_i32_e32 v5, s0
 ; GFX9-NEXT:    v_rcp_iflag_f32_e32 v6, v0
-; GFX9-NEXT:    s_xor_b32 s2, s2, s4
-; GFX9-NEXT:    s_ashr_i32 s2, s2, 30
-; GFX9-NEXT:    s_or_b32 s4, s2, 1
+; GFX9-NEXT:    s_xor_b32 s0, s0, s2
+; GFX9-NEXT:    s_ashr_i32 s0, s0, 30
+; GFX9-NEXT:    s_or_b32 s2, s0, 1
 ; GFX9-NEXT:    v_mul_f32_e32 v6, v5, v6
 ; GFX9-NEXT:    v_trunc_f32_e32 v6, v6
 ; GFX9-NEXT:    v_mad_f32 v5, -v6, v0, v5
 ; GFX9-NEXT:    v_cvt_i32_f32_e32 v6, v6
-; GFX9-NEXT:    v_cmp_ge_f32_e64 s[2:3], |v5|, |v0|
-; GFX9-NEXT:    s_and_b64 s[2:3], s[2:3], exec
-; GFX9-NEXT:    s_cselect_b32 s2, s4, 0
-; GFX9-NEXT:    v_add_u32_e32 v0, s2, v6
+; GFX9-NEXT:    v_cmp_ge_f32_e64 s[0:1], |v5|, |v0|
+; GFX9-NEXT:    s_and_b64 s[0:1], s[0:1], exec
+; GFX9-NEXT:    s_cselect_b32 s0, s2, 0
+; GFX9-NEXT:    v_add_u32_e32 v0, s0, v6
 ; GFX9-NEXT:    v_and_b32_e32 v1, 0xffff, v1
 ; GFX9-NEXT:    v_lshl_or_b32 v1, v0, 16, v1
 ; GFX9-NEXT:    v_and_b32_e32 v0, 0xffff, v3
 ; GFX9-NEXT:    v_lshl_or_b32 v0, v4, 16, v0
-; GFX9-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX9-NEXT:    global_store_dwordx2 v2, v[0:1], s[6:7]
 ; GFX9-NEXT:    s_endpgm
   %r = sdiv <4 x i16> %x, %y
   store <4 x i16> %r, ptr addrspace(1) %out
@@ -3271,55 +3266,53 @@ define amdgpu_kernel void @srem_v4i16(ptr addrspace(1) %out, <4 x i16> %x, <4 x
 ;
 ; GFX6-LABEL: srem_v4i16:
 ; GFX6:       ; %bb.0:
-; GFX6-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0xd
-; GFX6-NEXT:    s_load_dwordx4 s[4:7], s[4:5], 0x9
+; GFX6-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0xb
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x9
 ; GFX6-NEXT:    s_mov_b32 s3, 0xf000
 ; GFX6-NEXT:    s_mov_b32 s2, -1
 ; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX6-NEXT:    s_sext_i32_i16 s0, s8
-; GFX6-NEXT:    v_cvt_f32_i32_e32 v0, s0
-; GFX6-NEXT:    s_sext_i32_i16 s1, s6
-; GFX6-NEXT:    v_cvt_f32_i32_e32 v1, s1
-; GFX6-NEXT:    s_xor_b32 s0, s1, s0
+; GFX6-NEXT:    s_sext_i32_i16 s4, s10
+; GFX6-NEXT:    v_cvt_f32_i32_e32 v0, s4
+; GFX6-NEXT:    s_sext_i32_i16 s5, s8
+; GFX6-NEXT:    v_cvt_f32_i32_e32 v1, s5
+; GFX6-NEXT:    s_xor_b32 s4, s5, s4
 ; GFX6-NEXT:    v_rcp_iflag_f32_e32 v2, v0
-; GFX6-NEXT:    s_ashr_i32 s0, s0, 30
-; GFX6-NEXT:    s_or_b32 s10, s0, 1
+; GFX6-NEXT:    s_ashr_i32 s4, s4, 30
+; GFX6-NEXT:    s_or_b32 s6, s4, 1
 ; GFX6-NEXT:    v_mul_f32_e32 v2, v1, v2
 ; GFX6-NEXT:    v_trunc_f32_e32 v2, v2
 ; GFX6-NEXT:    v_mad_f32 v1, -v2, v0, v1
 ; GFX6-NEXT:    v_cvt_i32_f32_e32 v2, v2
-; GFX6-NEXT:    v_cmp_ge_f32_e64 s[0:1], |v1|, |v0|
-; GFX6-NEXT:    s_and_b64 s[0:1], s[0:1], exec
-; GFX6-NEXT:    s_cselect_b32 s0, s10, 0
-; GFX6-NEXT:    v_add_i32_e32 v0, vcc, s0, v2
-; GFX6-NEXT:    s_mov_b32 s0, s4
-; GFX6-NEXT:    s_ashr_i32 s4, s8, 16
+; GFX6-NEXT:    v_cmp_ge_f32_e64 s[4:5], |v1|, |v0|
+; GFX6-NEXT:    s_and_b64 s[4:5], s[4:5], exec
+; GFX6-NEXT:    s_cselect_b32 s4, s6, 0
+; GFX6-NEXT:    v_add_i32_e32 v0, vcc, s4, v2
+; GFX6-NEXT:    s_ashr_i32 s4, s10, 16
 ; GFX6-NEXT:    v_cvt_f32_i32_e32 v1, s4
-; GFX6-NEXT:    s_mov_b32 s1, s5
-; GFX6-NEXT:    s_ashr_i32 s5, s6, 16
+; GFX6-NEXT:    s_ashr_i32 s5, s8, 16
 ; GFX6-NEXT:    v_cvt_f32_i32_e32 v2, s5
+; GFX6-NEXT:    v_mul_lo_u32 v0, v0, s10
 ; GFX6-NEXT:    v_rcp_iflag_f32_e32 v3, v1
-; GFX6-NEXT:    v_mul_lo_u32 v0, v0, s8
 ; GFX6-NEXT:    s_xor_b32 s4, s5, s4
 ; GFX6-NEXT:    s_ashr_i32 s4, s4, 30
+; GFX6-NEXT:    s_lshr_b32 s6, s8, 16
 ; GFX6-NEXT:    v_mul_f32_e32 v3, v2, v3
 ; GFX6-NEXT:    v_trunc_f32_e32 v3, v3
 ; GFX6-NEXT:    v_mad_f32 v2, -v3, v1, v2
 ; GFX6-NEXT:    v_cvt_i32_f32_e32 v3, v3
-; GFX6-NEXT:    v_sub_i32_e32 v0, vcc, s6, v0
-; GFX6-NEXT:    s_lshr_b32 s10, s6, 16
-; GFX6-NEXT:    s_lshr_b32 s8, s8, 16
-; GFX6-NEXT:    s_or_b32 s6, s4, 1
+; GFX6-NEXT:    v_sub_i32_e32 v0, vcc, s8, v0
+; GFX6-NEXT:    s_lshr_b32 s7, s10, 16
+; GFX6-NEXT:    s_or_b32 s8, s4, 1
 ; GFX6-NEXT:    v_cmp_ge_f32_e64 s[4:5], |v2|, |v1|
 ; GFX6-NEXT:    s_and_b64 s[4:5], s[4:5], exec
-; GFX6-NEXT:    s_cselect_b32 s4, s6, 0
+; GFX6-NEXT:    s_cselect_b32 s4, s8, 0
 ; GFX6-NEXT:    v_add_i32_e32 v1, vcc, s4, v3
-; GFX6-NEXT:    s_sext_i32_i16 s4, s9
-; GFX6-NEXT:    v_mul_lo_u32 v1, v1, s8
+; GFX6-NEXT:    s_sext_i32_i16 s4, s11
+; GFX6-NEXT:    v_mul_lo_u32 v1, v1, s7
 ; GFX6-NEXT:    v_cvt_f32_i32_e32 v2, s4
-; GFX6-NEXT:    s_sext_i32_i16 s5, s7
+; GFX6-NEXT:    s_sext_i32_i16 s5, s9
 ; GFX6-NEXT:    s_xor_b32 s4, s5, s4
-; GFX6-NEXT:    v_sub_i32_e32 v3, vcc, s10, v1
+; GFX6-NEXT:    v_sub_i32_e32 v3, vcc, s6, v1
 ; GFX6-NEXT:    v_cvt_f32_i32_e32 v1, s5
 ; GFX6-NEXT:    v_rcp_iflag_f32_e32 v4, v2
 ; GFX6-NEXT:    s_ashr_i32 s4, s4, 30
@@ -3333,30 +3326,30 @@ define amdgpu_kernel void @srem_v4i16(ptr addrspace(1) %out, <4 x i16> %x, <4 x
 ; GFX6-NEXT:    s_and_b64 s[4:5], s[4:5], exec
 ; GFX6-NEXT:    s_cselect_b32 s4, s6, 0
 ; GFX6-NEXT:    v_add_i32_e32 v1, vcc, s4, v4
-; GFX6-NEXT:    s_ashr_i32 s4, s9, 16
+; GFX6-NEXT:    s_ashr_i32 s4, s11, 16
 ; GFX6-NEXT:    v_cvt_f32_i32_e32 v2, s4
-; GFX6-NEXT:    s_ashr_i32 s5, s7, 16
+; GFX6-NEXT:    s_ashr_i32 s5, s9, 16
 ; GFX6-NEXT:    v_cvt_f32_i32_e32 v4, s5
 ; GFX6-NEXT:    s_xor_b32 s4, s5, s4
 ; GFX6-NEXT:    v_rcp_iflag_f32_e32 v5, v2
 ; GFX6-NEXT:    s_ashr_i32 s4, s4, 30
-; GFX6-NEXT:    v_mul_lo_u32 v1, v1, s9
-; GFX6-NEXT:    s_lshr_b32 s6, s7, 16
+; GFX6-NEXT:    s_lshr_b32 s6, s9, 16
+; GFX6-NEXT:    s_lshr_b32 s7, s11, 16
 ; GFX6-NEXT:    v_mul_f32_e32 v5, v4, v5
 ; GFX6-NEXT:    v_trunc_f32_e32 v5, v5
 ; GFX6-NEXT:    v_mad_f32 v4, -v5, v2, v4
 ; GFX6-NEXT:    v_cvt_i32_f32_e32 v5, v5
-; GFX6-NEXT:    s_lshr_b32 s8, s9, 16
-; GFX6-NEXT:    s_or_b32 s9, s4, 1
+; GFX6-NEXT:    s_or_b32 s8, s4, 1
 ; GFX6-NEXT:    v_cmp_ge_f32_e64 s[4:5], |v4|, |v2|
 ; GFX6-NEXT:    s_and_b64 s[4:5], s[4:5], exec
-; GFX6-NEXT:    s_cselect_b32 s4, s9, 0
+; GFX6-NEXT:    s_cselect_b32 s4, s8, 0
 ; GFX6-NEXT:    v_add_i32_e32 v2, vcc, s4, v5
-; GFX6-NEXT:    v_mul_lo_u32 v2, v2, s8
-; GFX6-NEXT:    v_sub_i32_e32 v1, vcc, s7, v1
-; GFX6-NEXT:    v_and_b32_e32 v1, 0xffff, v1
+; GFX6-NEXT:    v_mul_lo_u32 v1, v1, s11
+; GFX6-NEXT:    v_mul_lo_u32 v2, v2, s7
+; GFX6-NEXT:    v_sub_i32_e32 v1, vcc, s9, v1
 ; GFX6-NEXT:    v_sub_i32_e32 v2, vcc, s6, v2
 ; GFX6-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX6-NEXT:    v_and_b32_e32 v1, 0xffff, v1
 ; GFX6-NEXT:    v_or_b32_e32 v1, v1, v2
 ; GFX6-NEXT:    v_lshlrev_b32_e32 v2, 16, v3
 ; GFX6-NEXT:    v_or_b32_e32 v0, v0, v2
@@ -3365,13 +3358,13 @@ define amdgpu_kernel void @srem_v4i16(ptr addrspace(1) %out, <4 x i16> %x, <4 x
 ;
 ; GFX9-LABEL: srem_v4i16:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
-; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x2c
+; GFX9-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x24
 ; GFX9-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    s_sext_i32_i16 s8, s6
+; GFX9-NEXT:    s_sext_i32_i16 s8, s2
 ; GFX9-NEXT:    v_cvt_f32_i32_e32 v0, s8
-; GFX9-NEXT:    s_sext_i32_i16 s9, s2
+; GFX9-NEXT:    s_sext_i32_i16 s9, s0
 ; GFX9-NEXT:    v_cvt_f32_i32_e32 v1, s9
 ; GFX9-NEXT:    s_xor_b32 s4, s9, s8
 ; GFX9-NEXT:    v_rcp_iflag_f32_e32 v3, v0
@@ -3383,69 +3376,69 @@ define amdgpu_kernel void @srem_v4i16(ptr addrspace(1) %out, <4 x i16> %x, <4 x
 ; GFX9-NEXT:    v_cmp_ge_f32_e64 s[4:5], |v1|, |v0|
 ; GFX9-NEXT:    s_and_b64 s[4:5], s[4:5], exec
 ; GFX9-NEXT:    s_cselect_b32 s4, s10, 0
-; GFX9-NEXT:    s_ashr_i32 s10, s2, 16
-; GFX9-NEXT:    s_ashr_i32 s2, s6, 16
+; GFX9-NEXT:    s_ashr_i32 s10, s0, 16
+; GFX9-NEXT:    s_ashr_i32 s0, s2, 16
 ; GFX9-NEXT:    v_cvt_i32_f32_e32 v3, v3
-; GFX9-NEXT:    v_cvt_f32_i32_e32 v0, s2
+; GFX9-NEXT:    v_cvt_f32_i32_e32 v0, s0
+; GFX9-NEXT:    s_xor_b32 s2, s10, s0
+; GFX9-NEXT:    s_ashr_i32 s2, s2, 30
 ; GFX9-NEXT:    v_add_u32_e32 v1, s4, v3
 ; GFX9-NEXT:    v_cvt_f32_i32_e32 v3, s10
 ; GFX9-NEXT:    v_rcp_iflag_f32_e32 v4, v0
-; GFX9-NEXT:    s_xor_b32 s4, s10, s2
-; GFX9-NEXT:    s_ashr_i32 s4, s4, 30
-; GFX9-NEXT:    s_or_b32 s6, s4, 1
+; GFX9-NEXT:    s_or_b32 s2, s2, 1
+; GFX9-NEXT:    v_mul_lo_u32 v1, v1, s8
+; GFX9-NEXT:    s_sext_i32_i16 s8, s1
 ; GFX9-NEXT:    v_mul_f32_e32 v4, v3, v4
 ; GFX9-NEXT:    v_trunc_f32_e32 v4, v4
 ; GFX9-NEXT:    v_mad_f32 v3, -v4, v0, v3
+; GFX9-NEXT:    v_cvt_i32_f32_e32 v4, v4
 ; GFX9-NEXT:    v_cmp_ge_f32_e64 s[4:5], |v3|, |v0|
 ; GFX9-NEXT:    s_and_b64 s[4:5], s[4:5], exec
-; GFX9-NEXT:    s_cselect_b32 s4, s6, 0
-; GFX9-NEXT:    s_sext_i32_i16 s6, s7
-; GFX9-NEXT:    v_cvt_i32_f32_e32 v4, v4
-; GFX9-NEXT:    v_cvt_f32_i32_e32 v3, s6
-; GFX9-NEXT:    v_mul_lo_u32 v1, v1, s8
-; GFX9-NEXT:    s_sext_i32_i16 s8, s3
-; GFX9-NEXT:    v_add_u32_e32 v0, s4, v4
+; GFX9-NEXT:    s_cselect_b32 s2, s2, 0
+; GFX9-NEXT:    v_add_u32_e32 v0, s2, v4
+; GFX9-NEXT:    s_sext_i32_i16 s2, s3
+; GFX9-NEXT:    v_cvt_f32_i32_e32 v3, s2
 ; GFX9-NEXT:    v_cvt_f32_i32_e32 v4, s8
+; GFX9-NEXT:    v_mul_lo_u32 v0, v0, s0
+; GFX9-NEXT:    s_xor_b32 s0, s8, s2
 ; GFX9-NEXT:    v_rcp_iflag_f32_e32 v5, v3
-; GFX9-NEXT:    v_mul_lo_u32 v0, v0, s2
-; GFX9-NEXT:    s_xor_b32 s2, s8, s6
-; GFX9-NEXT:    s_ashr_i32 s2, s2, 30
+; GFX9-NEXT:    s_ashr_i32 s0, s0, 30
+; GFX9-NEXT:    s_or_b32 s0, s0, 1
+; GFX9-NEXT:    v_sub_u32_e32 v0, s10, v0
 ; GFX9-NEXT:    v_mul_f32_e32 v5, v4, v5
 ; GFX9-NEXT:    v_trunc_f32_e32 v5, v5
 ; GFX9-NEXT:    v_mad_f32 v4, -v5, v3, v4
-; GFX9-NEXT:    s_or_b32 s2, s2, 1
 ; GFX9-NEXT:    v_cmp_ge_f32_e64 s[4:5], |v4|, |v3|
-; GFX9-NEXT:    s_and_b64 s[4:5], s[4:5], exec
-; GFX9-NEXT:    s_cselect_b32 s2, s2, 0
-; GFX9-NEXT:    s_ashr_i32 s4, s7, 16
 ; GFX9-NEXT:    v_cvt_i32_f32_e32 v5, v5
-; GFX9-NEXT:    v_cvt_f32_i32_e32 v4, s4
-; GFX9-NEXT:    s_ashr_i32 s5, s3, 16
-; GFX9-NEXT:    v_sub_u32_e32 v0, s10, v0
-; GFX9-NEXT:    v_add_u32_e32 v3, s2, v5
-; GFX9-NEXT:    v_cvt_f32_i32_e32 v5, s5
+; GFX9-NEXT:    s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT:    s_cselect_b32 s0, s0, 0
+; GFX9-NEXT:    s_ashr_i32 s3, s3, 16
+; GFX9-NEXT:    v_cvt_f32_i32_e32 v4, s3
+; GFX9-NEXT:    v_add_u32_e32 v3, s0, v5
+; GFX9-NEXT:    v_mul_lo_u32 v3, v3, s2
+; GFX9-NEXT:    s_ashr_i32 s2, s1, 16
+; GFX9-NEXT:    v_cvt_f32_i32_e32 v5, s2
 ; GFX9-NEXT:    v_rcp_iflag_f32_e32 v6, v4
-; GFX9-NEXT:    s_xor_b32 s2, s5, s4
-; GFX9-NEXT:    s_ashr_i32 s2, s2, 30
-; GFX9-NEXT:    v_mul_lo_u32 v3, v3, s6
+; GFX9-NEXT:    s_xor_b32 s0, s2, s3
+; GFX9-NEXT:    s_ashr_i32 s0, s0, 30
+; GFX9-NEXT:    s_or_b32 s4, s0, 1
 ; GFX9-NEXT:    v_mul_f32_e32 v6, v5, v6
 ; GFX9-NEXT:    v_trunc_f32_e32 v6, v6
 ; GFX9-NEXT:    v_mad_f32 v5, -v6, v4, v5
 ; GFX9-NEXT:    v_cvt_i32_f32_e32 v6, v6
-; GFX9-NEXT:    s_or_b32 s6, s2, 1
-; GFX9-NEXT:    v_cmp_ge_f32_e64 s[2:3], |v5|, |v4|
-; GFX9-NEXT:    s_and_b64 s[2:3], s[2:3], exec
-; GFX9-NEXT:    s_cselect_b32 s2, s6, 0
-; GFX9-NEXT:    v_add_u32_e32 v4, s2, v6
-; GFX9-NEXT:    v_mul_lo_u32 v4, v4, s4
+; GFX9-NEXT:    v_cmp_ge_f32_e64 s[0:1], |v5|, |v4|
+; GFX9-NEXT:    s_and_b64 s[0:1], s[0:1], exec
+; GFX9-NEXT:    s_cselect_b32 s0, s4, 0
+; GFX9-NEXT:    v_add_u32_e32 v4, s0, v6
+; GFX9-NEXT:    v_mul_lo_u32 v4, v4, s3
 ; GFX9-NEXT:    v_sub_u32_e32 v5, s9, v1
 ; GFX9-NEXT:    v_sub_u32_e32 v1, s8, v3
 ; GFX9-NEXT:    v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NEXT:    v_sub_u32_e32 v3, s5, v4
+; GFX9-NEXT:    v_sub_u32_e32 v3, s2, v4
 ; GFX9-NEXT:    v_lshl_or_b32 v1, v3, 16, v1
 ; GFX9-NEXT:    v_and_b32_e32 v3, 0xffff, v5
 ; GFX9-NEXT:    v_lshl_or_b32 v0, v0, 16, v3
-; GFX9-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX9-NEXT:    global_store_dwordx2 v2, v[0:1], s[6:7]
 ; GFX9-NEXT:    s_endpgm
   %r = srem <4 x i16> %x, %y
   store <4 x i16> %r, ptr addrspace(1) %out
@@ -3841,48 +3834,46 @@ define amdgpu_kernel void @udiv_v3i16(ptr addrspace(1) %out, <3 x i16> %x, <3 x
 ;
 ; GFX6-LABEL: udiv_v3i16:
 ; GFX6:       ; %bb.0:
-; GFX6-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0xd
-; GFX6-NEXT:    s_load_dwordx4 s[4:7], s[4:5], 0x9
+; GFX6-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0xb
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x9
 ; GFX6-NEXT:    s_mov_b32 s3, 0xf000
 ; GFX6-NEXT:    s_mov_b32 s2, -1
 ; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX6-NEXT:    s_and_b32 s0, s8, 0xffff
-; GFX6-NEXT:    v_cvt_f32_u32_e32 v0, s0
-; GFX6-NEXT:    s_mov_b32 s0, s4
-; GFX6-NEXT:    s_and_b32 s1, s6, 0xffff
+; GFX6-NEXT:    s_and_b32 s5, s10, 0xffff
+; GFX6-NEXT:    v_cvt_f32_u32_e32 v0, s5
+; GFX6-NEXT:    s_lshr_b32 s5, s10, 16
+; GFX6-NEXT:    s_and_b32 s4, s8, 0xffff
+; GFX6-NEXT:    v_cvt_f32_u32_e32 v2, s5
+; GFX6-NEXT:    v_cvt_f32_u32_e32 v1, s4
+; GFX6-NEXT:    v_rcp_iflag_f32_e32 v3, v0
 ; GFX6-NEXT:    s_lshr_b32 s4, s8, 16
-; GFX6-NEXT:    v_cvt_f32_u32_e32 v1, s1
-; GFX6-NEXT:    v_rcp_iflag_f32_e32 v2, v0
-; GFX6-NEXT:    v_cvt_f32_u32_e32 v3, s4
-; GFX6-NEXT:    s_lshr_b32 s4, s6, 16
 ; GFX6-NEXT:    v_cvt_f32_u32_e32 v4, s4
-; GFX6-NEXT:    v_mul_f32_e32 v2, v1, v2
-; GFX6-NEXT:    v_rcp_iflag_f32_e32 v5, v3
-; GFX6-NEXT:    v_trunc_f32_e32 v2, v2
-; GFX6-NEXT:    v_mad_f32 v1, -v2, v0, v1
-; GFX6-NEXT:    v_cvt_u32_f32_e32 v2, v2
+; GFX6-NEXT:    v_rcp_iflag_f32_e32 v5, v2
+; GFX6-NEXT:    v_mul_f32_e32 v3, v1, v3
+; GFX6-NEXT:    v_trunc_f32_e32 v3, v3
+; GFX6-NEXT:    v_mad_f32 v1, -v3, v0, v1
 ; GFX6-NEXT:    v_cmp_ge_f32_e64 vcc, |v1|, v0
 ; GFX6-NEXT:    v_mul_f32_e32 v1, v4, v5
 ; GFX6-NEXT:    v_trunc_f32_e32 v1, v1
-; GFX6-NEXT:    s_and_b32 s4, s9, 0xffff
-; GFX6-NEXT:    v_addc_u32_e32 v0, vcc, 0, v2, vcc
-; GFX6-NEXT:    v_mad_f32 v2, -v1, v3, v4
+; GFX6-NEXT:    s_and_b32 s4, s11, 0xffff
+; GFX6-NEXT:    v_cvt_u32_f32_e32 v6, v3
+; GFX6-NEXT:    v_mad_f32 v3, -v1, v2, v4
 ; GFX6-NEXT:    v_cvt_f32_u32_e32 v4, s4
-; GFX6-NEXT:    s_and_b32 s4, s7, 0xffff
+; GFX6-NEXT:    s_and_b32 s4, s9, 0xffff
+; GFX6-NEXT:    v_addc_u32_e32 v0, vcc, 0, v6, vcc
 ; GFX6-NEXT:    v_cvt_f32_u32_e32 v5, s4
-; GFX6-NEXT:    v_cvt_u32_f32_e32 v1, v1
 ; GFX6-NEXT:    v_rcp_iflag_f32_e32 v6, v4
-; GFX6-NEXT:    v_cmp_ge_f32_e64 vcc, |v2|, v3
-; GFX6-NEXT:    s_mov_b32 s1, s5
-; GFX6-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX6-NEXT:    v_cvt_u32_f32_e32 v1, v1
+; GFX6-NEXT:    v_cmp_ge_f32_e64 vcc, |v3|, v2
+; GFX6-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX6-NEXT:    v_mul_f32_e32 v2, v5, v6
 ; GFX6-NEXT:    v_trunc_f32_e32 v2, v2
 ; GFX6-NEXT:    v_cvt_u32_f32_e32 v3, v2
+; GFX6-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
 ; GFX6-NEXT:    v_mad_f32 v2, -v2, v4, v5
 ; GFX6-NEXT:    v_cmp_ge_f32_e64 vcc, |v2|, v4
-; GFX6-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; GFX6-NEXT:    v_addc_u32_e32 v2, vcc, 0, v3, vcc
-; GFX6-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GFX6-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; GFX6-NEXT:    v_or_b32_e32 v0, v0, v1
 ; GFX6-NEXT:    buffer_store_short v2, off, s[0:3], 0 offset:4
 ; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
@@ -3890,47 +3881,48 @@ define amdgpu_kernel void @udiv_v3i16(ptr addrspace(1) %out, <3 x i16> %x, <3 x
 ;
 ; GFX9-LABEL: udiv_v3i16:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX9-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x2c
 ; GFX9-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    s_and_b32 s4, s2, 0xffff
-; GFX9-NEXT:    s_and_b32 s5, s6, 0xffff
-; GFX9-NEXT:    v_cvt_f32_u32_e32 v0, s5
-; GFX9-NEXT:    s_lshr_b32 s6, s6, 16
-; GFX9-NEXT:    v_cvt_f32_u32_e32 v1, s6
-; GFX9-NEXT:    v_cvt_f32_u32_e32 v2, s4
-; GFX9-NEXT:    v_rcp_iflag_f32_e32 v4, v0
+; GFX9-NEXT:    s_and_b32 s7, s2, 0xffff
+; GFX9-NEXT:    v_cvt_f32_u32_e32 v0, s7
+; GFX9-NEXT:    s_and_b32 s6, s0, 0xffff
 ; GFX9-NEXT:    s_lshr_b32 s2, s2, 16
-; GFX9-NEXT:    v_cvt_f32_u32_e32 v3, s2
+; GFX9-NEXT:    v_cvt_f32_u32_e32 v1, s2
+; GFX9-NEXT:    v_cvt_f32_u32_e32 v2, s6
+; GFX9-NEXT:    v_rcp_iflag_f32_e32 v4, v0
+; GFX9-NEXT:    s_lshr_b32 s0, s0, 16
+; GFX9-NEXT:    v_cvt_f32_u32_e32 v3, s0
 ; GFX9-NEXT:    v_rcp_iflag_f32_e32 v5, v1
 ; GFX9-NEXT:    v_mul_f32_e32 v4, v2, v4
 ; GFX9-NEXT:    v_trunc_f32_e32 v4, v4
-; GFX9-NEXT:    s_and_b32 s2, s7, 0xffff
+; GFX9-NEXT:    s_and_b32 s0, s3, 0xffff
 ; GFX9-NEXT:    v_cvt_u32_f32_e32 v7, v4
 ; GFX9-NEXT:    v_mad_f32 v2, -v4, v0, v2
-; GFX9-NEXT:    v_cvt_f32_u32_e32 v4, s2
+; GFX9-NEXT:    v_cvt_f32_u32_e32 v4, s0
 ; GFX9-NEXT:    v_mul_f32_e32 v5, v3, v5
 ; GFX9-NEXT:    v_cmp_ge_f32_e64 vcc, |v2|, v0
 ; GFX9-NEXT:    v_trunc_f32_e32 v2, v5
-; GFX9-NEXT:    s_and_b32 s2, s3, 0xffff
+; GFX9-NEXT:    s_and_b32 s0, s1, 0xffff
 ; GFX9-NEXT:    v_addc_co_u32_e32 v0, vcc, 0, v7, vcc
 ; GFX9-NEXT:    v_mad_f32 v3, -v2, v1, v3
 ; GFX9-NEXT:    v_cvt_u32_f32_e32 v2, v2
-; GFX9-NEXT:    v_cvt_f32_u32_e32 v5, s2
+; GFX9-NEXT:    v_cvt_f32_u32_e32 v5, s0
 ; GFX9-NEXT:    v_rcp_iflag_f32_e32 v7, v4
 ; GFX9-NEXT:    v_cmp_ge_f32_e64 vcc, |v3|, v1
 ; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v2, vcc
 ; GFX9-NEXT:    v_mul_f32_e32 v2, v5, v7
 ; GFX9-NEXT:    v_trunc_f32_e32 v2, v2
+; GFX9-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x24
 ; GFX9-NEXT:    v_cvt_u32_f32_e32 v3, v2
 ; GFX9-NEXT:    v_mad_f32 v2, -v2, v4, v5
 ; GFX9-NEXT:    v_cmp_ge_f32_e64 vcc, |v2|, v4
 ; GFX9-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX9-NEXT:    v_addc_co_u32_e32 v2, vcc, 0, v3, vcc
 ; GFX9-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
-; GFX9-NEXT:    global_store_short v6, v2, s[0:1] offset:4
-; GFX9-NEXT:    global_store_dword v6, v0, s[0:1]
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    global_store_short v6, v2, s[6:7] offset:4
+; GFX9-NEXT:    global_store_dword v6, v0, s[6:7]
 ; GFX9-NEXT:    s_endpgm
   %r = udiv <3 x i16> %x, %y
   store <3 x i16> %r, ptr addrspace(1) %out
@@ -4010,54 +4002,52 @@ define amdgpu_kernel void @urem_v3i16(ptr addrspace(1) %out, <3 x i16> %x, <3 x
 ;
 ; GFX6-LABEL: urem_v3i16:
 ; GFX6:       ; %bb.0:
-; GFX6-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x9
-; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xd
+; GFX6-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0xb
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x9
 ; GFX6-NEXT:    s_mov_b32 s3, 0xf000
 ; GFX6-NEXT:    s_mov_b32 s2, -1
 ; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX6-NEXT:    s_lshr_b32 s6, s10, 16
-; GFX6-NEXT:    s_and_b32 s1, s4, 0xffff
-; GFX6-NEXT:    v_cvt_f32_u32_e32 v0, s1
-; GFX6-NEXT:    s_and_b32 s1, s10, 0xffff
-; GFX6-NEXT:    v_cvt_f32_u32_e32 v1, s1
-; GFX6-NEXT:    s_lshr_b32 s7, s4, 16
-; GFX6-NEXT:    v_rcp_iflag_f32_e32 v2, v0
-; GFX6-NEXT:    v_cvt_f32_u32_e32 v4, s7
-; GFX6-NEXT:    v_cvt_f32_u32_e32 v3, s6
-; GFX6-NEXT:    s_mov_b32 s0, s8
-; GFX6-NEXT:    v_mul_f32_e32 v2, v1, v2
-; GFX6-NEXT:    v_trunc_f32_e32 v2, v2
-; GFX6-NEXT:    v_mad_f32 v1, -v2, v0, v1
-; GFX6-NEXT:    v_cvt_u32_f32_e32 v5, v2
+; GFX6-NEXT:    s_and_b32 s5, s10, 0xffff
+; GFX6-NEXT:    v_cvt_f32_u32_e32 v0, s5
+; GFX6-NEXT:    s_lshr_b32 s5, s10, 16
+; GFX6-NEXT:    s_and_b32 s4, s8, 0xffff
+; GFX6-NEXT:    v_cvt_f32_u32_e32 v2, s5
+; GFX6-NEXT:    v_cvt_f32_u32_e32 v1, s4
+; GFX6-NEXT:    v_rcp_iflag_f32_e32 v3, v0
+; GFX6-NEXT:    s_lshr_b32 s4, s8, 16
+; GFX6-NEXT:    v_cvt_f32_u32_e32 v4, s4
+; GFX6-NEXT:    v_rcp_iflag_f32_e32 v5, v2
+; GFX6-NEXT:    v_mul_f32_e32 v3, v1, v3
+; GFX6-NEXT:    v_trunc_f32_e32 v3, v3
+; GFX6-NEXT:    v_mad_f32 v1, -v3, v0, v1
+; GFX6-NEXT:    v_cvt_u32_f32_e32 v6, v3
 ; GFX6-NEXT:    v_cmp_ge_f32_e64 vcc, |v1|, v0
-; GFX6-NEXT:    v_rcp_iflag_f32_e32 v1, v4
-; GFX6-NEXT:    s_mov_b32 s1, s9
-; GFX6-NEXT:    v_addc_u32_e32 v0, vcc, 0, v5, vcc
-; GFX6-NEXT:    v_mul_f32_e32 v1, v3, v1
-; GFX6-NEXT:    v_mul_lo_u32 v0, v0, s4
+; GFX6-NEXT:    v_mul_f32_e32 v1, v4, v5
 ; GFX6-NEXT:    v_trunc_f32_e32 v1, v1
-; GFX6-NEXT:    s_and_b32 s4, s5, 0xffff
-; GFX6-NEXT:    v_mad_f32 v2, -v1, v4, v3
-; GFX6-NEXT:    v_cvt_f32_u32_e32 v3, s4
-; GFX6-NEXT:    s_and_b32 s4, s11, 0xffff
-; GFX6-NEXT:    v_cvt_f32_u32_e32 v5, s4
-; GFX6-NEXT:    v_sub_i32_e32 v0, vcc, s10, v0
-; GFX6-NEXT:    v_rcp_iflag_f32_e32 v6, v3
+; GFX6-NEXT:    s_and_b32 s6, s11, 0xffff
+; GFX6-NEXT:    v_mad_f32 v3, -v1, v2, v4
+; GFX6-NEXT:    v_cvt_f32_u32_e32 v4, s6
+; GFX6-NEXT:    v_addc_u32_e32 v0, vcc, 0, v6, vcc
+; GFX6-NEXT:    v_mul_lo_u32 v0, v0, s10
+; GFX6-NEXT:    s_and_b32 s6, s9, 0xffff
+; GFX6-NEXT:    v_cvt_f32_u32_e32 v5, s6
+; GFX6-NEXT:    v_rcp_iflag_f32_e32 v6, v4
+; GFX6-NEXT:    v_sub_i32_e32 v0, vcc, s8, v0
 ; GFX6-NEXT:    v_cvt_u32_f32_e32 v1, v1
-; GFX6-NEXT:    v_cmp_ge_f32_e64 vcc, |v2|, v4
-; GFX6-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GFX6-NEXT:    v_cmp_ge_f32_e64 vcc, |v3|, v2
 ; GFX6-NEXT:    v_mul_f32_e32 v2, v5, v6
 ; GFX6-NEXT:    v_trunc_f32_e32 v2, v2
-; GFX6-NEXT:    v_cvt_u32_f32_e32 v4, v2
+; GFX6-NEXT:    v_cvt_u32_f32_e32 v3, v2
 ; GFX6-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GFX6-NEXT:    v_mad_f32 v2, -v2, v3, v5
-; GFX6-NEXT:    v_cmp_ge_f32_e64 vcc, |v2|, v3
-; GFX6-NEXT:    v_mul_lo_u32 v1, v1, s7
-; GFX6-NEXT:    v_addc_u32_e32 v2, vcc, 0, v4, vcc
-; GFX6-NEXT:    v_mul_lo_u32 v2, v2, s5
-; GFX6-NEXT:    v_sub_i32_e32 v1, vcc, s6, v1
+; GFX6-NEXT:    v_mad_f32 v2, -v2, v4, v5
+; GFX6-NEXT:    v_cmp_ge_f32_e64 vcc, |v2|, v4
+; GFX6-NEXT:    v_mul_lo_u32 v1, v1, s5
+; GFX6-NEXT:    v_addc_u32_e32 v2, vcc, 0, v3, vcc
+; GFX6-NEXT:    v_mul_lo_u32 v2, v2, s11
+; GFX6-NEXT:    v_sub_i32_e32 v1, vcc, s4, v1
 ; GFX6-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
-; GFX6-NEXT:    v_sub_i32_e32 v2, vcc, s11, v2
+; GFX6-NEXT:    v_sub_i32_e32 v2, vcc, s9, v2
+; GFX6-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX6-NEXT:    v_or_b32_e32 v0, v0, v1
 ; GFX6-NEXT:    buffer_store_short v2, off, s[0:3], 0 offset:4
 ; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
@@ -4065,34 +4055,33 @@ define amdgpu_kernel void @urem_v3i16(ptr addrspace(1) %out, <3 x i16> %x, <3 x
 ;
 ; GFX9-LABEL: urem_v3i16:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX9-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x2c
+; GFX9-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x24
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    s_and_b32 s4, s2, 0xffff
-; GFX9-NEXT:    s_and_b32 s5, s6, 0xffff
-; GFX9-NEXT:    v_cvt_f32_u32_e32 v0, s5
-; GFX9-NEXT:    v_cvt_f32_u32_e32 v2, s4
-; GFX9-NEXT:    s_lshr_b32 s6, s6, 16
-; GFX9-NEXT:    v_cvt_f32_u32_e32 v1, s6
-; GFX9-NEXT:    v_rcp_iflag_f32_e32 v4, v0
+; GFX9-NEXT:    s_and_b32 s9, s2, 0xffff
 ; GFX9-NEXT:    s_lshr_b32 s2, s2, 16
-; GFX9-NEXT:    v_cvt_f32_u32_e32 v3, s2
+; GFX9-NEXT:    v_cvt_f32_u32_e32 v0, s9
+; GFX9-NEXT:    v_cvt_f32_u32_e32 v1, s2
+; GFX9-NEXT:    s_and_b32 s8, s0, 0xffff
+; GFX9-NEXT:    s_lshr_b32 s0, s0, 16
+; GFX9-NEXT:    v_cvt_f32_u32_e32 v2, s8
+; GFX9-NEXT:    v_rcp_iflag_f32_e32 v4, v0
+; GFX9-NEXT:    v_cvt_f32_u32_e32 v3, s0
 ; GFX9-NEXT:    v_rcp_iflag_f32_e32 v5, v1
+; GFX9-NEXT:    s_and_b32 s3, s3, 0xffff
 ; GFX9-NEXT:    v_mul_f32_e32 v4, v2, v4
 ; GFX9-NEXT:    v_trunc_f32_e32 v4, v4
-; GFX9-NEXT:    v_cvt_u32_f32_e32 v6, v4
-; GFX9-NEXT:    v_mad_f32 v2, -v4, v0, v2
-; GFX9-NEXT:    v_cmp_ge_f32_e64 vcc, |v2|, v0
 ; GFX9-NEXT:    v_mul_f32_e32 v5, v3, v5
-; GFX9-NEXT:    v_addc_co_u32_e32 v0, vcc, 0, v6, vcc
 ; GFX9-NEXT:    v_trunc_f32_e32 v5, v5
-; GFX9-NEXT:    v_mul_lo_u32 v0, v0, s5
-; GFX9-NEXT:    s_and_b32 s5, s7, 0xffff
+; GFX9-NEXT:    v_mad_f32 v2, -v4, v0, v2
+; GFX9-NEXT:    v_cvt_u32_f32_e32 v6, v4
+; GFX9-NEXT:    v_cmp_ge_f32_e64 vcc, |v2|, v0
 ; GFX9-NEXT:    v_mad_f32 v2, -v5, v1, v3
-; GFX9-NEXT:    v_cvt_f32_u32_e32 v3, s5
-; GFX9-NEXT:    s_and_b32 s3, s3, 0xffff
+; GFX9-NEXT:    v_cvt_f32_u32_e32 v3, s3
+; GFX9-NEXT:    s_and_b32 s1, s1, 0xffff
+; GFX9-NEXT:    v_addc_co_u32_e32 v0, vcc, 0, v6, vcc
 ; GFX9-NEXT:    v_cvt_u32_f32_e32 v4, v5
-; GFX9-NEXT:    v_cvt_f32_u32_e32 v5, s3
+; GFX9-NEXT:    v_cvt_f32_u32_e32 v5, s1
 ; GFX9-NEXT:    v_rcp_iflag_f32_e32 v6, v3
 ; GFX9-NEXT:    v_cmp_ge_f32_e64 vcc, |v2|, v1
 ; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v4, vcc
@@ -4101,17 +4090,18 @@ define amdgpu_kernel void @urem_v3i16(ptr addrspace(1) %out, <3 x i16> %x, <3 x
 ; GFX9-NEXT:    v_cvt_u32_f32_e32 v4, v2
 ; GFX9-NEXT:    v_mad_f32 v2, -v2, v3, v5
 ; GFX9-NEXT:    v_cmp_ge_f32_e64 vcc, |v2|, v3
-; GFX9-NEXT:    v_mul_lo_u32 v1, v1, s6
+; GFX9-NEXT:    v_mul_lo_u32 v0, v0, s9
 ; GFX9-NEXT:    v_addc_co_u32_e32 v2, vcc, 0, v4, vcc
-; GFX9-NEXT:    v_mul_lo_u32 v2, v2, s5
-; GFX9-NEXT:    v_sub_u32_e32 v0, s4, v0
+; GFX9-NEXT:    v_mul_lo_u32 v1, v1, s2
+; GFX9-NEXT:    v_mul_lo_u32 v2, v2, s3
+; GFX9-NEXT:    v_sub_u32_e32 v0, s8, v0
 ; GFX9-NEXT:    v_mov_b32_e32 v3, 0
-; GFX9-NEXT:    v_sub_u32_e32 v1, s2, v1
-; GFX9-NEXT:    v_sub_u32_e32 v2, s3, v2
+; GFX9-NEXT:    v_sub_u32_e32 v1, s0, v1
+; GFX9-NEXT:    v_sub_u32_e32 v2, s1, v2
 ; GFX9-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX9-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
-; GFX9-NEXT:    global_store_short v3, v2, s[0:1] offset:4
-; GFX9-NEXT:    global_store_dword v3, v0, s[0:1]
+; GFX9-NEXT:    global_store_short v3, v2, s[6:7] offset:4
+; GFX9-NEXT:    global_store_dword v3, v0, s[6:7]
 ; GFX9-NEXT:    s_endpgm
   %r = urem <3 x i16> %x, %y
   store <3 x i16> %r, ptr addrspace(1) %out
@@ -4197,47 +4187,46 @@ define amdgpu_kernel void @sdiv_v3i16(ptr addrspace(1) %out, <3 x i16> %x, <3 x
 ;
 ; GFX6-LABEL: sdiv_v3i16:
 ; GFX6:       ; %bb.0:
-; GFX6-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x9
-; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xd
+; GFX6-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0xb
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x9
 ; GFX6-NEXT:    s_mov_b32 s3, 0xf000
 ; GFX6-NEXT:    s_mov_b32 s2, -1
 ; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX6-NEXT:    s_sext_i32_i16 s7, s10
-; GFX6-NEXT:    s_sext_i32_i16 s6, s4
-; GFX6-NEXT:    v_cvt_f32_i32_e32 v0, s6
-; GFX6-NEXT:    v_cvt_f32_i32_e32 v1, s7
-; GFX6-NEXT:    s_xor_b32 s6, s7, s6
-; GFX6-NEXT:    s_ashr_i32 s6, s6, 30
+; GFX6-NEXT:    s_sext_i32_i16 s4, s10
+; GFX6-NEXT:    v_cvt_f32_i32_e32 v0, s4
+; GFX6-NEXT:    s_sext_i32_i16 s5, s8
+; GFX6-NEXT:    v_cvt_f32_i32_e32 v1, s5
+; GFX6-NEXT:    s_xor_b32 s4, s5, s4
 ; GFX6-NEXT:    v_rcp_iflag_f32_e32 v2, v0
-; GFX6-NEXT:    s_mov_b32 s0, s8
-; GFX6-NEXT:    s_or_b32 s8, s6, 1
-; GFX6-NEXT:    s_sext_i32_i16 s5, s5
+; GFX6-NEXT:    s_ashr_i32 s4, s4, 30
+; GFX6-NEXT:    s_or_b32 s6, s4, 1
 ; GFX6-NEXT:    v_mul_f32_e32 v2, v1, v2
 ; GFX6-NEXT:    v_trunc_f32_e32 v2, v2
 ; GFX6-NEXT:    v_mad_f32 v1, -v2, v0, v1
-; GFX6-NEXT:    v_cmp_ge_f32_e64 s[6:7], |v1|, |v0|
-; GFX6-NEXT:    s_and_b64 s[6:7], s[6:7], exec
+; GFX6-NEXT:    v_cmp_ge_f32_e64 s[4:5], |v1|, |v0|
+; GFX6-NEXT:    s_and_b64 s[4:5], s[4:5], exec
 ; GFX6-NEXT:    v_cvt_i32_f32_e32 v2, v2
-; GFX6-NEXT:    s_cselect_b32 s6, s8, 0
-; GFX6-NEXT:    s_ashr_i32 s4, s4, 16
-; GFX6-NEXT:    v_cvt_f32_i32_e32 v0, s4
-; GFX6-NEXT:    v_add_i32_e32 v1, vcc, s6, v2
-; GFX6-NEXT:    s_ashr_i32 s6, s10, 16
-; GFX6-NEXT:    v_cvt_f32_i32_e32 v2, s6
+; GFX6-NEXT:    s_cselect_b32 s4, s6, 0
+; GFX6-NEXT:    s_ashr_i32 s5, s10, 16
+; GFX6-NEXT:    v_cvt_f32_i32_e32 v0, s5
+; GFX6-NEXT:    v_add_i32_e32 v1, vcc, s4, v2
+; GFX6-NEXT:    s_ashr_i32 s4, s8, 16
+; GFX6-NEXT:    v_cvt_f32_i32_e32 v2, s4
 ; GFX6-NEXT:    v_rcp_iflag_f32_e32 v3, v0
-; GFX6-NEXT:    s_xor_b32 s4, s6, s4
+; GFX6-NEXT:    s_xor_b32 s4, s4, s5
 ; GFX6-NEXT:    s_ashr_i32 s4, s4, 30
-; GFX6-NEXT:    s_or_b32 s4, s4, 1
+; GFX6-NEXT:    s_or_b32 s6, s4, 1
 ; GFX6-NEXT:    v_mul_f32_e32 v3, v2, v3
 ; GFX6-NEXT:    v_trunc_f32_e32 v3, v3
 ; GFX6-NEXT:    v_mad_f32 v2, -v3, v0, v2
+; GFX6-NEXT:    v_cmp_ge_f32_e64 s[4:5], |v2|, |v0|
+; GFX6-NEXT:    s_and_b64 s[4:5], s[4:5], exec
 ; GFX6-NEXT:    v_cvt_i32_f32_e32 v3, v3
-; GFX6-NEXT:    v_cmp_ge_f32_e64 s[6:7], |v2|, |v0|
+; GFX6-NEXT:    s_sext_i32_i16 s5, s11
 ; GFX6-NEXT:    v_cvt_f32_i32_e32 v0, s5
-; GFX6-NEXT:    s_and_b64 s[6:7], s[6:7], exec
-; GFX6-NEXT:    s_cselect_b32 s4, s4, 0
+; GFX6-NEXT:    s_cselect_b32 s4, s6, 0
 ; GFX6-NEXT:    v_add_i32_e32 v2, vcc, s4, v3
-; GFX6-NEXT:    s_sext_i32_i16 s4, s11
+; GFX6-NEXT:    s_sext_i32_i16 s4, s9
 ; GFX6-NEXT:    v_cvt_f32_i32_e32 v3, s4
 ; GFX6-NEXT:    v_rcp_iflag_f32_e32 v4, v0
 ; GFX6-NEXT:    s_xor_b32 s4, s4, s5
@@ -4250,7 +4239,6 @@ define amdgpu_kernel void @sdiv_v3i16(ptr addrspace(1) %out, <3 x i16> %x, <3 x
 ; GFX6-NEXT:    v_cmp_ge_f32_e64 s[4:5], |v3|, |v0|
 ; GFX6-NEXT:    s_and_b64 s[4:5], s[4:5], exec
 ; GFX6-NEXT:    s_cselect_b32 s4, s6, 0
-; GFX6-NEXT:    s_mov_b32 s1, s9
 ; GFX6-NEXT:    v_add_i32_e32 v0, vcc, s4, v4
 ; GFX6-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
 ; GFX6-NEXT:    v_and_b32_e32 v1, 0xffff, v1
@@ -4261,13 +4249,13 @@ define amdgpu_kernel void @sdiv_v3i16(ptr addrspace(1) %out, <3 x i16> %x, <3 x
 ;
 ; GFX9-LABEL: sdiv_v3i16:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
-; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x2c
+; GFX9-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x24
 ; GFX9-NEXT:    v_mov_b32_e32 v1, 0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    s_sext_i32_i16 s4, s6
+; GFX9-NEXT:    s_sext_i32_i16 s4, s2
 ; GFX9-NEXT:    v_cvt_f32_i32_e32 v0, s4
-; GFX9-NEXT:    s_sext_i32_i16 s5, s2
+; GFX9-NEXT:    s_sext_i32_i16 s5, s0
 ; GFX9-NEXT:    v_cvt_f32_i32_e32 v2, s5
 ; GFX9-NEXT:    s_xor_b32 s4, s5, s4
 ; GFX9-NEXT:    v_rcp_iflag_f32_e32 v3, v0
@@ -4279,44 +4267,44 @@ define amdgpu_kernel void @sdiv_v3i16(ptr addrspace(1) %out, <3 x i16> %x, <3 x
 ; GFX9-NEXT:    v_cmp_ge_f32_e64 s[4:5], |v2|, |v0|
 ; GFX9-NEXT:    s_and_b64 s[4:5], s[4:5], exec
 ; GFX9-NEXT:    s_cselect_b32 s4, s8, 0
-; GFX9-NEXT:    s_ashr_i32 s5, s6, 16
-; GFX9-NEXT:    v_cvt_i32_f32_e32 v3, v3
-; GFX9-NEXT:    v_cvt_f32_i32_e32 v0, s5
 ; GFX9-NEXT:    s_ashr_i32 s2, s2, 16
+; GFX9-NEXT:    v_cvt_i32_f32_e32 v3, v3
+; GFX9-NEXT:    v_cvt_f32_i32_e32 v0, s2
+; GFX9-NEXT:    s_ashr_i32 s0, s0, 16
 ; GFX9-NEXT:    v_add_u32_e32 v2, s4, v3
-; GFX9-NEXT:    v_cvt_f32_i32_e32 v3, s2
+; GFX9-NEXT:    v_cvt_f32_i32_e32 v3, s0
 ; GFX9-NEXT:    v_rcp_iflag_f32_e32 v4, v0
-; GFX9-NEXT:    s_xor_b32 s2, s2, s5
-; GFX9-NEXT:    s_ashr_i32 s2, s2, 30
-; GFX9-NEXT:    s_or_b32 s2, s2, 1
+; GFX9-NEXT:    s_xor_b32 s0, s0, s2
+; GFX9-NEXT:    s_ashr_i32 s0, s0, 30
+; GFX9-NEXT:    s_sext_i32_i16 s2, s3
 ; GFX9-NEXT:    v_mul_f32_e32 v4, v3, v4
 ; GFX9-NEXT:    v_trunc_f32_e32 v4, v4
 ; GFX9-NEXT:    v_mad_f32 v3, -v4, v0, v3
+; GFX9-NEXT:    v_cvt_i32_f32_e32 v4, v4
+; GFX9-NEXT:    s_or_b32 s0, s0, 1
 ; GFX9-NEXT:    v_cmp_ge_f32_e64 s[4:5], |v3|, |v0|
+; GFX9-NEXT:    v_cvt_f32_i32_e32 v0, s2
 ; GFX9-NEXT:    s_and_b64 s[4:5], s[4:5], exec
-; GFX9-NEXT:    v_cvt_i32_f32_e32 v4, v4
-; GFX9-NEXT:    s_sext_i32_i16 s4, s7
-; GFX9-NEXT:    v_cvt_f32_i32_e32 v0, s4
-; GFX9-NEXT:    s_cselect_b32 s2, s2, 0
-; GFX9-NEXT:    v_add_u32_e32 v3, s2, v4
-; GFX9-NEXT:    s_sext_i32_i16 s2, s3
-; GFX9-NEXT:    v_cvt_f32_i32_e32 v4, s2
+; GFX9-NEXT:    s_cselect_b32 s0, s0, 0
+; GFX9-NEXT:    v_add_u32_e32 v3, s0, v4
+; GFX9-NEXT:    s_sext_i32_i16 s0, s1
+; GFX9-NEXT:    v_cvt_f32_i32_e32 v4, s0
 ; GFX9-NEXT:    v_rcp_iflag_f32_e32 v5, v0
-; GFX9-NEXT:    s_xor_b32 s2, s2, s4
-; GFX9-NEXT:    s_ashr_i32 s2, s2, 30
-; GFX9-NEXT:    s_or_b32 s4, s2, 1
+; GFX9-NEXT:    s_xor_b32 s0, s0, s2
+; GFX9-NEXT:    s_ashr_i32 s0, s0, 30
+; GFX9-NEXT:    s_or_b32 s2, s0, 1
 ; GFX9-NEXT:    v_mul_f32_e32 v5, v4, v5
 ; GFX9-NEXT:    v_trunc_f32_e32 v5, v5
 ; GFX9-NEXT:    v_mad_f32 v4, -v5, v0, v4
 ; GFX9-NEXT:    v_cvt_i32_f32_e32 v5, v5
-; GFX9-NEXT:    v_cmp_ge_f32_e64 s[2:3], |v4|, |v0|
-; GFX9-NEXT:    s_and_b64 s[2:3], s[2:3], exec
-; GFX9-NEXT:    s_cselect_b32 s2, s4, 0
-; GFX9-NEXT:    v_add_u32_e32 v0, s2, v5
+; GFX9-NEXT:    v_cmp_ge_f32_e64 s[0:1], |v4|, |v0|
+; GFX9-NEXT:    s_and_b64 s[0:1], s[0:1], exec
+; GFX9-NEXT:    s_cselect_b32 s0, s2, 0
+; GFX9-NEXT:    v_add_u32_e32 v0, s0, v5
 ; GFX9-NEXT:    v_and_b32_e32 v2, 0xffff, v2
 ; GFX9-NEXT:    v_lshl_or_b32 v2, v3, 16, v2
-; GFX9-NEXT:    global_store_short v1, v0, s[0:1] offset:4
-; GFX9-NEXT:    global_store_dword v1, v2, s[0:1]
+; GFX9-NEXT:    global_store_short v1, v0, s[6:7] offset:4
+; GFX9-NEXT:    global_store_dword v1, v2, s[6:7]
 ; GFX9-NEXT:    s_endpgm
   %r = sdiv <3 x i16> %x, %y
   store <3 x i16> %r, ptr addrspace(1) %out
@@ -4408,70 +4396,68 @@ define amdgpu_kernel void @srem_v3i16(ptr addrspace(1) %out, <3 x i16> %x, <3 x
 ;
 ; GFX6-LABEL: srem_v3i16:
 ; GFX6:       ; %bb.0:
-; GFX6-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0xd
-; GFX6-NEXT:    s_load_dwordx4 s[4:7], s[4:5], 0x9
+; GFX6-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0xb
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x9
 ; GFX6-NEXT:    s_mov_b32 s3, 0xf000
 ; GFX6-NEXT:    s_mov_b32 s2, -1
 ; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX6-NEXT:    s_sext_i32_i16 s0, s8
-; GFX6-NEXT:    v_cvt_f32_i32_e32 v0, s0
-; GFX6-NEXT:    s_sext_i32_i16 s1, s6
-; GFX6-NEXT:    v_cvt_f32_i32_e32 v1, s1
-; GFX6-NEXT:    s_xor_b32 s0, s1, s0
+; GFX6-NEXT:    s_sext_i32_i16 s4, s10
+; GFX6-NEXT:    v_cvt_f32_i32_e32 v0, s4
+; GFX6-NEXT:    s_sext_i32_i16 s5, s8
+; GFX6-NEXT:    v_cvt_f32_i32_e32 v1, s5
+; GFX6-NEXT:    s_xor_b32 s4, s5, s4
 ; GFX6-NEXT:    v_rcp_iflag_f32_e32 v2, v0
-; GFX6-NEXT:    s_ashr_i32 s0, s0, 30
-; GFX6-NEXT:    s_or_b32 s10, s0, 1
+; GFX6-NEXT:    s_ashr_i32 s4, s4, 30
+; GFX6-NEXT:    s_or_b32 s6, s4, 1
 ; GFX6-NEXT:    v_mul_f32_e32 v2, v1, v2
 ; GFX6-NEXT:    v_trunc_f32_e32 v2, v2
 ; GFX6-NEXT:    v_mad_f32 v1, -v2, v0, v1
 ; GFX6-NEXT:    v_cvt_i32_f32_e32 v2, v2
-; GFX6-NEXT:    v_cmp_ge_f32_e64 s[0:1], |v1|, |v0|
-; GFX6-NEXT:    s_and_b64 s[0:1], s[0:1], exec
-; GFX6-NEXT:    s_cselect_b32 s0, s10, 0
-; GFX6-NEXT:    v_add_i32_e32 v0, vcc, s0, v2
-; GFX6-NEXT:    s_mov_b32 s0, s4
-; GFX6-NEXT:    s_ashr_i32 s4, s8, 16
+; GFX6-NEXT:    v_cmp_ge_f32_e64 s[4:5], |v1|, |v0|
+; GFX6-NEXT:    s_and_b64 s[4:5], s[4:5], exec
+; GFX6-NEXT:    s_cselect_b32 s4, s6, 0
+; GFX6-NEXT:    v_add_i32_e32 v0, vcc, s4, v2
+; GFX6-NEXT:    s_ashr_i32 s4, s10, 16
 ; GFX6-NEXT:    v_cvt_f32_i32_e32 v1, s4
-; GFX6-NEXT:    s_mov_b32 s1, s5
-; GFX6-NEXT:    s_ashr_i32 s5, s6, 16
+; GFX6-NEXT:    s_ashr_i32 s5, s8, 16
 ; GFX6-NEXT:    v_cvt_f32_i32_e32 v2, s5
+; GFX6-NEXT:    v_mul_lo_u32 v0, v0, s10
 ; GFX6-NEXT:    v_rcp_iflag_f32_e32 v3, v1
-; GFX6-NEXT:    v_mul_lo_u32 v0, v0, s8
 ; GFX6-NEXT:    s_xor_b32 s4, s5, s4
 ; GFX6-NEXT:    s_ashr_i32 s4, s4, 30
+; GFX6-NEXT:    s_lshr_b32 s6, s8, 16
 ; GFX6-NEXT:    v_mul_f32_e32 v3, v2, v3
 ; GFX6-NEXT:    v_trunc_f32_e32 v3, v3
 ; GFX6-NEXT:    v_mad_f32 v2, -v3, v1, v2
 ; GFX6-NEXT:    v_cvt_i32_f32_e32 v3, v3
-; GFX6-NEXT:    v_sub_i32_e32 v0, vcc, s6, v0
-; GFX6-NEXT:    s_lshr_b32 s10, s6, 16
-; GFX6-NEXT:    s_lshr_b32 s8, s8, 16
-; GFX6-NEXT:    s_or_b32 s6, s4, 1
+; GFX6-NEXT:    v_sub_i32_e32 v0, vcc, s8, v0
+; GFX6-NEXT:    s_lshr_b32 s7, s10, 16
+; GFX6-NEXT:    s_or_b32 s8, s4, 1
 ; GFX6-NEXT:    v_cmp_ge_f32_e64 s[4:5], |v2|, |v1|
 ; GFX6-NEXT:    s_and_b64 s[4:5], s[4:5], exec
-; GFX6-NEXT:    s_cselect_b32 s4, s6, 0
+; GFX6-NEXT:    s_cselect_b32 s4, s8, 0
 ; GFX6-NEXT:    v_add_i32_e32 v1, vcc, s4, v3
-; GFX6-NEXT:    s_sext_i32_i16 s4, s9
+; GFX6-NEXT:    s_sext_i32_i16 s4, s11
 ; GFX6-NEXT:    v_cvt_f32_i32_e32 v2, s4
-; GFX6-NEXT:    s_sext_i32_i16 s5, s7
+; GFX6-NEXT:    s_sext_i32_i16 s5, s9
 ; GFX6-NEXT:    v_cvt_f32_i32_e32 v3, s5
 ; GFX6-NEXT:    s_xor_b32 s4, s5, s4
 ; GFX6-NEXT:    v_rcp_iflag_f32_e32 v4, v2
 ; GFX6-NEXT:    s_ashr_i32 s4, s4, 30
-; GFX6-NEXT:    s_or_b32 s6, s4, 1
-; GFX6-NEXT:    v_mul_lo_u32 v1, v1, s8
+; GFX6-NEXT:    v_mul_lo_u32 v1, v1, s7
+; GFX6-NEXT:    s_or_b32 s7, s4, 1
 ; GFX6-NEXT:    v_mul_f32_e32 v4, v3, v4
 ; GFX6-NEXT:    v_trunc_f32_e32 v4, v4
 ; GFX6-NEXT:    v_mad_f32 v3, -v4, v2, v3
 ; GFX6-NEXT:    v_cvt_i32_f32_e32 v4, v4
 ; GFX6-NEXT:    v_cmp_ge_f32_e64 s[4:5], |v3|, |v2|
 ; GFX6-NEXT:    s_and_b64 s[4:5], s[4:5], exec
-; GFX6-NEXT:    s_cselect_b32 s4, s6, 0
+; GFX6-NEXT:    s_cselect_b32 s4, s7, 0
 ; GFX6-NEXT:    v_add_i32_e32 v2, vcc, s4, v4
-; GFX6-NEXT:    v_mul_lo_u32 v2, v2, s9
-; GFX6-NEXT:    v_sub_i32_e32 v1, vcc, s10, v1
+; GFX6-NEXT:    v_mul_lo_u32 v2, v2, s11
+; GFX6-NEXT:    v_sub_i32_e32 v1, vcc, s6, v1
 ; GFX6-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
-; GFX6-NEXT:    v_sub_i32_e32 v2, vcc, s7, v2
+; GFX6-NEXT:    v_sub_i32_e32 v2, vcc, s9, v2
 ; GFX6-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX6-NEXT:    v_or_b32_e32 v0, v0, v1
 ; GFX6-NEXT:    buffer_store_short v2, off, s[0:3], 0 offset:4
@@ -4480,12 +4466,12 @@ define amdgpu_kernel void @srem_v3i16(ptr addrspace(1) %out, <3 x i16> %x, <3 x
 ;
 ; GFX9-LABEL: srem_v3i16:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
-; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x2c
+; GFX9-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x24
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    s_sext_i32_i16 s8, s6
+; GFX9-NEXT:    s_sext_i32_i16 s8, s2
 ; GFX9-NEXT:    v_cvt_f32_i32_e32 v0, s8
-; GFX9-NEXT:    s_sext_i32_i16 s9, s2
+; GFX9-NEXT:    s_sext_i32_i16 s9, s0
 ; GFX9-NEXT:    v_cvt_f32_i32_e32 v1, s9
 ; GFX9-NEXT:    s_xor_b32 s4, s9, s8
 ; GFX9-NEXT:    v_rcp_iflag_f32_e32 v2, v0
@@ -4497,51 +4483,51 @@ define amdgpu_kernel void @srem_v3i16(ptr addrspace(1) %out, <3 x i16> %x, <3 x
 ; GFX9-NEXT:    v_cmp_ge_f32_e64 s[4:5], |v1|, |v0|
 ; GFX9-NEXT:    s_and_b64 s[4:5], s[4:5], exec
 ; GFX9-NEXT:    s_cselect_b32 s4, s10, 0
-; GFX9-NEXT:    s_ashr_i32 s10, s2, 16
-; GFX9-NEXT:    s_ashr_i32 s2, s6, 16
+; GFX9-NEXT:    s_ashr_i32 s10, s0, 16
+; GFX9-NEXT:    s_ashr_i32 s0, s2, 16
 ; GFX9-NEXT:    v_cvt_i32_f32_e32 v2, v2
-; GFX9-NEXT:    v_cvt_f32_i32_e32 v0, s2
+; GFX9-NEXT:    v_cvt_f32_i32_e32 v0, s0
+; GFX9-NEXT:    s_xor_b32 s2, s10, s0
+; GFX9-NEXT:    s_ashr_i32 s2, s2, 30
 ; GFX9-NEXT:    v_add_u32_e32 v1, s4, v2
 ; GFX9-NEXT:    v_cvt_f32_i32_e32 v2, s10
 ; GFX9-NEXT:    v_rcp_iflag_f32_e32 v3, v0
-; GFX9-NEXT:    s_xor_b32 s4, s10, s2
-; GFX9-NEXT:    s_ashr_i32 s4, s4, 30
-; GFX9-NEXT:    s_or_b32 s6, s4, 1
+; GFX9-NEXT:    s_or_b32 s2, s2, 1
+; GFX9-NEXT:    v_mul_lo_u32 v1, v1, s8
 ; GFX9-NEXT:    v_mul_f32_e32 v3, v2, v3
 ; GFX9-NEXT:    v_trunc_f32_e32 v3, v3
 ; GFX9-NEXT:    v_mad_f32 v2, -v3, v0, v2
 ; GFX9-NEXT:    v_cvt_i32_f32_e32 v3, v3
 ; GFX9-NEXT:    v_cmp_ge_f32_e64 s[4:5], |v2|, |v0|
 ; GFX9-NEXT:    s_and_b64 s[4:5], s[4:5], exec
-; GFX9-NEXT:    s_cselect_b32 s4, s6, 0
-; GFX9-NEXT:    v_add_u32_e32 v0, s4, v3
-; GFX9-NEXT:    s_sext_i32_i16 s4, s7
-; GFX9-NEXT:    v_cvt_f32_i32_e32 v2, s4
-; GFX9-NEXT:    s_sext_i32_i16 s5, s3
-; GFX9-NEXT:    v_cvt_f32_i32_e32 v3, s5
-; GFX9-NEXT:    v_mul_lo_u32 v0, v0, s2
+; GFX9-NEXT:    s_cselect_b32 s2, s2, 0
+; GFX9-NEXT:    v_add_u32_e32 v0, s2, v3
+; GFX9-NEXT:    s_sext_i32_i16 s2, s3
+; GFX9-NEXT:    v_cvt_f32_i32_e32 v2, s2
+; GFX9-NEXT:    s_sext_i32_i16 s3, s1
+; GFX9-NEXT:    v_cvt_f32_i32_e32 v3, s3
+; GFX9-NEXT:    v_mul_lo_u32 v0, v0, s0
 ; GFX9-NEXT:    v_rcp_iflag_f32_e32 v4, v2
-; GFX9-NEXT:    s_xor_b32 s2, s5, s4
-; GFX9-NEXT:    s_ashr_i32 s2, s2, 30
-; GFX9-NEXT:    s_or_b32 s6, s2, 1
+; GFX9-NEXT:    s_xor_b32 s0, s3, s2
+; GFX9-NEXT:    s_ashr_i32 s0, s0, 30
+; GFX9-NEXT:    s_or_b32 s4, s0, 1
 ; GFX9-NEXT:    v_mul_f32_e32 v4, v3, v4
 ; GFX9-NEXT:    v_trunc_f32_e32 v4, v4
 ; GFX9-NEXT:    v_mad_f32 v3, -v4, v2, v3
 ; GFX9-NEXT:    v_cvt_i32_f32_e32 v4, v4
-; GFX9-NEXT:    v_cmp_ge_f32_e64 s[2:3], |v3|, |v2|
-; GFX9-NEXT:    s_and_b64 s[2:3], s[2:3], exec
-; GFX9-NEXT:    s_cselect_b32 s2, s6, 0
-; GFX9-NEXT:    v_mul_lo_u32 v1, v1, s8
-; GFX9-NEXT:    v_add_u32_e32 v2, s2, v4
-; GFX9-NEXT:    v_mul_lo_u32 v2, v2, s4
-; GFX9-NEXT:    v_mov_b32_e32 v3, 0
+; GFX9-NEXT:    v_cmp_ge_f32_e64 s[0:1], |v3|, |v2|
+; GFX9-NEXT:    s_and_b64 s[0:1], s[0:1], exec
+; GFX9-NEXT:    s_cselect_b32 s0, s4, 0
+; GFX9-NEXT:    v_add_u32_e32 v2, s0, v4
+; GFX9-NEXT:    v_mul_lo_u32 v2, v2, s2
 ; GFX9-NEXT:    v_sub_u32_e32 v1, s9, v1
+; GFX9-NEXT:    v_mov_b32_e32 v3, 0
 ; GFX9-NEXT:    v_sub_u32_e32 v0, s10, v0
-; GFX9-NEXT:    v_sub_u32_e32 v2, s5, v2
+; GFX9-NEXT:    v_sub_u32_e32 v2, s3, v2
 ; GFX9-NEXT:    v_and_b32_e32 v1, 0xffff, v1
 ; GFX9-NEXT:    v_lshl_or_b32 v0, v0, 16, v1
-; GFX9-NEXT:    global_store_short v3, v2, s[0:1] offset:4
-; GFX9-NEXT:    global_store_dword v3, v0, s[0:1]
+; GFX9-NEXT:    global_store_short v3, v2, s[6:7] offset:4
+; GFX9-NEXT:    global_store_dword v3, v0, s[6:7]
 ; GFX9-NEXT:    s_endpgm
   %r = srem <3 x i16> %x, %y
   store <3 x i16> %r, ptr addrspace(1) %out
@@ -5512,16 +5498,15 @@ define amdgpu_kernel void @udiv_v2i32_pow2k_denom(ptr addrspace(1) %out, <2 x i3
 ; GFX6-LABEL: udiv_v2i32_pow2k_denom:
 ; GFX6:       ; %bb.0:
 ; GFX6-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
-; GFX6-NEXT:    s_mov_b32 s7, 0xf000
-; GFX6-NEXT:    s_mov_b32 s6, -1
 ; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX6-NEXT:    s_mov_b32 s4, s0
-; GFX6-NEXT:    s_mov_b32 s5, s1
-; GFX6-NEXT:    s_lshr_b32 s0, s2, 12
-; GFX6-NEXT:    s_lshr_b32 s1, s3, 12
-; GFX6-NEXT:    v_mov_b32_e32 v0, s0
-; GFX6-NEXT:    v_mov_b32_e32 v1, s1
-; GFX6-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; GFX6-NEXT:    s_mov_b64 s[4:5], s[2:3]
+; GFX6-NEXT:    s_lshr_b32 s4, s4, 12
+; GFX6-NEXT:    s_lshr_b32 s5, s5, 12
+; GFX6-NEXT:    s_mov_b32 s3, 0xf000
+; GFX6-NEXT:    s_mov_b32 s2, -1
+; GFX6-NEXT:    v_mov_b32_e32 v0, s4
+; GFX6-NEXT:    v_mov_b32_e32 v1, s5
+; GFX6-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
 ; GFX6-NEXT:    s_endpgm
 ;
 ; GFX9-LABEL: udiv_v2i32_pow2k_denom:
@@ -5555,19 +5540,18 @@ define amdgpu_kernel void @udiv_v2i32_mixed_pow2k_denom(ptr addrspace(1) %out, <
 ; GFX6:       ; %bb.0:
 ; GFX6-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
 ; GFX6-NEXT:    v_mov_b32_e32 v0, 0x100101
-; GFX6-NEXT:    s_mov_b32 s7, 0xf000
-; GFX6-NEXT:    s_mov_b32 s6, -1
 ; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX6-NEXT:    v_mul_hi_u32 v0, s3, v0
-; GFX6-NEXT:    s_mov_b32 s4, s0
-; GFX6-NEXT:    s_lshr_b32 s0, s2, 12
-; GFX6-NEXT:    s_mov_b32 s5, s1
-; GFX6-NEXT:    v_sub_i32_e32 v1, vcc, s3, v0
+; GFX6-NEXT:    s_mov_b64 s[4:5], s[2:3]
+; GFX6-NEXT:    v_mul_hi_u32 v0, s5, v0
+; GFX6-NEXT:    s_lshr_b32 s4, s4, 12
+; GFX6-NEXT:    s_mov_b32 s3, 0xf000
+; GFX6-NEXT:    s_mov_b32 s2, -1
+; GFX6-NEXT:    v_sub_i32_e32 v1, vcc, s5, v0
 ; GFX6-NEXT:    v_lshrrev_b32_e32 v1, 1, v1
 ; GFX6-NEXT:    v_add_i32_e32 v0, vcc, v1, v0
 ; GFX6-NEXT:    v_lshrrev_b32_e32 v1, 11, v0
-; GFX6-NEXT:    v_mov_b32_e32 v0, s0
-; GFX6-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; GFX6-NEXT:    v_mov_b32_e32 v0, s4
+; GFX6-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
 ; GFX6-NEXT:    s_endpgm
 ;
 ; GFX9-LABEL: udiv_v2i32_mixed_pow2k_denom:
@@ -5662,31 +5646,29 @@ define amdgpu_kernel void @udiv_v2i32_pow2_shl_denom(ptr addrspace(1) %out, <2 x
 ;
 ; GFX6-LABEL: udiv_v2i32_pow2_shl_denom:
 ; GFX6:       ; %bb.0:
-; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0xd
-; GFX6-NEXT:    s_load_dwordx4 s[4:7], s[4:5], 0x9
-; GFX6-NEXT:    s_mov_b32 s11, 0xf000
-; GFX6-NEXT:    s_mov_b32 s10, -1
+; GFX6-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0xb
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x9
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
 ; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX6-NEXT:    s_lshl_b32 s0, 0x1000, s0
+; GFX6-NEXT:    s_lshl_b32 s0, 0x1000, s10
 ; GFX6-NEXT:    v_cvt_f32_u32_e32 v0, s0
-; GFX6-NEXT:    s_sub_i32 s2, 0, s0
-; GFX6-NEXT:    s_mov_b32 s8, s4
-; GFX6-NEXT:    s_mov_b32 s9, s5
+; GFX6-NEXT:    s_sub_i32 s1, 0, s0
+; GFX6-NEXT:    s_lshl_b32 s2, 0x1000, s11
+; GFX6-NEXT:    v_cvt_f32_u32_e32 v2, s2
 ; GFX6-NEXT:    v_rcp_iflag_f32_e32 v0, v0
+; GFX6-NEXT:    v_rcp_iflag_f32_e32 v2, v2
 ; GFX6-NEXT:    v_mul_f32_e32 v0, 0x4f7ffffe, v0
 ; GFX6-NEXT:    v_cvt_u32_f32_e32 v0, v0
-; GFX6-NEXT:    v_mul_lo_u32 v1, s2, v0
-; GFX6-NEXT:    s_lshl_b32 s2, 0x1000, s1
-; GFX6-NEXT:    v_cvt_f32_u32_e32 v2, s2
+; GFX6-NEXT:    v_mul_lo_u32 v1, s1, v0
 ; GFX6-NEXT:    v_mul_hi_u32 v1, v0, v1
-; GFX6-NEXT:    v_rcp_iflag_f32_e32 v2, v2
 ; GFX6-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
-; GFX6-NEXT:    v_mul_hi_u32 v0, s6, v0
+; GFX6-NEXT:    v_mul_hi_u32 v0, s8, v0
 ; GFX6-NEXT:    v_mul_f32_e32 v1, 0x4f7ffffe, v2
 ; GFX6-NEXT:    v_cvt_u32_f32_e32 v1, v1
 ; GFX6-NEXT:    v_readfirstlane_b32 s1, v0
 ; GFX6-NEXT:    s_mul_i32 s1, s1, s0
-; GFX6-NEXT:    s_sub_i32 s1, s6, s1
+; GFX6-NEXT:    s_sub_i32 s1, s8, s1
 ; GFX6-NEXT:    s_sub_i32 s3, s1, s0
 ; GFX6-NEXT:    s_cmp_ge_u32 s1, s0
 ; GFX6-NEXT:    v_add_i32_e32 v2, vcc, 1, v0
@@ -5701,10 +5683,10 @@ define amdgpu_kernel void @udiv_v2i32_pow2_shl_denom(ptr addrspace(1) %out, <2 x
 ; GFX6-NEXT:    v_mul_hi_u32 v3, v1, v3
 ; GFX6-NEXT:    v_cndmask_b32_e64 v0, v0, v2, s[0:1]
 ; GFX6-NEXT:    v_add_i32_e32 v1, vcc, v1, v3
-; GFX6-NEXT:    v_mul_hi_u32 v1, s7, v1
+; GFX6-NEXT:    v_mul_hi_u32 v1, s9, v1
 ; GFX6-NEXT:    v_readfirstlane_b32 s0, v1
 ; GFX6-NEXT:    s_mul_i32 s0, s0, s2
-; GFX6-NEXT:    s_sub_i32 s0, s7, s0
+; GFX6-NEXT:    s_sub_i32 s0, s9, s0
 ; GFX6-NEXT:    s_sub_i32 s1, s0, s2
 ; GFX6-NEXT:    v_add_i32_e32 v2, vcc, 1, v1
 ; GFX6-NEXT:    s_cmp_ge_u32 s0, s2
@@ -5715,19 +5697,19 @@ define amdgpu_kernel void @udiv_v2i32_pow2_shl_denom(ptr addrspace(1) %out, <2 x
 ; GFX6-NEXT:    s_cmp_ge_u32 s0, s2
 ; GFX6-NEXT:    s_cselect_b64 vcc, -1, 0
 ; GFX6-NEXT:    v_cndmask_b32_e32 v1, v1, v2, vcc
-; GFX6-NEXT:    buffer_store_dwordx2 v[0:1], off, s[8:11], 0
+; GFX6-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
 ; GFX6-NEXT:    s_endpgm
 ;
 ; GFX9-LABEL: udiv_v2i32_pow2_shl_denom:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x34
+; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x2c
 ; GFX9-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    s_lshl_b32 s7, 0x1000, s0
+; GFX9-NEXT:    s_lshl_b32 s7, 0x1000, s2
 ; GFX9-NEXT:    v_cvt_f32_u32_e32 v0, s7
-; GFX9-NEXT:    s_lshl_b32 s6, 0x1000, s1
+; GFX9-NEXT:    s_lshl_b32 s6, 0x1000, s3
 ; GFX9-NEXT:    v_cvt_f32_u32_e32 v1, s6
-; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX9-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x24
 ; GFX9-NEXT:    v_rcp_iflag_f32_e32 v0, v0
 ; GFX9-NEXT:    s_sub_i32 s4, 0, s7
 ; GFX9-NEXT:    v_rcp_iflag_f32_e32 v1, v1
@@ -5739,37 +5721,37 @@ define amdgpu_kernel void @udiv_v2i32_pow2_shl_denom(ptr addrspace(1) %out, <2 x
 ; GFX9-NEXT:    s_mul_i32 s4, s4, s5
 ; GFX9-NEXT:    s_mul_hi_u32 s4, s5, s4
 ; GFX9-NEXT:    s_add_i32 s5, s5, s4
-; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    s_mul_hi_u32 s4, s2, s5
+; GFX9-NEXT:    s_mul_hi_u32 s4, s0, s5
 ; GFX9-NEXT:    s_mul_i32 s5, s4, s7
-; GFX9-NEXT:    s_sub_i32 s2, s2, s5
+; GFX9-NEXT:    s_sub_i32 s0, s0, s5
 ; GFX9-NEXT:    s_add_i32 s9, s4, 1
-; GFX9-NEXT:    s_sub_i32 s5, s2, s7
-; GFX9-NEXT:    s_cmp_ge_u32 s2, s7
+; GFX9-NEXT:    s_sub_i32 s5, s0, s7
+; GFX9-NEXT:    s_cmp_ge_u32 s0, s7
 ; GFX9-NEXT:    s_cselect_b32 s4, s9, s4
-; GFX9-NEXT:    s_cselect_b32 s2, s5, s2
+; GFX9-NEXT:    s_cselect_b32 s0, s5, s0
 ; GFX9-NEXT:    s_add_i32 s5, s4, 1
-; GFX9-NEXT:    s_cmp_ge_u32 s2, s7
+; GFX9-NEXT:    s_cmp_ge_u32 s0, s7
 ; GFX9-NEXT:    v_readfirstlane_b32 s8, v1
-; GFX9-NEXT:    s_cselect_b32 s2, s5, s4
+; GFX9-NEXT:    s_cselect_b32 s0, s5, s4
 ; GFX9-NEXT:    s_sub_i32 s4, 0, s6
 ; GFX9-NEXT:    s_mul_i32 s4, s4, s8
 ; GFX9-NEXT:    s_mul_hi_u32 s4, s8, s4
 ; GFX9-NEXT:    s_add_i32 s8, s8, s4
-; GFX9-NEXT:    s_mul_hi_u32 s4, s3, s8
+; GFX9-NEXT:    s_mul_hi_u32 s4, s1, s8
 ; GFX9-NEXT:    s_mul_i32 s5, s4, s6
-; GFX9-NEXT:    s_sub_i32 s3, s3, s5
+; GFX9-NEXT:    s_sub_i32 s1, s1, s5
 ; GFX9-NEXT:    s_add_i32 s7, s4, 1
-; GFX9-NEXT:    s_sub_i32 s5, s3, s6
-; GFX9-NEXT:    s_cmp_ge_u32 s3, s6
+; GFX9-NEXT:    s_sub_i32 s5, s1, s6
+; GFX9-NEXT:    s_cmp_ge_u32 s1, s6
 ; GFX9-NEXT:    s_cselect_b32 s4, s7, s4
-; GFX9-NEXT:    s_cselect_b32 s3, s5, s3
+; GFX9-NEXT:    s_cselect_b32 s1, s5, s1
 ; GFX9-NEXT:    s_add_i32 s5, s4, 1
-; GFX9-NEXT:    s_cmp_ge_u32 s3, s6
-; GFX9-NEXT:    s_cselect_b32 s3, s5, s4
-; GFX9-NEXT:    v_mov_b32_e32 v0, s2
-; GFX9-NEXT:    v_mov_b32_e32 v1, s3
-; GFX9-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX9-NEXT:    s_cmp_ge_u32 s1, s6
+; GFX9-NEXT:    s_cselect_b32 s1, s5, s4
+; GFX9-NEXT:    v_mov_b32_e32 v0, s0
+; GFX9-NEXT:    v_mov_b32_e32 v1, s1
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    global_store_dwordx2 v2, v[0:1], s[2:3]
 ; GFX9-NEXT:    s_endpgm
   %shl.y = shl <2 x i32> <i32 4096, i32 4096>, %y
   %r = udiv <2 x i32> %x, %shl.y
@@ -5908,16 +5890,15 @@ define amdgpu_kernel void @urem_v2i32_pow2k_denom(ptr addrspace(1) %out, <2 x i3
 ; GFX6-LABEL: urem_v2i32_pow2k_denom:
 ; GFX6:       ; %bb.0:
 ; GFX6-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
-; GFX6-NEXT:    s_mov_b32 s7, 0xf000
-; GFX6-NEXT:    s_mov_b32 s6, -1
 ; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX6-NEXT:    s_mov_b32 s4, s0
-; GFX6-NEXT:    s_mov_b32 s5, s1
-; GFX6-NEXT:    s_and_b32 s0, s2, 0xfff
-; GFX6-NEXT:    s_and_b32 s1, s3, 0xfff
-; GFX6-NEXT:    v_mov_b32_e32 v0, s0
-; GFX6-NEXT:    v_mov_b32_e32 v1, s1
-; GFX6-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; GFX6-NEXT:    s_mov_b64 s[4:5], s[2:3]
+; GFX6-NEXT:    s_and_b32 s4, s4, 0xfff
+; GFX6-NEXT:    s_and_b32 s5, s5, 0xfff
+; GFX6-NEXT:    s_mov_b32 s3, 0xf000
+; GFX6-NEXT:    s_mov_b32 s2, -1
+; GFX6-NEXT:    v_mov_b32_e32 v0, s4
+; GFX6-NEXT:    v_mov_b32_e32 v1, s5
+; GFX6-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
 ; GFX6-NEXT:    s_endpgm
 ;
 ; GFX9-LABEL: urem_v2i32_pow2k_denom:
@@ -6004,67 +5985,64 @@ define amdgpu_kernel void @urem_v2i32_pow2_shl_denom(ptr addrspace(1) %out, <2 x
 ;
 ; GFX6-LABEL: urem_v2i32_pow2_shl_denom:
 ; GFX6:       ; %bb.0:
-; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0xd
+; GFX6-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0xb
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x9
 ; GFX6-NEXT:    s_mov_b32 s7, 0xf000
 ; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX6-NEXT:    s_lshl_b32 s6, 0x1000, s0
-; GFX6-NEXT:    v_cvt_f32_u32_e32 v0, s6
-; GFX6-NEXT:    s_sub_i32 s0, 0, s6
-; GFX6-NEXT:    s_lshl_b32 s8, 0x1000, s1
-; GFX6-NEXT:    v_cvt_f32_u32_e32 v2, s8
+; GFX6-NEXT:    s_lshl_b32 s2, 0x1000, s2
+; GFX6-NEXT:    v_cvt_f32_u32_e32 v0, s2
+; GFX6-NEXT:    s_sub_i32 s6, 0, s2
+; GFX6-NEXT:    s_lshl_b32 s3, 0x1000, s3
+; GFX6-NEXT:    v_cvt_f32_u32_e32 v2, s3
 ; GFX6-NEXT:    v_rcp_iflag_f32_e32 v0, v0
 ; GFX6-NEXT:    v_rcp_iflag_f32_e32 v2, v2
 ; GFX6-NEXT:    v_mul_f32_e32 v0, 0x4f7ffffe, v0
 ; GFX6-NEXT:    v_cvt_u32_f32_e32 v0, v0
-; GFX6-NEXT:    v_mul_lo_u32 v1, s0, v0
-; GFX6-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GFX6-NEXT:    v_mul_lo_u32 v1, s6, v0
 ; GFX6-NEXT:    v_mul_hi_u32 v1, v0, v1
-; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX6-NEXT:    s_mov_b32 s5, s1
 ; GFX6-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
-; GFX6-NEXT:    v_mul_hi_u32 v0, s2, v0
+; GFX6-NEXT:    v_mul_hi_u32 v0, s0, v0
 ; GFX6-NEXT:    v_mul_f32_e32 v1, 0x4f7ffffe, v2
 ; GFX6-NEXT:    v_cvt_u32_f32_e32 v1, v1
-; GFX6-NEXT:    v_readfirstlane_b32 s4, v0
-; GFX6-NEXT:    s_mul_i32 s4, s4, s6
-; GFX6-NEXT:    s_sub_i32 s2, s2, s4
-; GFX6-NEXT:    s_sub_i32 s4, s2, s6
-; GFX6-NEXT:    s_cmp_ge_u32 s2, s6
-; GFX6-NEXT:    s_cselect_b32 s2, s4, s2
-; GFX6-NEXT:    s_sub_i32 s4, s2, s6
-; GFX6-NEXT:    s_cmp_ge_u32 s2, s6
-; GFX6-NEXT:    s_cselect_b32 s2, s4, s2
-; GFX6-NEXT:    s_sub_i32 s4, 0, s8
-; GFX6-NEXT:    v_mul_lo_u32 v0, s4, v1
-; GFX6-NEXT:    s_mov_b32 s4, s0
+; GFX6-NEXT:    v_readfirstlane_b32 s6, v0
+; GFX6-NEXT:    s_mul_i32 s6, s6, s2
+; GFX6-NEXT:    s_sub_i32 s0, s0, s6
+; GFX6-NEXT:    s_sub_i32 s6, s0, s2
+; GFX6-NEXT:    s_cmp_ge_u32 s0, s2
+; GFX6-NEXT:    s_cselect_b32 s0, s6, s0
+; GFX6-NEXT:    s_sub_i32 s6, s0, s2
+; GFX6-NEXT:    s_cmp_ge_u32 s0, s2
+; GFX6-NEXT:    s_cselect_b32 s0, s6, s0
+; GFX6-NEXT:    s_sub_i32 s2, 0, s3
+; GFX6-NEXT:    v_mul_lo_u32 v0, s2, v1
 ; GFX6-NEXT:    s_mov_b32 s6, -1
 ; GFX6-NEXT:    v_mul_hi_u32 v0, v1, v0
 ; GFX6-NEXT:    v_add_i32_e32 v0, vcc, v1, v0
-; GFX6-NEXT:    v_mul_hi_u32 v0, s3, v0
-; GFX6-NEXT:    v_readfirstlane_b32 s0, v0
-; GFX6-NEXT:    s_mul_i32 s0, s0, s8
-; GFX6-NEXT:    s_sub_i32 s0, s3, s0
-; GFX6-NEXT:    s_sub_i32 s1, s0, s8
-; GFX6-NEXT:    s_cmp_ge_u32 s0, s8
-; GFX6-NEXT:    s_cselect_b32 s0, s1, s0
-; GFX6-NEXT:    s_sub_i32 s1, s0, s8
-; GFX6-NEXT:    s_cmp_ge_u32 s0, s8
-; GFX6-NEXT:    s_cselect_b32 s0, s1, s0
-; GFX6-NEXT:    v_mov_b32_e32 v0, s2
-; GFX6-NEXT:    v_mov_b32_e32 v1, s0
+; GFX6-NEXT:    v_mul_hi_u32 v0, s1, v0
+; GFX6-NEXT:    v_readfirstlane_b32 s2, v0
+; GFX6-NEXT:    s_mul_i32 s2, s2, s3
+; GFX6-NEXT:    s_sub_i32 s1, s1, s2
+; GFX6-NEXT:    s_sub_i32 s2, s1, s3
+; GFX6-NEXT:    s_cmp_ge_u32 s1, s3
+; GFX6-NEXT:    s_cselect_b32 s1, s2, s1
+; GFX6-NEXT:    s_sub_i32 s2, s1, s3
+; GFX6-NEXT:    s_cmp_ge_u32 s1, s3
+; GFX6-NEXT:    s_cselect_b32 s1, s2, s1
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX6-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
 ; GFX6-NEXT:    s_endpgm
 ;
 ; GFX9-LABEL: urem_v2i32_pow2_shl_denom:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x34
+; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x2c
 ; GFX9-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    s_lshl_b32 s7, 0x1000, s0
+; GFX9-NEXT:    s_lshl_b32 s7, 0x1000, s2
 ; GFX9-NEXT:    v_cvt_f32_u32_e32 v0, s7
-; GFX9-NEXT:    s_lshl_b32 s6, 0x1000, s1
+; GFX9-NEXT:    s_lshl_b32 s6, 0x1000, s3
 ; GFX9-NEXT:    v_cvt_f32_u32_e32 v1, s6
-; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX9-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x24
 ; GFX9-NEXT:    v_rcp_iflag_f32_e32 v0, v0
 ; GFX9-NEXT:    s_sub_i32 s4, 0, s7
 ; GFX9-NEXT:    v_rcp_iflag_f32_e32 v1, v1
@@ -6076,33 +6054,33 @@ define amdgpu_kernel void @urem_v2i32_pow2_shl_denom(ptr addrspace(1) %out, <2 x
 ; GFX9-NEXT:    s_mul_i32 s4, s4, s5
 ; GFX9-NEXT:    s_mul_hi_u32 s4, s5, s4
 ; GFX9-NEXT:    s_add_i32 s5, s5, s4
-; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    s_mul_hi_u32 s4, s2, s5
+; GFX9-NEXT:    s_mul_hi_u32 s4, s0, s5
 ; GFX9-NEXT:    s_mul_i32 s4, s4, s7
-; GFX9-NEXT:    s_sub_i32 s2, s2, s4
-; GFX9-NEXT:    s_sub_i32 s4, s2, s7
-; GFX9-NEXT:    s_cmp_ge_u32 s2, s7
-; GFX9-NEXT:    s_cselect_b32 s2, s4, s2
-; GFX9-NEXT:    s_sub_i32 s4, s2, s7
-; GFX9-NEXT:    s_cmp_ge_u32 s2, s7
+; GFX9-NEXT:    s_sub_i32 s0, s0, s4
+; GFX9-NEXT:    s_sub_i32 s4, s0, s7
+; GFX9-NEXT:    s_cmp_ge_u32 s0, s7
+; GFX9-NEXT:    s_cselect_b32 s0, s4, s0
+; GFX9-NEXT:    s_sub_i32 s4, s0, s7
+; GFX9-NEXT:    s_cmp_ge_u32 s0, s7
 ; GFX9-NEXT:    v_readfirstlane_b32 s8, v1
-; GFX9-NEXT:    s_cselect_b32 s2, s4, s2
+; GFX9-NEXT:    s_cselect_b32 s0, s4, s0
 ; GFX9-NEXT:    s_sub_i32 s4, 0, s6
 ; GFX9-NEXT:    s_mul_i32 s4, s4, s8
 ; GFX9-NEXT:    s_mul_hi_u32 s4, s8, s4
 ; GFX9-NEXT:    s_add_i32 s8, s8, s4
-; GFX9-NEXT:    s_mul_hi_u32 s4, s3, s8
+; GFX9-NEXT:    s_mul_hi_u32 s4, s1, s8
 ; GFX9-NEXT:    s_mul_i32 s4, s4, s6
-; GFX9-NEXT:    s_sub_i32 s3, s3, s4
-; GFX9-NEXT:    s_sub_i32 s4, s3, s6
-; GFX9-NEXT:    s_cmp_ge_u32 s3, s6
-; GFX9-NEXT:    s_cselect_b32 s3, s4, s3
-; GFX9-NEXT:    s_sub_i32 s4, s3, s6
-; GFX9-NEXT:    s_cmp_ge_u32 s3, s6
-; GFX9-NEXT:    s_cselect_b32 s3, s4, s3
-; GFX9-NEXT:    v_mov_b32_e32 v0, s2
-; GFX9-NEXT:    v_mov_b32_e32 v1, s3
-; GFX9-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX9-NEXT:    s_sub_i32 s1, s1, s4
+; GFX9-NEXT:    s_sub_i32 s4, s1, s6
+; GFX9-NEXT:    s_cmp_ge_u32 s1, s6
+; GFX9-NEXT:    s_cselect_b32 s1, s4, s1
+; GFX9-NEXT:    s_sub_i32 s4, s1, s6
+; GFX9-NEXT:    s_cmp_ge_u32 s1, s6
+; GFX9-NEXT:    s_cselect_b32 s1, s4, s1
+; GFX9-NEXT:    v_mov_b32_e32 v0, s0
+; GFX9-NEXT:    v_mov_b32_e32 v1, s1
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    global_store_dwordx2 v2, v[0:1], s[2:3]
 ; GFX9-NEXT:    s_endpgm
   %shl.y = shl <2 x i32> <i32 4096, i32 4096>, %y
   %r = urem <2 x i32> %x, %shl.y
@@ -6291,22 +6269,21 @@ define amdgpu_kernel void @sdiv_v2i32_pow2k_denom(ptr addrspace(1) %out, <2 x i3
 ; GFX6-LABEL: sdiv_v2i32_pow2k_denom:
 ; GFX6:       ; %bb.0:
 ; GFX6-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
-; GFX6-NEXT:    s_mov_b32 s7, 0xf000
-; GFX6-NEXT:    s_mov_b32 s6, -1
 ; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX6-NEXT:    s_mov_b32 s4, s0
-; GFX6-NEXT:    s_mov_b32 s5, s1
-; GFX6-NEXT:    s_ashr_i32 s0, s2, 31
-; GFX6-NEXT:    s_ashr_i32 s1, s3, 31
-; GFX6-NEXT:    s_lshr_b32 s0, s0, 20
-; GFX6-NEXT:    s_lshr_b32 s1, s1, 20
-; GFX6-NEXT:    s_add_i32 s0, s2, s0
-; GFX6-NEXT:    s_add_i32 s1, s3, s1
-; GFX6-NEXT:    s_ashr_i32 s0, s0, 12
-; GFX6-NEXT:    s_ashr_i32 s1, s1, 12
-; GFX6-NEXT:    v_mov_b32_e32 v0, s0
-; GFX6-NEXT:    v_mov_b32_e32 v1, s1
-; GFX6-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; GFX6-NEXT:    s_mov_b64 s[4:5], s[2:3]
+; GFX6-NEXT:    s_ashr_i32 s6, s4, 31
+; GFX6-NEXT:    s_ashr_i32 s7, s5, 31
+; GFX6-NEXT:    s_lshr_b32 s6, s6, 20
+; GFX6-NEXT:    s_lshr_b32 s7, s7, 20
+; GFX6-NEXT:    s_add_i32 s4, s4, s6
+; GFX6-NEXT:    s_add_i32 s5, s5, s7
+; GFX6-NEXT:    s_ashr_i32 s4, s4, 12
+; GFX6-NEXT:    s_ashr_i32 s5, s5, 12
+; GFX6-NEXT:    s_mov_b32 s3, 0xf000
+; GFX6-NEXT:    s_mov_b32 s2, -1
+; GFX6-NEXT:    v_mov_b32_e32 v0, s4
+; GFX6-NEXT:    v_mov_b32_e32 v1, s5
+; GFX6-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
 ; GFX6-NEXT:    s_endpgm
 ;
 ; GFX9-LABEL: sdiv_v2i32_pow2k_denom:
@@ -6346,22 +6323,21 @@ define amdgpu_kernel void @ssdiv_v2i32_mixed_pow2k_denom(ptr addrspace(1) %out,
 ; GFX6:       ; %bb.0:
 ; GFX6-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
 ; GFX6-NEXT:    v_mov_b32_e32 v0, 0x80080081
-; GFX6-NEXT:    s_mov_b32 s7, 0xf000
-; GFX6-NEXT:    s_mov_b32 s6, -1
 ; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX6-NEXT:    v_mul_hi_i32 v0, s3, v0
-; GFX6-NEXT:    s_mov_b32 s4, s0
-; GFX6-NEXT:    s_ashr_i32 s0, s2, 31
-; GFX6-NEXT:    s_lshr_b32 s0, s0, 20
-; GFX6-NEXT:    s_add_i32 s0, s2, s0
-; GFX6-NEXT:    v_add_i32_e32 v0, vcc, s3, v0
-; GFX6-NEXT:    s_ashr_i32 s0, s0, 12
+; GFX6-NEXT:    s_mov_b64 s[4:5], s[2:3]
+; GFX6-NEXT:    v_mul_hi_i32 v0, s5, v0
+; GFX6-NEXT:    s_ashr_i32 s6, s4, 31
+; GFX6-NEXT:    s_lshr_b32 s6, s6, 20
+; GFX6-NEXT:    s_add_i32 s4, s4, s6
+; GFX6-NEXT:    v_add_i32_e32 v0, vcc, s5, v0
+; GFX6-NEXT:    s_ashr_i32 s4, s4, 12
 ; GFX6-NEXT:    v_lshrrev_b32_e32 v1, 31, v0
 ; GFX6-NEXT:    v_ashrrev_i32_e32 v0, 11, v0
-; GFX6-NEXT:    s_mov_b32 s5, s1
+; GFX6-NEXT:    s_mov_b32 s3, 0xf000
+; GFX6-NEXT:    s_mov_b32 s2, -1
 ; GFX6-NEXT:    v_add_i32_e32 v1, vcc, v0, v1
-; GFX6-NEXT:    v_mov_b32_e32 v0, s0
-; GFX6-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; GFX6-NEXT:    v_mov_b32_e32 v0, s4
+; GFX6-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
 ; GFX6-NEXT:    s_endpgm
 ;
 ; GFX9-LABEL: ssdiv_v2i32_mixed_pow2k_denom:
@@ -6477,138 +6453,136 @@ define amdgpu_kernel void @sdiv_v2i32_pow2_shl_denom(ptr addrspace(1) %out, <2 x
 ;
 ; GFX6-LABEL: sdiv_v2i32_pow2_shl_denom:
 ; GFX6:       ; %bb.0:
-; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0xd
+; GFX6-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0xb
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x9
 ; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX6-NEXT:    s_lshl_b32 s6, 0x1000, s6
-; GFX6-NEXT:    s_abs_i32 s8, s6
-; GFX6-NEXT:    v_cvt_f32_u32_e32 v0, s8
-; GFX6-NEXT:    s_sub_i32 s0, 0, s8
-; GFX6-NEXT:    s_lshl_b32 s9, 0x1000, s7
-; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_lshl_b32 s2, 0x1000, s2
+; GFX6-NEXT:    s_abs_i32 s6, s2
+; GFX6-NEXT:    v_cvt_f32_u32_e32 v0, s6
+; GFX6-NEXT:    s_sub_i32 s7, 0, s6
+; GFX6-NEXT:    s_lshl_b32 s3, 0x1000, s3
 ; GFX6-NEXT:    v_rcp_iflag_f32_e32 v0, v0
 ; GFX6-NEXT:    v_mul_f32_e32 v0, 0x4f7ffffe, v0
 ; GFX6-NEXT:    v_cvt_u32_f32_e32 v0, v0
-; GFX6-NEXT:    v_mul_lo_u32 v1, s0, v0
-; GFX6-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GFX6-NEXT:    v_mul_lo_u32 v1, s7, v0
+; GFX6-NEXT:    s_abs_i32 s7, s0
+; GFX6-NEXT:    s_xor_b32 s0, s0, s2
+; GFX6-NEXT:    s_ashr_i32 s0, s0, 31
 ; GFX6-NEXT:    v_mul_hi_u32 v1, v0, v1
-; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX6-NEXT:    s_abs_i32 s4, s2
-; GFX6-NEXT:    s_xor_b32 s2, s2, s6
 ; GFX6-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
-; GFX6-NEXT:    v_mul_hi_u32 v0, s4, v0
-; GFX6-NEXT:    s_ashr_i32 s2, s2, 31
-; GFX6-NEXT:    s_mov_b32 s6, -1
-; GFX6-NEXT:    v_readfirstlane_b32 s5, v0
-; GFX6-NEXT:    s_mul_i32 s5, s5, s8
-; GFX6-NEXT:    s_sub_i32 s4, s4, s5
-; GFX6-NEXT:    s_sub_i32 s5, s4, s8
+; GFX6-NEXT:    v_mul_hi_u32 v0, s7, v0
+; GFX6-NEXT:    v_readfirstlane_b32 s2, v0
+; GFX6-NEXT:    s_mul_i32 s2, s2, s6
+; GFX6-NEXT:    s_sub_i32 s2, s7, s2
+; GFX6-NEXT:    s_sub_i32 s7, s2, s6
 ; GFX6-NEXT:    v_add_i32_e32 v1, vcc, 1, v0
-; GFX6-NEXT:    s_cmp_ge_u32 s4, s8
+; GFX6-NEXT:    s_cmp_ge_u32 s2, s6
 ; GFX6-NEXT:    s_cselect_b64 vcc, -1, 0
 ; GFX6-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
-; GFX6-NEXT:    s_cselect_b32 s4, s5, s4
+; GFX6-NEXT:    s_cselect_b32 s2, s7, s2
 ; GFX6-NEXT:    v_add_i32_e32 v1, vcc, 1, v0
-; GFX6-NEXT:    s_cmp_ge_u32 s4, s8
+; GFX6-NEXT:    s_cmp_ge_u32 s2, s6
 ; GFX6-NEXT:    s_cselect_b64 vcc, -1, 0
-; GFX6-NEXT:    s_abs_i32 s8, s9
-; GFX6-NEXT:    v_cvt_f32_u32_e32 v2, s8
-; GFX6-NEXT:    s_mov_b32 s4, s0
-; GFX6-NEXT:    s_sub_i32 s0, 0, s8
+; GFX6-NEXT:    s_abs_i32 s2, s3
+; GFX6-NEXT:    v_cvt_f32_u32_e32 v2, s2
+; GFX6-NEXT:    s_sub_i32 s6, 0, s2
 ; GFX6-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
+; GFX6-NEXT:    s_xor_b32 s3, s1, s3
 ; GFX6-NEXT:    v_rcp_iflag_f32_e32 v2, v2
-; GFX6-NEXT:    s_mov_b32 s5, s1
-; GFX6-NEXT:    s_abs_i32 s1, s3
-; GFX6-NEXT:    v_xor_b32_e32 v0, s2, v0
+; GFX6-NEXT:    s_abs_i32 s1, s1
+; GFX6-NEXT:    v_xor_b32_e32 v0, s0, v0
+; GFX6-NEXT:    v_subrev_i32_e32 v0, vcc, s0, v0
 ; GFX6-NEXT:    v_mul_f32_e32 v2, 0x4f7ffffe, v2
 ; GFX6-NEXT:    v_cvt_u32_f32_e32 v2, v2
-; GFX6-NEXT:    v_subrev_i32_e32 v0, vcc, s2, v0
-; GFX6-NEXT:    v_mul_lo_u32 v3, s0, v2
-; GFX6-NEXT:    s_xor_b32 s0, s3, s9
-; GFX6-NEXT:    s_ashr_i32 s0, s0, 31
+; GFX6-NEXT:    s_ashr_i32 s3, s3, 31
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    v_mul_lo_u32 v3, s6, v2
+; GFX6-NEXT:    s_mov_b32 s6, -1
 ; GFX6-NEXT:    v_mul_hi_u32 v1, v2, v3
 ; GFX6-NEXT:    v_add_i32_e32 v1, vcc, v2, v1
 ; GFX6-NEXT:    v_mul_hi_u32 v1, s1, v1
-; GFX6-NEXT:    v_readfirstlane_b32 s2, v1
-; GFX6-NEXT:    s_mul_i32 s2, s2, s8
-; GFX6-NEXT:    s_sub_i32 s1, s1, s2
-; GFX6-NEXT:    s_sub_i32 s2, s1, s8
+; GFX6-NEXT:    v_readfirstlane_b32 s0, v1
+; GFX6-NEXT:    s_mul_i32 s0, s0, s2
+; GFX6-NEXT:    s_sub_i32 s0, s1, s0
+; GFX6-NEXT:    s_sub_i32 s1, s0, s2
 ; GFX6-NEXT:    v_add_i32_e32 v2, vcc, 1, v1
-; GFX6-NEXT:    s_cmp_ge_u32 s1, s8
+; GFX6-NEXT:    s_cmp_ge_u32 s0, s2
 ; GFX6-NEXT:    s_cselect_b64 vcc, -1, 0
 ; GFX6-NEXT:    v_cndmask_b32_e32 v1, v1, v2, vcc
-; GFX6-NEXT:    s_cselect_b32 s1, s2, s1
+; GFX6-NEXT:    s_cselect_b32 s0, s1, s0
 ; GFX6-NEXT:    v_add_i32_e32 v2, vcc, 1, v1
-; GFX6-NEXT:    s_cmp_ge_u32 s1, s8
+; GFX6-NEXT:    s_cmp_ge_u32 s0, s2
 ; GFX6-NEXT:    s_cselect_b64 vcc, -1, 0
 ; GFX6-NEXT:    v_cndmask_b32_e32 v1, v1, v2, vcc
-; GFX6-NEXT:    v_xor_b32_e32 v1, s0, v1
-; GFX6-NEXT:    v_subrev_i32_e32 v1, vcc, s0, v1
+; GFX6-NEXT:    v_xor_b32_e32 v1, s3, v1
+; GFX6-NEXT:    v_subrev_i32_e32 v1, vcc, s3, v1
 ; GFX6-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
 ; GFX6-NEXT:    s_endpgm
 ;
 ; GFX9-LABEL: sdiv_v2i32_pow2_shl_denom:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
-; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x2c
 ; GFX9-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    s_lshl_b32 s6, 0x1000, s6
-; GFX9-NEXT:    s_abs_i32 s8, s6
-; GFX9-NEXT:    v_cvt_f32_u32_e32 v0, s8
-; GFX9-NEXT:    s_lshl_b32 s4, 0x1000, s7
-; GFX9-NEXT:    s_abs_i32 s5, s2
-; GFX9-NEXT:    s_xor_b32 s2, s2, s6
+; GFX9-NEXT:    s_lshl_b32 s2, 0x1000, s2
+; GFX9-NEXT:    s_abs_i32 s6, s2
+; GFX9-NEXT:    v_cvt_f32_u32_e32 v0, s6
+; GFX9-NEXT:    s_lshl_b32 s7, 0x1000, s3
+; GFX9-NEXT:    s_abs_i32 s3, s0
+; GFX9-NEXT:    s_xor_b32 s0, s0, s2
 ; GFX9-NEXT:    v_rcp_iflag_f32_e32 v0, v0
-; GFX9-NEXT:    s_sub_i32 s6, 0, s8
-; GFX9-NEXT:    s_ashr_i32 s2, s2, 31
+; GFX9-NEXT:    s_sub_i32 s2, 0, s6
+; GFX9-NEXT:    s_ashr_i32 s0, s0, 31
 ; GFX9-NEXT:    v_mul_f32_e32 v0, 0x4f7ffffe, v0
 ; GFX9-NEXT:    v_cvt_u32_f32_e32 v0, v0
-; GFX9-NEXT:    v_readfirstlane_b32 s7, v0
-; GFX9-NEXT:    s_mul_i32 s6, s6, s7
-; GFX9-NEXT:    s_mul_hi_u32 s6, s7, s6
-; GFX9-NEXT:    s_add_i32 s7, s7, s6
-; GFX9-NEXT:    s_mul_hi_u32 s6, s5, s7
-; GFX9-NEXT:    s_mul_i32 s7, s6, s8
-; GFX9-NEXT:    s_sub_i32 s5, s5, s7
-; GFX9-NEXT:    s_add_i32 s9, s6, 1
-; GFX9-NEXT:    s_sub_i32 s7, s5, s8
-; GFX9-NEXT:    s_cmp_ge_u32 s5, s8
-; GFX9-NEXT:    s_cselect_b32 s6, s9, s6
-; GFX9-NEXT:    s_cselect_b32 s5, s7, s5
-; GFX9-NEXT:    s_add_i32 s7, s6, 1
-; GFX9-NEXT:    s_cmp_ge_u32 s5, s8
-; GFX9-NEXT:    s_cselect_b32 s5, s7, s6
-; GFX9-NEXT:    s_abs_i32 s6, s4
-; GFX9-NEXT:    v_cvt_f32_u32_e32 v0, s6
-; GFX9-NEXT:    s_xor_b32 s5, s5, s2
-; GFX9-NEXT:    s_sub_i32 s7, 0, s6
-; GFX9-NEXT:    s_sub_i32 s2, s5, s2
+; GFX9-NEXT:    v_readfirstlane_b32 s8, v0
+; GFX9-NEXT:    s_mul_i32 s2, s2, s8
+; GFX9-NEXT:    s_mul_hi_u32 s2, s8, s2
+; GFX9-NEXT:    s_add_i32 s8, s8, s2
+; GFX9-NEXT:    s_mul_hi_u32 s2, s3, s8
+; GFX9-NEXT:    s_mul_i32 s8, s2, s6
+; GFX9-NEXT:    s_sub_i32 s3, s3, s8
+; GFX9-NEXT:    s_add_i32 s9, s2, 1
+; GFX9-NEXT:    s_sub_i32 s8, s3, s6
+; GFX9-NEXT:    s_cmp_ge_u32 s3, s6
+; GFX9-NEXT:    s_cselect_b32 s2, s9, s2
+; GFX9-NEXT:    s_cselect_b32 s3, s8, s3
+; GFX9-NEXT:    s_add_i32 s8, s2, 1
+; GFX9-NEXT:    s_cmp_ge_u32 s3, s6
+; GFX9-NEXT:    s_cselect_b32 s6, s8, s2
+; GFX9-NEXT:    s_abs_i32 s8, s7
+; GFX9-NEXT:    v_cvt_f32_u32_e32 v0, s8
+; GFX9-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX9-NEXT:    s_xor_b32 s5, s6, s0
+; GFX9-NEXT:    s_sub_i32 s6, 0, s8
 ; GFX9-NEXT:    v_rcp_iflag_f32_e32 v0, v0
-; GFX9-NEXT:    s_xor_b32 s4, s3, s4
-; GFX9-NEXT:    s_abs_i32 s3, s3
-; GFX9-NEXT:    s_ashr_i32 s4, s4, 31
+; GFX9-NEXT:    s_sub_i32 s0, s5, s0
+; GFX9-NEXT:    s_xor_b32 s4, s1, s7
+; GFX9-NEXT:    s_abs_i32 s1, s1
 ; GFX9-NEXT:    v_mul_f32_e32 v0, 0x4f7ffffe, v0
 ; GFX9-NEXT:    v_cvt_u32_f32_e32 v0, v0
+; GFX9-NEXT:    s_ashr_i32 s4, s4, 31
 ; GFX9-NEXT:    v_readfirstlane_b32 s5, v0
-; GFX9-NEXT:    s_mul_i32 s7, s7, s5
-; GFX9-NEXT:    s_mul_hi_u32 s7, s5, s7
-; GFX9-NEXT:    s_add_i32 s5, s5, s7
-; GFX9-NEXT:    s_mul_hi_u32 s5, s3, s5
-; GFX9-NEXT:    s_mul_i32 s7, s5, s6
-; GFX9-NEXT:    s_sub_i32 s3, s3, s7
-; GFX9-NEXT:    s_add_i32 s8, s5, 1
-; GFX9-NEXT:    s_sub_i32 s7, s3, s6
-; GFX9-NEXT:    s_cmp_ge_u32 s3, s6
-; GFX9-NEXT:    s_cselect_b32 s5, s8, s5
-; GFX9-NEXT:    s_cselect_b32 s3, s7, s3
+; GFX9-NEXT:    s_mul_i32 s6, s6, s5
+; GFX9-NEXT:    s_mul_hi_u32 s6, s5, s6
+; GFX9-NEXT:    s_add_i32 s5, s5, s6
+; GFX9-NEXT:    s_mul_hi_u32 s5, s1, s5
+; GFX9-NEXT:    s_mul_i32 s6, s5, s8
+; GFX9-NEXT:    s_sub_i32 s1, s1, s6
 ; GFX9-NEXT:    s_add_i32 s7, s5, 1
-; GFX9-NEXT:    s_cmp_ge_u32 s3, s6
-; GFX9-NEXT:    s_cselect_b32 s3, s7, s5
-; GFX9-NEXT:    s_xor_b32 s3, s3, s4
-; GFX9-NEXT:    s_sub_i32 s3, s3, s4
-; GFX9-NEXT:    v_mov_b32_e32 v0, s2
-; GFX9-NEXT:    v_mov_b32_e32 v1, s3
-; GFX9-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX9-NEXT:    s_sub_i32 s6, s1, s8
+; GFX9-NEXT:    s_cmp_ge_u32 s1, s8
+; GFX9-NEXT:    s_cselect_b32 s5, s7, s5
+; GFX9-NEXT:    s_cselect_b32 s1, s6, s1
+; GFX9-NEXT:    s_add_i32 s6, s5, 1
+; GFX9-NEXT:    s_cmp_ge_u32 s1, s8
+; GFX9-NEXT:    s_cselect_b32 s1, s6, s5
+; GFX9-NEXT:    s_xor_b32 s1, s1, s4
+; GFX9-NEXT:    s_sub_i32 s1, s1, s4
+; GFX9-NEXT:    v_mov_b32_e32 v0, s0
+; GFX9-NEXT:    v_mov_b32_e32 v1, s1
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    global_store_dwordx2 v2, v[0:1], s[2:3]
 ; GFX9-NEXT:    s_endpgm
   %shl.y = shl <2 x i32> <i32 4096, i32 4096>, %y
   %r = sdiv <2 x i32> %x, %shl.y
@@ -6798,24 +6772,23 @@ define amdgpu_kernel void @srem_v2i32_pow2k_denom(ptr addrspace(1) %out, <2 x i3
 ; GFX6-LABEL: srem_v2i32_pow2k_denom:
 ; GFX6:       ; %bb.0:
 ; GFX6-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
-; GFX6-NEXT:    s_mov_b32 s7, 0xf000
-; GFX6-NEXT:    s_mov_b32 s6, -1
 ; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX6-NEXT:    s_mov_b32 s4, s0
-; GFX6-NEXT:    s_mov_b32 s5, s1
-; GFX6-NEXT:    s_ashr_i32 s0, s2, 31
-; GFX6-NEXT:    s_ashr_i32 s1, s3, 31
-; GFX6-NEXT:    s_lshr_b32 s0, s0, 20
-; GFX6-NEXT:    s_lshr_b32 s1, s1, 20
-; GFX6-NEXT:    s_add_i32 s0, s2, s0
-; GFX6-NEXT:    s_add_i32 s1, s3, s1
-; GFX6-NEXT:    s_and_b32 s0, s0, 0xfffff000
-; GFX6-NEXT:    s_and_b32 s1, s1, 0xfffff000
-; GFX6-NEXT:    s_sub_i32 s0, s2, s0
-; GFX6-NEXT:    s_sub_i32 s1, s3, s1
-; GFX6-NEXT:    v_mov_b32_e32 v0, s0
-; GFX6-NEXT:    v_mov_b32_e32 v1, s1
-; GFX6-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; GFX6-NEXT:    s_mov_b64 s[4:5], s[2:3]
+; GFX6-NEXT:    s_ashr_i32 s6, s4, 31
+; GFX6-NEXT:    s_lshr_b32 s6, s6, 20
+; GFX6-NEXT:    s_ashr_i32 s7, s5, 31
+; GFX6-NEXT:    s_add_i32 s6, s4, s6
+; GFX6-NEXT:    s_lshr_b32 s7, s7, 20
+; GFX6-NEXT:    s_and_b32 s6, s6, 0xfffff000
+; GFX6-NEXT:    s_sub_i32 s4, s4, s6
+; GFX6-NEXT:    s_add_i32 s6, s5, s7
+; GFX6-NEXT:    s_and_b32 s6, s6, 0xfffff000
+; GFX6-NEXT:    s_sub_i32 s5, s5, s6
+; GFX6-NEXT:    s_mov_b32 s3, 0xf000
+; GFX6-NEXT:    s_mov_b32 s2, -1
+; GFX6-NEXT:    v_mov_b32_e32 v0, s4
+; GFX6-NEXT:    v_mov_b32_e32 v1, s5
+; GFX6-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
 ; GFX6-NEXT:    s_endpgm
 ;
 ; GFX9-LABEL: srem_v2i32_pow2k_denom:
@@ -6926,125 +6899,122 @@ define amdgpu_kernel void @srem_v2i32_pow2_shl_denom(ptr addrspace(1) %out, <2 x
 ;
 ; GFX6-LABEL: srem_v2i32_pow2_shl_denom:
 ; GFX6:       ; %bb.0:
-; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0xd
+; GFX6-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0xb
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x9
 ; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX6-NEXT:    s_lshl_b32 s0, 0x1000, s6
-; GFX6-NEXT:    s_abs_i32 s6, s0
-; GFX6-NEXT:    v_cvt_f32_u32_e32 v0, s6
-; GFX6-NEXT:    s_sub_i32 s0, 0, s6
+; GFX6-NEXT:    s_lshl_b32 s2, 0x1000, s2
+; GFX6-NEXT:    s_abs_i32 s2, s2
+; GFX6-NEXT:    v_cvt_f32_u32_e32 v0, s2
+; GFX6-NEXT:    s_sub_i32 s6, 0, s2
+; GFX6-NEXT:    s_lshl_b32 s3, 0x1000, s3
 ; GFX6-NEXT:    v_rcp_iflag_f32_e32 v0, v0
 ; GFX6-NEXT:    v_mul_f32_e32 v0, 0x4f7ffffe, v0
 ; GFX6-NEXT:    v_cvt_u32_f32_e32 v0, v0
-; GFX6-NEXT:    v_mul_lo_u32 v1, s0, v0
-; GFX6-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
-; GFX6-NEXT:    s_lshl_b32 s5, 0x1000, s7
+; GFX6-NEXT:    v_mul_lo_u32 v1, s6, v0
+; GFX6-NEXT:    s_abs_i32 s6, s0
+; GFX6-NEXT:    s_ashr_i32 s0, s0, 31
 ; GFX6-NEXT:    v_mul_hi_u32 v1, v0, v1
-; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX6-NEXT:    s_abs_i32 s4, s2
-; GFX6-NEXT:    s_ashr_i32 s2, s2, 31
 ; GFX6-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
-; GFX6-NEXT:    v_mul_hi_u32 v0, s4, v0
+; GFX6-NEXT:    v_mul_hi_u32 v0, s6, v0
 ; GFX6-NEXT:    v_readfirstlane_b32 s7, v0
-; GFX6-NEXT:    s_mul_i32 s7, s7, s6
-; GFX6-NEXT:    s_sub_i32 s4, s4, s7
-; GFX6-NEXT:    s_sub_i32 s7, s4, s6
-; GFX6-NEXT:    s_cmp_ge_u32 s4, s6
-; GFX6-NEXT:    s_cselect_b32 s4, s7, s4
-; GFX6-NEXT:    s_sub_i32 s7, s4, s6
-; GFX6-NEXT:    s_cmp_ge_u32 s4, s6
-; GFX6-NEXT:    s_cselect_b32 s8, s7, s4
-; GFX6-NEXT:    s_abs_i32 s9, s5
-; GFX6-NEXT:    v_cvt_f32_u32_e32 v0, s9
-; GFX6-NEXT:    s_sub_i32 s4, 0, s9
-; GFX6-NEXT:    s_mov_b32 s5, s1
-; GFX6-NEXT:    s_ashr_i32 s1, s3, 31
+; GFX6-NEXT:    s_mul_i32 s7, s7, s2
+; GFX6-NEXT:    s_sub_i32 s6, s6, s7
+; GFX6-NEXT:    s_sub_i32 s7, s6, s2
+; GFX6-NEXT:    s_cmp_ge_u32 s6, s2
+; GFX6-NEXT:    s_cselect_b32 s6, s7, s6
+; GFX6-NEXT:    s_sub_i32 s7, s6, s2
+; GFX6-NEXT:    s_cmp_ge_u32 s6, s2
+; GFX6-NEXT:    s_cselect_b32 s2, s7, s6
+; GFX6-NEXT:    s_abs_i32 s3, s3
+; GFX6-NEXT:    v_cvt_f32_u32_e32 v0, s3
+; GFX6-NEXT:    s_sub_i32 s6, 0, s3
+; GFX6-NEXT:    s_abs_i32 s8, s1
+; GFX6-NEXT:    s_xor_b32 s2, s2, s0
 ; GFX6-NEXT:    v_rcp_iflag_f32_e32 v0, v0
+; GFX6-NEXT:    s_sub_i32 s0, s2, s0
+; GFX6-NEXT:    s_ashr_i32 s1, s1, 31
 ; GFX6-NEXT:    s_mov_b32 s7, 0xf000
-; GFX6-NEXT:    s_mov_b32 s6, -1
 ; GFX6-NEXT:    v_mul_f32_e32 v0, 0x4f7ffffe, v0
 ; GFX6-NEXT:    v_cvt_u32_f32_e32 v0, v0
-; GFX6-NEXT:    v_mul_lo_u32 v1, s4, v0
-; GFX6-NEXT:    s_mov_b32 s4, s0
-; GFX6-NEXT:    s_abs_i32 s0, s3
-; GFX6-NEXT:    s_xor_b32 s3, s8, s2
+; GFX6-NEXT:    v_mul_lo_u32 v1, s6, v0
+; GFX6-NEXT:    s_mov_b32 s6, -1
 ; GFX6-NEXT:    v_mul_hi_u32 v1, v0, v1
-; GFX6-NEXT:    s_sub_i32 s2, s3, s2
 ; GFX6-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
-; GFX6-NEXT:    v_mul_hi_u32 v0, s0, v0
-; GFX6-NEXT:    v_readfirstlane_b32 s3, v0
-; GFX6-NEXT:    s_mul_i32 s3, s3, s9
-; GFX6-NEXT:    s_sub_i32 s0, s0, s3
-; GFX6-NEXT:    s_sub_i32 s3, s0, s9
-; GFX6-NEXT:    s_cmp_ge_u32 s0, s9
-; GFX6-NEXT:    s_cselect_b32 s0, s3, s0
-; GFX6-NEXT:    s_sub_i32 s3, s0, s9
-; GFX6-NEXT:    s_cmp_ge_u32 s0, s9
-; GFX6-NEXT:    s_cselect_b32 s0, s3, s0
-; GFX6-NEXT:    s_xor_b32 s0, s0, s1
-; GFX6-NEXT:    s_sub_i32 s0, s0, s1
-; GFX6-NEXT:    v_mov_b32_e32 v0, s2
-; GFX6-NEXT:    v_mov_b32_e32 v1, s0
+; GFX6-NEXT:    v_mul_hi_u32 v0, s8, v0
+; GFX6-NEXT:    v_readfirstlane_b32 s2, v0
+; GFX6-NEXT:    s_mul_i32 s2, s2, s3
+; GFX6-NEXT:    s_sub_i32 s2, s8, s2
+; GFX6-NEXT:    s_sub_i32 s8, s2, s3
+; GFX6-NEXT:    s_cmp_ge_u32 s2, s3
+; GFX6-NEXT:    s_cselect_b32 s2, s8, s2
+; GFX6-NEXT:    s_sub_i32 s8, s2, s3
+; GFX6-NEXT:    s_cmp_ge_u32 s2, s3
+; GFX6-NEXT:    s_cselect_b32 s2, s8, s2
+; GFX6-NEXT:    s_xor_b32 s2, s2, s1
+; GFX6-NEXT:    s_sub_i32 s1, s2, s1
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX6-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
 ; GFX6-NEXT:    s_endpgm
 ;
 ; GFX9-LABEL: srem_v2i32_pow2_shl_denom:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x2c
 ; GFX9-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    s_lshl_b32 s0, 0x1000, s6
-; GFX9-NEXT:    s_abs_i32 s6, s0
-; GFX9-NEXT:    v_cvt_f32_u32_e32 v0, s6
-; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX9-NEXT:    s_lshl_b32 s4, 0x1000, s7
-; GFX9-NEXT:    s_sub_i32 s7, 0, s6
-; GFX9-NEXT:    v_rcp_iflag_f32_e32 v0, v0
-; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    s_ashr_i32 s5, s2, 31
+; GFX9-NEXT:    s_lshl_b32 s2, 0x1000, s2
 ; GFX9-NEXT:    s_abs_i32 s2, s2
+; GFX9-NEXT:    v_cvt_f32_u32_e32 v0, s2
+; GFX9-NEXT:    s_sub_i32 s7, 0, s2
+; GFX9-NEXT:    s_ashr_i32 s6, s0, 31
+; GFX9-NEXT:    s_abs_i32 s0, s0
+; GFX9-NEXT:    v_rcp_iflag_f32_e32 v0, v0
+; GFX9-NEXT:    s_lshl_b32 s3, 0x1000, s3
 ; GFX9-NEXT:    v_mul_f32_e32 v0, 0x4f7ffffe, v0
 ; GFX9-NEXT:    v_cvt_u32_f32_e32 v0, v0
 ; GFX9-NEXT:    v_readfirstlane_b32 s8, v0
 ; GFX9-NEXT:    s_mul_i32 s7, s7, s8
 ; GFX9-NEXT:    s_mul_hi_u32 s7, s8, s7
 ; GFX9-NEXT:    s_add_i32 s8, s8, s7
-; GFX9-NEXT:    s_mul_hi_u32 s7, s2, s8
-; GFX9-NEXT:    s_mul_i32 s7, s7, s6
-; GFX9-NEXT:    s_sub_i32 s2, s2, s7
-; GFX9-NEXT:    s_sub_i32 s7, s2, s6
-; GFX9-NEXT:    s_cmp_ge_u32 s2, s6
-; GFX9-NEXT:    s_cselect_b32 s2, s7, s2
-; GFX9-NEXT:    s_sub_i32 s7, s2, s6
-; GFX9-NEXT:    s_cmp_ge_u32 s2, s6
-; GFX9-NEXT:    s_cselect_b32 s2, s7, s2
-; GFX9-NEXT:    s_abs_i32 s4, s4
-; GFX9-NEXT:    v_cvt_f32_u32_e32 v0, s4
-; GFX9-NEXT:    s_xor_b32 s2, s2, s5
-; GFX9-NEXT:    s_sub_i32 s7, 0, s4
-; GFX9-NEXT:    s_sub_i32 s2, s2, s5
+; GFX9-NEXT:    s_mul_hi_u32 s7, s0, s8
+; GFX9-NEXT:    s_mul_i32 s7, s7, s2
+; GFX9-NEXT:    s_sub_i32 s0, s0, s7
+; GFX9-NEXT:    s_sub_i32 s7, s0, s2
+; GFX9-NEXT:    s_cmp_ge_u32 s0, s2
+; GFX9-NEXT:    s_cselect_b32 s0, s7, s0
+; GFX9-NEXT:    s_sub_i32 s7, s0, s2
+; GFX9-NEXT:    s_cmp_ge_u32 s0, s2
+; GFX9-NEXT:    s_cselect_b32 s0, s7, s0
+; GFX9-NEXT:    s_abs_i32 s7, s3
+; GFX9-NEXT:    v_cvt_f32_u32_e32 v0, s7
+; GFX9-NEXT:    s_xor_b32 s0, s0, s6
+; GFX9-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX9-NEXT:    s_sub_i32 s5, 0, s7
 ; GFX9-NEXT:    v_rcp_iflag_f32_e32 v0, v0
-; GFX9-NEXT:    s_ashr_i32 s6, s3, 31
-; GFX9-NEXT:    s_abs_i32 s3, s3
+; GFX9-NEXT:    s_sub_i32 s0, s0, s6
+; GFX9-NEXT:    s_ashr_i32 s4, s1, 31
+; GFX9-NEXT:    s_abs_i32 s1, s1
 ; GFX9-NEXT:    v_mul_f32_e32 v0, 0x4f7ffffe, v0
 ; GFX9-NEXT:    v_cvt_u32_f32_e32 v0, v0
-; GFX9-NEXT:    v_readfirstlane_b32 s5, v0
-; GFX9-NEXT:    s_mul_i32 s7, s7, s5
-; GFX9-NEXT:    s_mul_hi_u32 s7, s5, s7
-; GFX9-NEXT:    s_add_i32 s5, s5, s7
-; GFX9-NEXT:    s_mul_hi_u32 s5, s3, s5
-; GFX9-NEXT:    s_mul_i32 s5, s5, s4
-; GFX9-NEXT:    s_sub_i32 s3, s3, s5
-; GFX9-NEXT:    s_sub_i32 s5, s3, s4
-; GFX9-NEXT:    s_cmp_ge_u32 s3, s4
-; GFX9-NEXT:    s_cselect_b32 s3, s5, s3
-; GFX9-NEXT:    s_sub_i32 s5, s3, s4
-; GFX9-NEXT:    s_cmp_ge_u32 s3, s4
-; GFX9-NEXT:    s_cselect_b32 s3, s5, s3
-; GFX9-NEXT:    s_xor_b32 s3, s3, s6
-; GFX9-NEXT:    s_sub_i32 s3, s3, s6
-; GFX9-NEXT:    v_mov_b32_e32 v0, s2
-; GFX9-NEXT:    v_mov_b32_e32 v1, s3
-; GFX9-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX9-NEXT:    v_readfirstlane_b32 s6, v0
+; GFX9-NEXT:    s_mul_i32 s5, s5, s6
+; GFX9-NEXT:    s_mul_hi_u32 s5, s6, s5
+; GFX9-NEXT:    s_add_i32 s6, s6, s5
+; GFX9-NEXT:    s_mul_hi_u32 s5, s1, s6
+; GFX9-NEXT:    s_mul_i32 s5, s5, s7
+; GFX9-NEXT:    s_sub_i32 s1, s1, s5
+; GFX9-NEXT:    s_sub_i32 s5, s1, s7
+; GFX9-NEXT:    s_cmp_ge_u32 s1, s7
+; GFX9-NEXT:    s_cselect_b32 s1, s5, s1
+; GFX9-NEXT:    s_sub_i32 s5, s1, s7
+; GFX9-NEXT:    s_cmp_ge_u32 s1, s7
+; GFX9-NEXT:    s_cselect_b32 s1, s5, s1
+; GFX9-NEXT:    s_xor_b32 s1, s1, s4
+; GFX9-NEXT:    s_sub_i32 s1, s1, s4
+; GFX9-NEXT:    v_mov_b32_e32 v0, s0
+; GFX9-NEXT:    v_mov_b32_e32 v1, s1
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    global_store_dwordx2 v2, v[0:1], s[2:3]
 ; GFX9-NEXT:    s_endpgm
   %shl.y = shl <2 x i32> <i32 4096, i32 4096>, %y
   %r = srem <2 x i32> %x, %shl.y
@@ -10151,9 +10121,6 @@ define i64 @udiv_i64_9divbits(i8 %size) {
 }
 
 define <2 x i64> @srem_zero_zero() {
-; GCN-LABEL: kernel:
-; GCN:       ; %bb.0: ; %entry
-; GCN-NEXT:    s_endpgm
 ; GFX6-LABEL: srem_zero_zero:
 ; GFX6:       ; %bb.0: ; %entry
 ; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
diff --git a/llvm/test/CodeGen/AMDGPU/build_vector.ll b/llvm/test/CodeGen/AMDGPU/build_vector.ll
index 56ad037f65641..37f4094806637 100644
--- a/llvm/test/CodeGen/AMDGPU/build_vector.ll
+++ b/llvm/test/CodeGen/AMDGPU/build_vector.ll
@@ -257,29 +257,28 @@ define amdgpu_kernel void @build_v2i32_from_v4i16_shuffle(ptr addrspace(1) %out,
 ; GFX6-LABEL: build_v2i32_from_v4i16_shuffle:
 ; GFX6:       ; %bb.0: ; %entry
 ; GFX6-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
-; GFX6-NEXT:    s_mov_b32 s7, 0xf000
-; GFX6-NEXT:    s_mov_b32 s6, -1
 ; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX6-NEXT:    s_mov_b32 s4, s0
-; GFX6-NEXT:    s_mov_b32 s5, s1
-; GFX6-NEXT:    s_lshl_b32 s0, s3, 16
-; GFX6-NEXT:    s_lshl_b32 s1, s2, 16
-; GFX6-NEXT:    v_mov_b32_e32 v0, s1
-; GFX6-NEXT:    v_mov_b32_e32 v1, s0
-; GFX6-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; GFX6-NEXT:    s_mov_b64 s[4:5], s[2:3]
+; GFX6-NEXT:    s_lshl_b32 s5, s5, 16
+; GFX6-NEXT:    s_lshl_b32 s4, s4, 16
+; GFX6-NEXT:    s_mov_b32 s3, 0xf000
+; GFX6-NEXT:    s_mov_b32 s2, -1
+; GFX6-NEXT:    v_mov_b32_e32 v0, s4
+; GFX6-NEXT:    v_mov_b32_e32 v1, s5
+; GFX6-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
 ; GFX6-NEXT:    s_endpgm
 ;
 ; GFX8-LABEL: build_v2i32_from_v4i16_shuffle:
 ; GFX8:       ; %bb.0: ; %entry
 ; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    v_mov_b32_e32 v0, s0
-; GFX8-NEXT:    v_mov_b32_e32 v1, s1
-; GFX8-NEXT:    s_lshl_b32 s0, s3, 16
-; GFX8-NEXT:    s_lshl_b32 s1, s2, 16
-; GFX8-NEXT:    v_mov_b32_e32 v2, s1
-; GFX8-NEXT:    v_mov_b32_e32 v3, s0
-; GFX8-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
+; GFX8-NEXT:    s_lshl_b32 s3, s3, 16
+; GFX8-NEXT:    s_lshl_b32 s2, s2, 16
+; GFX8-NEXT:    v_mov_b32_e32 v3, s1
+; GFX8-NEXT:    v_mov_b32_e32 v0, s2
+; GFX8-NEXT:    v_mov_b32_e32 v1, s3
+; GFX8-NEXT:    v_mov_b32_e32 v2, s0
+; GFX8-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
 ; GFX8-NEXT:    s_endpgm
 ;
 ; GFX10-LABEL: build_v2i32_from_v4i16_shuffle:
diff --git a/llvm/test/CodeGen/AMDGPU/fabs.bf16.ll b/llvm/test/CodeGen/AMDGPU/fabs.bf16.ll
index 61fb18e00917b..c46fcde739b1c 100644
--- a/llvm/test/CodeGen/AMDGPU/fabs.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/fabs.bf16.ll
@@ -218,13 +218,13 @@ define amdgpu_kernel void @s_fabs_v4bf16(ptr addrspace(1) %out, <4 x bfloat> %in
 ; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
-; CI-NEXT:    v_mov_b32_e32 v0, s0
-; CI-NEXT:    v_mov_b32_e32 v1, s1
-; CI-NEXT:    s_and_b32 s0, s3, 0x7fff7fff
-; CI-NEXT:    s_and_b32 s1, s2, 0x7fff7fff
-; CI-NEXT:    v_mov_b32_e32 v2, s1
-; CI-NEXT:    v_mov_b32_e32 v3, s0
-; CI-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
+; CI-NEXT:    s_and_b32 s3, s3, 0x7fff7fff
+; CI-NEXT:    s_and_b32 s2, s2, 0x7fff7fff
+; CI-NEXT:    v_mov_b32_e32 v3, s1
+; CI-NEXT:    v_mov_b32_e32 v0, s2
+; CI-NEXT:    v_mov_b32_e32 v1, s3
+; CI-NEXT:    v_mov_b32_e32 v2, s0
+; CI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
 ; CI-NEXT:    s_endpgm
 ;
 ; VI-LABEL: s_fabs_v4bf16:
@@ -234,13 +234,23 @@ define amdgpu_kernel void @s_fabs_v4bf16(ptr addrspace(1) %out, <4 x bfloat> %in
 ; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
-; VI-NEXT:    v_mov_b32_e32 v0, s0
-; VI-NEXT:    v_mov_b32_e32 v1, s1
-; VI-NEXT:    s_and_b32 s0, s3, 0x7fff7fff
-; VI-NEXT:    s_and_b32 s1, s2, 0x7fff7fff
-; VI-NEXT:    v_mov_b32_e32 v2, s1
-; VI-NEXT:    v_mov_b32_e32 v3, s0
-; VI-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
+; VI-NEXT:    s_and_b32 s4, s3, 0x7fff
+; VI-NEXT:    s_lshr_b32 s3, s3, 16
+; VI-NEXT:    s_and_b32 s5, s2, 0x7fff
+; VI-NEXT:    s_lshr_b32 s2, s2, 16
+; VI-NEXT:    s_and_b32 s3, s3, 0x7fff
+; VI-NEXT:    s_and_b32 s2, s2, 0x7fff
+; VI-NEXT:    s_and_b32 s4, 0xffff, s4
+; VI-NEXT:    s_and_b32 s5, 0xffff, s5
+; VI-NEXT:    s_lshl_b32 s3, s3, 16
+; VI-NEXT:    s_lshl_b32 s2, s2, 16
+; VI-NEXT:    s_or_b32 s3, s4, s3
+; VI-NEXT:    s_or_b32 s2, s5, s2
+; VI-NEXT:    v_mov_b32_e32 v3, s1
+; VI-NEXT:    v_mov_b32_e32 v0, s2
+; VI-NEXT:    v_mov_b32_e32 v1, s3
+; VI-NEXT:    v_mov_b32_e32 v2, s0
+; VI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
 ; VI-NEXT:    s_endpgm
 ;
 ; GFX9-LABEL: s_fabs_v4bf16:
@@ -248,8 +258,14 @@ define amdgpu_kernel void @s_fabs_v4bf16(ptr addrspace(1) %out, <4 x bfloat> %in
 ; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; GFX9-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    s_and_b32 s3, s3, 0x7fff7fff
-; GFX9-NEXT:    s_and_b32 s2, s2, 0x7fff7fff
+; GFX9-NEXT:    s_and_b32 s4, s3, 0x7fff
+; GFX9-NEXT:    s_lshr_b32 s3, s3, 16
+; GFX9-NEXT:    s_and_b32 s5, s2, 0x7fff
+; GFX9-NEXT:    s_lshr_b32 s2, s2, 16
+; GFX9-NEXT:    s_and_b32 s3, s3, 0x7fff
+; GFX9-NEXT:    s_and_b32 s2, s2, 0x7fff
+; GFX9-NEXT:    s_pack_ll_b32_b16 s3, s4, s3
+; GFX9-NEXT:    s_pack_ll_b32_b16 s2, s5, s2
 ; GFX9-NEXT:    v_mov_b32_e32 v0, s2
 ; GFX9-NEXT:    v_mov_b32_e32 v1, s3
 ; GFX9-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1]
@@ -259,8 +275,14 @@ define amdgpu_kernel void @s_fabs_v4bf16(ptr addrspace(1) %out, <4 x bfloat> %in
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x0
 ; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX11-NEXT:    s_and_b32 s2, s2, 0x7fff7fff
-; GFX11-NEXT:    s_and_b32 s3, s3, 0x7fff7fff
+; GFX11-NEXT:    s_and_b32 s4, s3, 0x7fff
+; GFX11-NEXT:    s_lshr_b32 s3, s3, 16
+; GFX11-NEXT:    s_lshr_b32 s5, s2, 16
+; GFX11-NEXT:    s_and_b32 s2, s2, 0x7fff
+; GFX11-NEXT:    s_and_b32 s5, s5, 0x7fff
+; GFX11-NEXT:    s_and_b32 s3, s3, 0x7fff
+; GFX11-NEXT:    s_pack_ll_b32_b16 s2, s2, s5
+; GFX11-NEXT:    s_pack_ll_b32_b16 s3, s4, s3
 ; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
 ; GFX11-NEXT:    v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s3
 ; GFX11-NEXT:    v_mov_b32_e32 v0, s2
diff --git a/llvm/test/CodeGen/AMDGPU/fabs.f16.ll b/llvm/test/CodeGen/AMDGPU/fabs.f16.ll
index a77c7ae923d0f..27cf49aec8229 100644
--- a/llvm/test/CodeGen/AMDGPU/fabs.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/fabs.f16.ll
@@ -218,13 +218,13 @@ define amdgpu_kernel void @s_fabs_v4f16(ptr addrspace(1) %out, <4 x half> %in) {
 ; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
-; CI-NEXT:    v_mov_b32_e32 v0, s0
-; CI-NEXT:    v_mov_b32_e32 v1, s1
-; CI-NEXT:    s_and_b32 s0, s3, 0x7fff7fff
-; CI-NEXT:    s_and_b32 s1, s2, 0x7fff7fff
-; CI-NEXT:    v_mov_b32_e32 v2, s1
-; CI-NEXT:    v_mov_b32_e32 v3, s0
-; CI-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
+; CI-NEXT:    s_and_b32 s3, s3, 0x7fff7fff
+; CI-NEXT:    s_and_b32 s2, s2, 0x7fff7fff
+; CI-NEXT:    v_mov_b32_e32 v3, s1
+; CI-NEXT:    v_mov_b32_e32 v0, s2
+; CI-NEXT:    v_mov_b32_e32 v1, s3
+; CI-NEXT:    v_mov_b32_e32 v2, s0
+; CI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
 ; CI-NEXT:    s_endpgm
 ;
 ; VI-LABEL: s_fabs_v4f16:
@@ -234,13 +234,13 @@ define amdgpu_kernel void @s_fabs_v4f16(ptr addrspace(1) %out, <4 x half> %in) {
 ; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
-; VI-NEXT:    v_mov_b32_e32 v0, s0
-; VI-NEXT:    v_mov_b32_e32 v1, s1
-; VI-NEXT:    s_and_b32 s0, s3, 0x7fff7fff
-; VI-NEXT:    s_and_b32 s1, s2, 0x7fff7fff
-; VI-NEXT:    v_mov_b32_e32 v2, s1
-; VI-NEXT:    v_mov_b32_e32 v3, s0
-; VI-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
+; VI-NEXT:    s_and_b32 s3, s3, 0x7fff7fff
+; VI-NEXT:    s_and_b32 s2, s2, 0x7fff7fff
+; VI-NEXT:    v_mov_b32_e32 v3, s1
+; VI-NEXT:    v_mov_b32_e32 v0, s2
+; VI-NEXT:    v_mov_b32_e32 v1, s3
+; VI-NEXT:    v_mov_b32_e32 v2, s0
+; VI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
 ; VI-NEXT:    s_endpgm
 ;
 ; GFX9-LABEL: s_fabs_v4f16:
diff --git a/llvm/test/CodeGen/AMDGPU/fabs.ll b/llvm/test/CodeGen/AMDGPU/fabs.ll
index baf9b0abf7b0c..97e23fcdb2263 100644
--- a/llvm/test/CodeGen/AMDGPU/fabs.ll
+++ b/llvm/test/CodeGen/AMDGPU/fabs.ll
@@ -99,29 +99,28 @@ define amdgpu_kernel void @fabs_v2f32(ptr addrspace(1) %out, <2 x float> %in) {
 ; SI-LABEL: fabs_v2f32:
 ; SI:       ; %bb.0:
 ; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
-; SI-NEXT:    s_mov_b32 s7, 0xf000
-; SI-NEXT:    s_mov_b32 s6, -1
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
-; SI-NEXT:    s_mov_b32 s4, s0
-; SI-NEXT:    s_mov_b32 s5, s1
-; SI-NEXT:    s_and_b32 s0, s3, 0x7fffffff
-; SI-NEXT:    s_and_b32 s1, s2, 0x7fffffff
-; SI-NEXT:    v_mov_b32_e32 v0, s1
-; SI-NEXT:    v_mov_b32_e32 v1, s0
-; SI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; SI-NEXT:    s_mov_b64 s[4:5], s[2:3]
+; SI-NEXT:    s_mov_b32 s3, 0xf000
+; SI-NEXT:    s_mov_b32 s2, -1
+; SI-NEXT:    s_bitset0_b32 s5, 31
+; SI-NEXT:    s_bitset0_b32 s4, 31
+; SI-NEXT:    v_mov_b32_e32 v0, s4
+; SI-NEXT:    v_mov_b32_e32 v1, s5
+; SI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
 ; SI-NEXT:    s_endpgm
 ;
 ; VI-LABEL: fabs_v2f32:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
-; VI-NEXT:    v_mov_b32_e32 v0, s0
-; VI-NEXT:    v_mov_b32_e32 v1, s1
-; VI-NEXT:    s_and_b32 s0, s3, 0x7fffffff
-; VI-NEXT:    s_and_b32 s1, s2, 0x7fffffff
-; VI-NEXT:    v_mov_b32_e32 v2, s1
-; VI-NEXT:    v_mov_b32_e32 v3, s0
-; VI-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
+; VI-NEXT:    s_bitset0_b32 s3, 31
+; VI-NEXT:    s_bitset0_b32 s2, 31
+; VI-NEXT:    v_mov_b32_e32 v3, s1
+; VI-NEXT:    v_mov_b32_e32 v0, s2
+; VI-NEXT:    v_mov_b32_e32 v1, s3
+; VI-NEXT:    v_mov_b32_e32 v2, s0
+; VI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
 ; VI-NEXT:    s_endpgm
   %fabs = call <2 x float> @llvm.fabs.v2f32(<2 x float> %in)
   store <2 x float> %fabs, ptr addrspace(1) %out
diff --git a/llvm/test/CodeGen/AMDGPU/fcopysign.f32.ll b/llvm/test/CodeGen/AMDGPU/fcopysign.f32.ll
index 1d87d938cc41c..0a2e758f7cf21 100644
--- a/llvm/test/CodeGen/AMDGPU/fcopysign.f32.ll
+++ b/llvm/test/CodeGen/AMDGPU/fcopysign.f32.ll
@@ -472,52 +472,50 @@ define amdgpu_kernel void @s_test_copysign_f32_neg10_mag(ptr addrspace(1) %out,
 define amdgpu_kernel void @s_test_copysign_v2f32(ptr addrspace(1) %out, <2 x float> %mag, <2 x float> %sign) {
 ; SI-LABEL: s_test_copysign_v2f32:
 ; SI:       ; %bb.0:
-; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
-; SI-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0xd
+; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0xb
+; SI-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x9
+; SI-NEXT:    s_brev_b32 s8, -2
 ; SI-NEXT:    s_mov_b32 s7, 0xf000
 ; SI-NEXT:    s_mov_b32 s6, -1
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
-; SI-NEXT:    s_mov_b32 s4, s0
-; SI-NEXT:    s_brev_b32 s0, -2
-; SI-NEXT:    v_mov_b32_e32 v0, s3
-; SI-NEXT:    v_mov_b32_e32 v1, s9
-; SI-NEXT:    v_bfi_b32 v1, s0, v0, v1
-; SI-NEXT:    v_mov_b32_e32 v0, s2
-; SI-NEXT:    v_mov_b32_e32 v2, s8
-; SI-NEXT:    s_mov_b32 s5, s1
-; SI-NEXT:    v_bfi_b32 v0, s0, v0, v2
+; SI-NEXT:    v_mov_b32_e32 v0, s1
+; SI-NEXT:    v_mov_b32_e32 v1, s3
+; SI-NEXT:    v_bfi_b32 v1, s8, v0, v1
+; SI-NEXT:    v_mov_b32_e32 v0, s0
+; SI-NEXT:    v_mov_b32_e32 v2, s2
+; SI-NEXT:    v_bfi_b32 v0, s8, v0, v2
 ; SI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
 ; SI-NEXT:    s_endpgm
 ;
 ; VI-LABEL: s_test_copysign_v2f32:
 ; VI:       ; %bb.0:
-; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
-; VI-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x34
+; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x2c
+; VI-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x24
 ; VI-NEXT:    s_brev_b32 s6, -2
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
-; VI-NEXT:    v_mov_b32_e32 v2, s3
+; VI-NEXT:    v_mov_b32_e32 v0, s1
+; VI-NEXT:    v_mov_b32_e32 v1, s3
+; VI-NEXT:    v_mov_b32_e32 v2, s0
+; VI-NEXT:    v_bfi_b32 v1, s6, v0, v1
+; VI-NEXT:    v_mov_b32_e32 v0, s2
+; VI-NEXT:    v_bfi_b32 v0, s6, v2, v0
+; VI-NEXT:    v_mov_b32_e32 v2, s4
 ; VI-NEXT:    v_mov_b32_e32 v3, s5
-; VI-NEXT:    v_bfi_b32 v3, s6, v2, v3
-; VI-NEXT:    v_mov_b32_e32 v2, s2
-; VI-NEXT:    v_mov_b32_e32 v4, s4
-; VI-NEXT:    v_mov_b32_e32 v0, s0
-; VI-NEXT:    v_mov_b32_e32 v1, s1
-; VI-NEXT:    v_bfi_b32 v2, s6, v2, v4
-; VI-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
+; VI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
 ; VI-NEXT:    s_endpgm
 ;
 ; GFX11-LABEL: s_test_copysign_v2f32:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_clause 0x1
-; GFX11-NEXT:    s_load_b64 s[6:7], s[4:5], 0x34
-; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x2c
+; GFX11-NEXT:    s_load_b64 s[4:5], s[4:5], 0x24
 ; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX11-NEXT:    v_dual_mov_b32 v3, 0 :: v_dual_mov_b32 v0, s7
-; GFX11-NEXT:    v_mov_b32_e32 v2, s6
+; GFX11-NEXT:    v_dual_mov_b32 v3, 0 :: v_dual_mov_b32 v0, s3
+; GFX11-NEXT:    v_mov_b32_e32 v2, s2
 ; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT:    v_bfi_b32 v1, 0x7fffffff, s3, v0
-; GFX11-NEXT:    v_bfi_b32 v0, 0x7fffffff, s2, v2
-; GFX11-NEXT:    global_store_b64 v3, v[0:1], s[0:1]
+; GFX11-NEXT:    v_bfi_b32 v1, 0x7fffffff, s1, v0
+; GFX11-NEXT:    v_bfi_b32 v0, 0x7fffffff, s0, v2
+; GFX11-NEXT:    global_store_b64 v3, v[0:1], s[4:5]
 ; GFX11-NEXT:    s_endpgm
   %result = call <2 x float> @llvm.copysign.v2f32(<2 x float> %mag, <2 x float> %sign)
   store <2 x float> %result, ptr addrspace(1) %out, align 8
diff --git a/llvm/test/CodeGen/AMDGPU/fdiv.ll b/llvm/test/CodeGen/AMDGPU/fdiv.ll
index 0fc61cbe54bad..c510c40c8536c 100644
--- a/llvm/test/CodeGen/AMDGPU/fdiv.ll
+++ b/llvm/test/CodeGen/AMDGPU/fdiv.ll
@@ -932,18 +932,16 @@ entry:
 define amdgpu_kernel void @s_fdiv_v2f32(ptr addrspace(1) %out, <2 x float> %a, <2 x float> %b) #0 {
 ; GFX6-FASTFMA-LABEL: s_fdiv_v2f32:
 ; GFX6-FASTFMA:       ; %bb.0: ; %entry
-; GFX6-FASTFMA-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
-; GFX6-FASTFMA-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0xd
-; GFX6-FASTFMA-NEXT:    s_mov_b32 s7, 0xf000
-; GFX6-FASTFMA-NEXT:    s_mov_b32 s6, -1
+; GFX6-FASTFMA-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0xb
+; GFX6-FASTFMA-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x9
+; GFX6-FASTFMA-NEXT:    s_mov_b32 s3, 0xf000
+; GFX6-FASTFMA-NEXT:    s_mov_b32 s2, -1
 ; GFX6-FASTFMA-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX6-FASTFMA-NEXT:    v_mov_b32_e32 v1, s3
-; GFX6-FASTFMA-NEXT:    s_mov_b32 s4, s0
-; GFX6-FASTFMA-NEXT:    s_mov_b32 s5, s1
-; GFX6-FASTFMA-NEXT:    v_div_scale_f32 v2, s[0:1], s9, s9, v1
+; GFX6-FASTFMA-NEXT:    v_mov_b32_e32 v1, s9
+; GFX6-FASTFMA-NEXT:    v_div_scale_f32 v2, s[4:5], s11, s11, v1
 ; GFX6-FASTFMA-NEXT:    v_rcp_f32_e32 v3, v2
-; GFX6-FASTFMA-NEXT:    v_mov_b32_e32 v0, s9
-; GFX6-FASTFMA-NEXT:    v_div_scale_f32 v0, vcc, s3, v0, s3
+; GFX6-FASTFMA-NEXT:    v_mov_b32_e32 v0, s11
+; GFX6-FASTFMA-NEXT:    v_div_scale_f32 v0, vcc, s9, v0, s9
 ; GFX6-FASTFMA-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 3
 ; GFX6-FASTFMA-NEXT:    v_fma_f32 v4, -v2, v3, 1.0
 ; GFX6-FASTFMA-NEXT:    v_fma_f32 v3, v4, v3, v3
@@ -952,13 +950,13 @@ define amdgpu_kernel void @s_fdiv_v2f32(ptr addrspace(1) %out, <2 x float> %a, <
 ; GFX6-FASTFMA-NEXT:    v_fma_f32 v4, v5, v3, v4
 ; GFX6-FASTFMA-NEXT:    v_fma_f32 v0, -v2, v4, v0
 ; GFX6-FASTFMA-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 0
-; GFX6-FASTFMA-NEXT:    v_mov_b32_e32 v2, s2
+; GFX6-FASTFMA-NEXT:    v_mov_b32_e32 v2, s8
 ; GFX6-FASTFMA-NEXT:    v_div_fmas_f32 v0, v0, v3, v4
-; GFX6-FASTFMA-NEXT:    v_div_scale_f32 v3, s[0:1], s8, s8, v2
+; GFX6-FASTFMA-NEXT:    v_div_scale_f32 v3, s[4:5], s10, s10, v2
 ; GFX6-FASTFMA-NEXT:    v_rcp_f32_e32 v4, v3
-; GFX6-FASTFMA-NEXT:    v_div_fixup_f32 v1, v0, s9, v1
-; GFX6-FASTFMA-NEXT:    v_mov_b32_e32 v0, s8
-; GFX6-FASTFMA-NEXT:    v_div_scale_f32 v0, vcc, s2, v0, s2
+; GFX6-FASTFMA-NEXT:    v_div_fixup_f32 v1, v0, s11, v1
+; GFX6-FASTFMA-NEXT:    v_mov_b32_e32 v0, s10
+; GFX6-FASTFMA-NEXT:    v_div_scale_f32 v0, vcc, s8, v0, s8
 ; GFX6-FASTFMA-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 3
 ; GFX6-FASTFMA-NEXT:    v_fma_f32 v5, -v3, v4, 1.0
 ; GFX6-FASTFMA-NEXT:    v_fma_f32 v4, v5, v4, v4
@@ -968,21 +966,20 @@ define amdgpu_kernel void @s_fdiv_v2f32(ptr addrspace(1) %out, <2 x float> %a, <
 ; GFX6-FASTFMA-NEXT:    v_fma_f32 v0, -v3, v5, v0
 ; GFX6-FASTFMA-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 0
 ; GFX6-FASTFMA-NEXT:    v_div_fmas_f32 v0, v0, v4, v5
-; GFX6-FASTFMA-NEXT:    v_div_fixup_f32 v0, v0, s8, v2
-; GFX6-FASTFMA-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; GFX6-FASTFMA-NEXT:    v_div_fixup_f32 v0, v0, s10, v2
+; GFX6-FASTFMA-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
 ; GFX6-FASTFMA-NEXT:    s_endpgm
 ;
 ; GFX6-SLOWFMA-LABEL: s_fdiv_v2f32:
 ; GFX6-SLOWFMA:       ; %bb.0: ; %entry
-; GFX6-SLOWFMA-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
-; GFX6-SLOWFMA-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xd
+; GFX6-SLOWFMA-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0xb
+; GFX6-SLOWFMA-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x9
 ; GFX6-SLOWFMA-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX6-SLOWFMA-NEXT:    v_mov_b32_e32 v0, s3
-; GFX6-SLOWFMA-NEXT:    v_div_scale_f32 v1, s[6:7], s5, s5, v0
-; GFX6-SLOWFMA-NEXT:    v_mov_b32_e32 v2, s5
-; GFX6-SLOWFMA-NEXT:    v_div_scale_f32 v2, vcc, s3, v2, s3
-; GFX6-SLOWFMA-NEXT:    v_mov_b32_e32 v4, s2
-; GFX6-SLOWFMA-NEXT:    s_mov_b32 s3, 0xf000
+; GFX6-SLOWFMA-NEXT:    v_mov_b32_e32 v0, s1
+; GFX6-SLOWFMA-NEXT:    v_div_scale_f32 v1, s[6:7], s3, s3, v0
+; GFX6-SLOWFMA-NEXT:    v_mov_b32_e32 v2, s3
+; GFX6-SLOWFMA-NEXT:    v_div_scale_f32 v2, vcc, s1, v2, s1
+; GFX6-SLOWFMA-NEXT:    v_mov_b32_e32 v4, s0
 ; GFX6-SLOWFMA-NEXT:    v_rcp_f32_e32 v3, v1
 ; GFX6-SLOWFMA-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 3
 ; GFX6-SLOWFMA-NEXT:    v_fma_f32 v5, -v1, v3, 1.0
@@ -992,13 +989,14 @@ define amdgpu_kernel void @s_fdiv_v2f32(ptr addrspace(1) %out, <2 x float> %a, <
 ; GFX6-SLOWFMA-NEXT:    v_fma_f32 v5, v6, v3, v5
 ; GFX6-SLOWFMA-NEXT:    v_fma_f32 v1, -v1, v5, v2
 ; GFX6-SLOWFMA-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 0
-; GFX6-SLOWFMA-NEXT:    v_div_scale_f32 v2, s[6:7], s4, s4, v4
+; GFX6-SLOWFMA-NEXT:    v_div_scale_f32 v2, s[6:7], s2, s2, v4
 ; GFX6-SLOWFMA-NEXT:    v_div_fmas_f32 v1, v1, v3, v5
-; GFX6-SLOWFMA-NEXT:    v_mov_b32_e32 v3, s4
-; GFX6-SLOWFMA-NEXT:    v_div_scale_f32 v3, vcc, s2, v3, s2
-; GFX6-SLOWFMA-NEXT:    s_mov_b32 s2, -1
+; GFX6-SLOWFMA-NEXT:    v_mov_b32_e32 v3, s2
+; GFX6-SLOWFMA-NEXT:    v_div_scale_f32 v3, vcc, s0, v3, s0
+; GFX6-SLOWFMA-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-SLOWFMA-NEXT:    s_mov_b32 s6, -1
 ; GFX6-SLOWFMA-NEXT:    v_rcp_f32_e32 v5, v2
-; GFX6-SLOWFMA-NEXT:    v_div_fixup_f32 v1, v1, s5, v0
+; GFX6-SLOWFMA-NEXT:    v_div_fixup_f32 v1, v1, s3, v0
 ; GFX6-SLOWFMA-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 3
 ; GFX6-SLOWFMA-NEXT:    v_fma_f32 v0, -v2, v5, 1.0
 ; GFX6-SLOWFMA-NEXT:    v_fma_f32 v0, v0, v5, v5
@@ -1008,24 +1006,22 @@ define amdgpu_kernel void @s_fdiv_v2f32(ptr addrspace(1) %out, <2 x float> %a, <
 ; GFX6-SLOWFMA-NEXT:    v_fma_f32 v2, -v2, v5, v3
 ; GFX6-SLOWFMA-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 0
 ; GFX6-SLOWFMA-NEXT:    v_div_fmas_f32 v0, v2, v0, v5
-; GFX6-SLOWFMA-NEXT:    v_div_fixup_f32 v0, v0, s4, v4
-; GFX6-SLOWFMA-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; GFX6-SLOWFMA-NEXT:    v_div_fixup_f32 v0, v0, s2, v4
+; GFX6-SLOWFMA-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
 ; GFX6-SLOWFMA-NEXT:    s_endpgm
 ;
 ; GFX7-LABEL: s_fdiv_v2f32:
 ; GFX7:       ; %bb.0: ; %entry
-; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
-; GFX7-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0xd
-; GFX7-NEXT:    s_mov_b32 s7, 0xf000
-; GFX7-NEXT:    s_mov_b32 s6, -1
+; GFX7-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0xb
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x9
+; GFX7-NEXT:    s_mov_b32 s3, 0xf000
+; GFX7-NEXT:    s_mov_b32 s2, -1
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7-NEXT:    v_mov_b32_e32 v1, s3
-; GFX7-NEXT:    s_mov_b32 s4, s0
-; GFX7-NEXT:    s_mov_b32 s5, s1
-; GFX7-NEXT:    v_div_scale_f32 v2, s[0:1], s9, s9, v1
+; GFX7-NEXT:    v_mov_b32_e32 v1, s9
+; GFX7-NEXT:    v_div_scale_f32 v2, s[4:5], s11, s11, v1
 ; GFX7-NEXT:    v_rcp_f32_e32 v3, v2
-; GFX7-NEXT:    v_mov_b32_e32 v0, s9
-; GFX7-NEXT:    v_div_scale_f32 v0, vcc, s3, v0, s3
+; GFX7-NEXT:    v_mov_b32_e32 v0, s11
+; GFX7-NEXT:    v_div_scale_f32 v0, vcc, s9, v0, s9
 ; GFX7-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 3
 ; GFX7-NEXT:    v_fma_f32 v4, -v2, v3, 1.0
 ; GFX7-NEXT:    v_fma_f32 v3, v4, v3, v3
@@ -1034,13 +1030,13 @@ define amdgpu_kernel void @s_fdiv_v2f32(ptr addrspace(1) %out, <2 x float> %a, <
 ; GFX7-NEXT:    v_fma_f32 v4, v5, v3, v4
 ; GFX7-NEXT:    v_fma_f32 v0, -v2, v4, v0
 ; GFX7-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 0
-; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s8
 ; GFX7-NEXT:    v_div_fmas_f32 v0, v0, v3, v4
-; GFX7-NEXT:    v_div_scale_f32 v3, s[0:1], s8, s8, v2
+; GFX7-NEXT:    v_div_scale_f32 v3, s[4:5], s10, s10, v2
 ; GFX7-NEXT:    v_rcp_f32_e32 v4, v3
-; GFX7-NEXT:    v_div_fixup_f32 v1, v0, s9, v1
-; GFX7-NEXT:    v_mov_b32_e32 v0, s8
-; GFX7-NEXT:    v_div_scale_f32 v0, vcc, s2, v0, s2
+; GFX7-NEXT:    v_div_fixup_f32 v1, v0, s11, v1
+; GFX7-NEXT:    v_mov_b32_e32 v0, s10
+; GFX7-NEXT:    v_div_scale_f32 v0, vcc, s8, v0, s8
 ; GFX7-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 3
 ; GFX7-NEXT:    v_fma_f32 v5, -v3, v4, 1.0
 ; GFX7-NEXT:    v_fma_f32 v4, v5, v4, v4
@@ -1050,20 +1046,19 @@ define amdgpu_kernel void @s_fdiv_v2f32(ptr addrspace(1) %out, <2 x float> %a, <
 ; GFX7-NEXT:    v_fma_f32 v0, -v3, v5, v0
 ; GFX7-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 0
 ; GFX7-NEXT:    v_div_fmas_f32 v0, v0, v4, v5
-; GFX7-NEXT:    v_div_fixup_f32 v0, v0, s8, v2
-; GFX7-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; GFX7-NEXT:    v_div_fixup_f32 v0, v0, s10, v2
+; GFX7-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
 ; GFX7-NEXT:    s_endpgm
 ;
 ; GFX8-LABEL: s_fdiv_v2f32:
 ; GFX8:       ; %bb.0: ; %entry
-; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX8-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x34
+; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x2c
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    v_mov_b32_e32 v0, s3
-; GFX8-NEXT:    v_div_scale_f32 v1, s[6:7], s5, s5, v0
-; GFX8-NEXT:    v_mov_b32_e32 v2, s5
-; GFX8-NEXT:    v_div_scale_f32 v2, vcc, s3, v2, s3
-; GFX8-NEXT:    v_mov_b32_e32 v4, s2
+; GFX8-NEXT:    v_mov_b32_e32 v0, s1
+; GFX8-NEXT:    v_div_scale_f32 v1, s[6:7], s3, s3, v0
+; GFX8-NEXT:    v_mov_b32_e32 v2, s3
+; GFX8-NEXT:    v_div_scale_f32 v2, vcc, s1, v2, s1
+; GFX8-NEXT:    v_mov_b32_e32 v4, s0
 ; GFX8-NEXT:    v_rcp_f32_e32 v3, v1
 ; GFX8-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 3
 ; GFX8-NEXT:    v_fma_f32 v5, -v1, v3, 1.0
@@ -1073,12 +1068,13 @@ define amdgpu_kernel void @s_fdiv_v2f32(ptr addrspace(1) %out, <2 x float> %a, <
 ; GFX8-NEXT:    v_fma_f32 v5, v6, v3, v5
 ; GFX8-NEXT:    v_fma_f32 v1, -v1, v5, v2
 ; GFX8-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 0
-; GFX8-NEXT:    v_div_scale_f32 v2, s[6:7], s4, s4, v4
+; GFX8-NEXT:    v_div_scale_f32 v2, s[6:7], s2, s2, v4
 ; GFX8-NEXT:    v_div_fmas_f32 v1, v1, v3, v5
-; GFX8-NEXT:    v_mov_b32_e32 v3, s4
-; GFX8-NEXT:    v_div_scale_f32 v3, vcc, s2, v3, s2
+; GFX8-NEXT:    v_mov_b32_e32 v3, s2
+; GFX8-NEXT:    v_div_scale_f32 v3, vcc, s0, v3, s0
+; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
 ; GFX8-NEXT:    v_rcp_f32_e32 v5, v2
-; GFX8-NEXT:    v_div_fixup_f32 v1, v1, s5, v0
+; GFX8-NEXT:    v_div_fixup_f32 v1, v1, s3, v0
 ; GFX8-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 3
 ; GFX8-NEXT:    v_fma_f32 v0, -v2, v5, 1.0
 ; GFX8-NEXT:    v_fma_f32 v0, v0, v5, v5
@@ -1088,20 +1084,19 @@ define amdgpu_kernel void @s_fdiv_v2f32(ptr addrspace(1) %out, <2 x float> %a, <
 ; GFX8-NEXT:    v_fma_f32 v2, -v2, v5, v3
 ; GFX8-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 0
 ; GFX8-NEXT:    v_div_fmas_f32 v0, v2, v0, v5
-; GFX8-NEXT:    v_mov_b32_e32 v2, s0
+; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX8-NEXT:    v_mov_b32_e32 v3, s1
-; GFX8-NEXT:    v_div_fixup_f32 v0, v0, s4, v4
+; GFX8-NEXT:    v_mov_b32_e32 v2, s0
+; GFX8-NEXT:    v_div_fixup_f32 v0, v0, s2, v4
 ; GFX8-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
 ; GFX8-NEXT:    s_endpgm
 ;
 ; GFX10-LABEL: s_fdiv_v2f32:
 ; GFX10:       ; %bb.0: ; %entry
-; GFX10-NEXT:    s_clause 0x1
-; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX10-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x2c
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-NEXT:    v_div_scale_f32 v0, s4, s7, s7, s3
-; GFX10-NEXT:    v_div_scale_f32 v2, vcc_lo, s3, s7, s3
+; GFX10-NEXT:    v_div_scale_f32 v0, s6, s3, s3, s1
+; GFX10-NEXT:    v_div_scale_f32 v2, vcc_lo, s1, s3, s1
 ; GFX10-NEXT:    v_rcp_f32_e32 v1, v0
 ; GFX10-NEXT:    s_denorm_mode 15
 ; GFX10-NEXT:    v_fma_f32 v3, -v0, v1, 1.0
@@ -1111,11 +1106,12 @@ define amdgpu_kernel void @s_fdiv_v2f32(ptr addrspace(1) %out, <2 x float> %a, <
 ; GFX10-NEXT:    v_fmac_f32_e32 v3, v4, v1
 ; GFX10-NEXT:    v_fma_f32 v0, -v0, v3, v2
 ; GFX10-NEXT:    s_denorm_mode 12
-; GFX10-NEXT:    v_div_scale_f32 v2, s4, s6, s6, s2
+; GFX10-NEXT:    v_div_scale_f32 v2, s6, s2, s2, s0
 ; GFX10-NEXT:    v_div_fmas_f32 v0, v0, v1, v3
+; GFX10-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x24
 ; GFX10-NEXT:    v_rcp_f32_e32 v3, v2
-; GFX10-NEXT:    v_div_fixup_f32 v1, v0, s7, s3
-; GFX10-NEXT:    v_div_scale_f32 v0, vcc_lo, s2, s6, s2
+; GFX10-NEXT:    v_div_fixup_f32 v1, v0, s3, s1
+; GFX10-NEXT:    v_div_scale_f32 v0, vcc_lo, s0, s2, s0
 ; GFX10-NEXT:    s_denorm_mode 15
 ; GFX10-NEXT:    v_fma_f32 v4, -v2, v3, 1.0
 ; GFX10-NEXT:    v_fmac_f32_e32 v3, v4, v3
@@ -1126,18 +1122,19 @@ define amdgpu_kernel void @s_fdiv_v2f32(ptr addrspace(1) %out, <2 x float> %a, <
 ; GFX10-NEXT:    s_denorm_mode 12
 ; GFX10-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX10-NEXT:    v_div_fmas_f32 v0, v0, v3, v4
-; GFX10-NEXT:    v_div_fixup_f32 v0, v0, s6, s2
-; GFX10-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX10-NEXT:    v_div_fixup_f32 v0, v0, s2, s0
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    global_store_dwordx2 v2, v[0:1], s[6:7]
 ; GFX10-NEXT:    s_endpgm
 ;
 ; GFX11-LABEL: s_fdiv_v2f32:
 ; GFX11:       ; %bb.0: ; %entry
 ; GFX11-NEXT:    s_clause 0x1
-; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x2c
+; GFX11-NEXT:    s_load_b64 s[4:5], s[4:5], 0x24
 ; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX11-NEXT:    v_div_scale_f32 v0, null, s5, s5, s3
-; GFX11-NEXT:    v_div_scale_f32 v2, vcc_lo, s3, s5, s3
+; GFX11-NEXT:    v_div_scale_f32 v0, null, s3, s3, s1
+; GFX11-NEXT:    v_div_scale_f32 v2, vcc_lo, s1, s3, s1
 ; GFX11-NEXT:    v_rcp_f32_e32 v1, v0
 ; GFX11-NEXT:    s_denorm_mode 15
 ; GFX11-NEXT:    s_waitcnt_depctr depctr_va_vdst(0)
@@ -1148,11 +1145,11 @@ define amdgpu_kernel void @s_fdiv_v2f32(ptr addrspace(1) %out, <2 x float> %a, <
 ; GFX11-NEXT:    v_fmac_f32_e32 v3, v4, v1
 ; GFX11-NEXT:    v_fma_f32 v0, -v0, v3, v2
 ; GFX11-NEXT:    s_denorm_mode 12
-; GFX11-NEXT:    v_div_scale_f32 v2, null, s4, s4, s2
+; GFX11-NEXT:    v_div_scale_f32 v2, null, s2, s2, s0
 ; GFX11-NEXT:    v_div_fmas_f32 v0, v0, v1, v3
 ; GFX11-NEXT:    v_rcp_f32_e32 v3, v2
-; GFX11-NEXT:    v_div_fixup_f32 v1, v0, s5, s3
-; GFX11-NEXT:    v_div_scale_f32 v0, vcc_lo, s2, s4, s2
+; GFX11-NEXT:    v_div_fixup_f32 v1, v0, s3, s1
+; GFX11-NEXT:    v_div_scale_f32 v0, vcc_lo, s0, s2, s0
 ; GFX11-NEXT:    s_denorm_mode 15
 ; GFX11-NEXT:    s_waitcnt_depctr depctr_va_vdst(0)
 ; GFX11-NEXT:    v_fma_f32 v4, -v2, v3, 1.0
@@ -1164,8 +1161,8 @@ define amdgpu_kernel void @s_fdiv_v2f32(ptr addrspace(1) %out, <2 x float> %a, <
 ; GFX11-NEXT:    s_denorm_mode 12
 ; GFX11-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX11-NEXT:    v_div_fmas_f32 v0, v0, v3, v4
-; GFX11-NEXT:    v_div_fixup_f32 v0, v0, s4, s2
-; GFX11-NEXT:    global_store_b64 v2, v[0:1], s[0:1]
+; GFX11-NEXT:    v_div_fixup_f32 v0, v0, s2, s0
+; GFX11-NEXT:    global_store_b64 v2, v[0:1], s[4:5]
 ; GFX11-NEXT:    s_endpgm
 ;
 ; EG-LABEL: s_fdiv_v2f32:
@@ -1190,60 +1187,58 @@ entry:
 define amdgpu_kernel void @s_fdiv_ulp25_v2f32(ptr addrspace(1) %out, <2 x float> %a, <2 x float> %b) #0 {
 ; GFX67-LABEL: s_fdiv_ulp25_v2f32:
 ; GFX67:       ; %bb.0: ; %entry
-; GFX67-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0xd
-; GFX67-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GFX67-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0xb
+; GFX67-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x9
 ; GFX67-NEXT:    s_mov_b32 s7, 0xf000
 ; GFX67-NEXT:    s_mov_b32 s6, -1
 ; GFX67-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX67-NEXT:    v_rcp_f32_e32 v0, s8
-; GFX67-NEXT:    v_rcp_f32_e32 v1, s9
-; GFX67-NEXT:    s_mov_b32 s4, s0
-; GFX67-NEXT:    s_mov_b32 s5, s1
-; GFX67-NEXT:    v_mul_f32_e32 v0, s2, v0
-; GFX67-NEXT:    v_mul_f32_e32 v1, s3, v1
+; GFX67-NEXT:    v_rcp_f32_e32 v0, s2
+; GFX67-NEXT:    v_rcp_f32_e32 v1, s3
+; GFX67-NEXT:    v_mul_f32_e32 v0, s0, v0
+; GFX67-NEXT:    v_mul_f32_e32 v1, s1, v1
 ; GFX67-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
 ; GFX67-NEXT:    s_endpgm
 ;
 ; GFX8-LABEL: s_fdiv_ulp25_v2f32:
 ; GFX8:       ; %bb.0: ; %entry
-; GFX8-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
-; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x2c
+; GFX8-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x24
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    v_rcp_f32_e32 v2, s6
-; GFX8-NEXT:    v_rcp_f32_e32 v3, s7
-; GFX8-NEXT:    v_mov_b32_e32 v0, s0
-; GFX8-NEXT:    v_mov_b32_e32 v1, s1
-; GFX8-NEXT:    v_mul_f32_e32 v2, s2, v2
-; GFX8-NEXT:    v_mul_f32_e32 v3, s3, v3
-; GFX8-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
+; GFX8-NEXT:    v_rcp_f32_e32 v0, s2
+; GFX8-NEXT:    v_rcp_f32_e32 v1, s3
+; GFX8-NEXT:    v_mov_b32_e32 v2, s4
+; GFX8-NEXT:    v_mov_b32_e32 v3, s5
+; GFX8-NEXT:    v_mul_f32_e32 v0, s0, v0
+; GFX8-NEXT:    v_mul_f32_e32 v1, s1, v1
+; GFX8-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
 ; GFX8-NEXT:    s_endpgm
 ;
 ; GFX10-LABEL: s_fdiv_ulp25_v2f32:
 ; GFX10:       ; %bb.0: ; %entry
 ; GFX10-NEXT:    s_clause 0x1
-; GFX10-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
-; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x2c
+; GFX10-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x24
 ; GFX10-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-NEXT:    v_rcp_f32_e32 v0, s6
-; GFX10-NEXT:    v_rcp_f32_e32 v1, s7
-; GFX10-NEXT:    v_mul_f32_e32 v0, s2, v0
-; GFX10-NEXT:    v_mul_f32_e32 v1, s3, v1
-; GFX10-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX10-NEXT:    v_rcp_f32_e32 v0, s2
+; GFX10-NEXT:    v_rcp_f32_e32 v1, s3
+; GFX10-NEXT:    v_mul_f32_e32 v0, s0, v0
+; GFX10-NEXT:    v_mul_f32_e32 v1, s1, v1
+; GFX10-NEXT:    global_store_dwordx2 v2, v[0:1], s[6:7]
 ; GFX10-NEXT:    s_endpgm
 ;
 ; GFX11-LABEL: s_fdiv_ulp25_v2f32:
 ; GFX11:       ; %bb.0: ; %entry
 ; GFX11-NEXT:    s_clause 0x1
-; GFX11-NEXT:    s_load_b64 s[6:7], s[4:5], 0x34
-; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x2c
+; GFX11-NEXT:    s_load_b64 s[4:5], s[4:5], 0x24
 ; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX11-NEXT:    v_rcp_f32_e32 v0, s6
-; GFX11-NEXT:    v_rcp_f32_e32 v1, s7
+; GFX11-NEXT:    v_rcp_f32_e32 v0, s2
+; GFX11-NEXT:    v_rcp_f32_e32 v1, s3
 ; GFX11-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX11-NEXT:    s_waitcnt_depctr depctr_va_vdst(0)
-; GFX11-NEXT:    v_dual_mul_f32 v0, s2, v0 :: v_dual_mul_f32 v1, s3, v1
-; GFX11-NEXT:    global_store_b64 v2, v[0:1], s[0:1]
+; GFX11-NEXT:    v_dual_mul_f32 v0, s0, v0 :: v_dual_mul_f32 v1, s1, v1
+; GFX11-NEXT:    global_store_b64 v2, v[0:1], s[4:5]
 ; GFX11-NEXT:    s_endpgm
 ;
 ; EG-LABEL: s_fdiv_ulp25_v2f32:
@@ -1268,60 +1263,58 @@ entry:
 define amdgpu_kernel void @s_fdiv_v2f32_fast_math(ptr addrspace(1) %out, <2 x float> %a, <2 x float> %b) #0 {
 ; GFX67-LABEL: s_fdiv_v2f32_fast_math:
 ; GFX67:       ; %bb.0: ; %entry
-; GFX67-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0xd
-; GFX67-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GFX67-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0xb
+; GFX67-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x9
 ; GFX67-NEXT:    s_mov_b32 s7, 0xf000
 ; GFX67-NEXT:    s_mov_b32 s6, -1
 ; GFX67-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX67-NEXT:    v_rcp_f32_e32 v0, s9
-; GFX67-NEXT:    v_rcp_f32_e32 v2, s8
-; GFX67-NEXT:    s_mov_b32 s4, s0
-; GFX67-NEXT:    s_mov_b32 s5, s1
-; GFX67-NEXT:    v_mul_f32_e32 v1, s3, v0
-; GFX67-NEXT:    v_mul_f32_e32 v0, s2, v2
+; GFX67-NEXT:    v_rcp_f32_e32 v0, s3
+; GFX67-NEXT:    v_rcp_f32_e32 v2, s2
+; GFX67-NEXT:    v_mul_f32_e32 v1, s1, v0
+; GFX67-NEXT:    v_mul_f32_e32 v0, s0, v2
 ; GFX67-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
 ; GFX67-NEXT:    s_endpgm
 ;
 ; GFX8-LABEL: s_fdiv_v2f32_fast_math:
 ; GFX8:       ; %bb.0: ; %entry
-; GFX8-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
-; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x2c
+; GFX8-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x24
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    v_rcp_f32_e32 v2, s7
-; GFX8-NEXT:    v_rcp_f32_e32 v4, s6
-; GFX8-NEXT:    v_mov_b32_e32 v0, s0
-; GFX8-NEXT:    v_mov_b32_e32 v1, s1
-; GFX8-NEXT:    v_mul_f32_e32 v3, s3, v2
-; GFX8-NEXT:    v_mul_f32_e32 v2, s2, v4
-; GFX8-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
+; GFX8-NEXT:    v_rcp_f32_e32 v0, s3
+; GFX8-NEXT:    v_rcp_f32_e32 v2, s2
+; GFX8-NEXT:    v_mul_f32_e32 v1, s1, v0
+; GFX8-NEXT:    v_mul_f32_e32 v0, s0, v2
+; GFX8-NEXT:    v_mov_b32_e32 v2, s4
+; GFX8-NEXT:    v_mov_b32_e32 v3, s5
+; GFX8-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
 ; GFX8-NEXT:    s_endpgm
 ;
 ; GFX10-LABEL: s_fdiv_v2f32_fast_math:
 ; GFX10:       ; %bb.0: ; %entry
 ; GFX10-NEXT:    s_clause 0x1
-; GFX10-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
-; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x2c
+; GFX10-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x24
 ; GFX10-NEXT:    v_mov_b32_e32 v3, 0
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-NEXT:    v_rcp_f32_e32 v0, s7
-; GFX10-NEXT:    v_rcp_f32_e32 v2, s6
-; GFX10-NEXT:    v_mul_f32_e32 v1, s3, v0
-; GFX10-NEXT:    v_mul_f32_e32 v0, s2, v2
-; GFX10-NEXT:    global_store_dwordx2 v3, v[0:1], s[0:1]
+; GFX10-NEXT:    v_rcp_f32_e32 v0, s3
+; GFX10-NEXT:    v_rcp_f32_e32 v2, s2
+; GFX10-NEXT:    v_mul_f32_e32 v1, s1, v0
+; GFX10-NEXT:    v_mul_f32_e32 v0, s0, v2
+; GFX10-NEXT:    global_store_dwordx2 v3, v[0:1], s[6:7]
 ; GFX10-NEXT:    s_endpgm
 ;
 ; GFX11-LABEL: s_fdiv_v2f32_fast_math:
 ; GFX11:       ; %bb.0: ; %entry
 ; GFX11-NEXT:    s_clause 0x1
-; GFX11-NEXT:    s_load_b64 s[6:7], s[4:5], 0x34
-; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x2c
+; GFX11-NEXT:    s_load_b64 s[4:5], s[4:5], 0x24
 ; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX11-NEXT:    v_rcp_f32_e32 v0, s7
-; GFX11-NEXT:    v_rcp_f32_e32 v2, s6
+; GFX11-NEXT:    v_rcp_f32_e32 v0, s3
+; GFX11-NEXT:    v_rcp_f32_e32 v2, s2
 ; GFX11-NEXT:    v_mov_b32_e32 v3, 0
 ; GFX11-NEXT:    s_waitcnt_depctr depctr_va_vdst(0)
-; GFX11-NEXT:    v_dual_mul_f32 v1, s3, v0 :: v_dual_mul_f32 v0, s2, v2
-; GFX11-NEXT:    global_store_b64 v3, v[0:1], s[0:1]
+; GFX11-NEXT:    v_dual_mul_f32 v1, s1, v0 :: v_dual_mul_f32 v0, s0, v2
+; GFX11-NEXT:    global_store_b64 v3, v[0:1], s[4:5]
 ; GFX11-NEXT:    s_endpgm
 ;
 ; EG-LABEL: s_fdiv_v2f32_fast_math:
@@ -1346,60 +1339,58 @@ entry:
 define amdgpu_kernel void @s_fdiv_v2f32_arcp_math(ptr addrspace(1) %out, <2 x float> %a, <2 x float> %b) #0 {
 ; GFX67-LABEL: s_fdiv_v2f32_arcp_math:
 ; GFX67:       ; %bb.0: ; %entry
-; GFX67-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0xd
-; GFX67-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GFX67-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0xb
+; GFX67-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x9
 ; GFX67-NEXT:    s_mov_b32 s7, 0xf000
 ; GFX67-NEXT:    s_mov_b32 s6, -1
 ; GFX67-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX67-NEXT:    v_rcp_f32_e32 v0, s9
-; GFX67-NEXT:    v_rcp_f32_e32 v2, s8
-; GFX67-NEXT:    s_mov_b32 s4, s0
-; GFX67-NEXT:    s_mov_b32 s5, s1
-; GFX67-NEXT:    v_mul_f32_e32 v1, s3, v0
-; GFX67-NEXT:    v_mul_f32_e32 v0, s2, v2
+; GFX67-NEXT:    v_rcp_f32_e32 v0, s3
+; GFX67-NEXT:    v_rcp_f32_e32 v2, s2
+; GFX67-NEXT:    v_mul_f32_e32 v1, s1, v0
+; GFX67-NEXT:    v_mul_f32_e32 v0, s0, v2
 ; GFX67-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
 ; GFX67-NEXT:    s_endpgm
 ;
 ; GFX8-LABEL: s_fdiv_v2f32_arcp_math:
 ; GFX8:       ; %bb.0: ; %entry
-; GFX8-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
-; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x2c
+; GFX8-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x24
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    v_rcp_f32_e32 v2, s7
-; GFX8-NEXT:    v_rcp_f32_e32 v4, s6
-; GFX8-NEXT:    v_mov_b32_e32 v0, s0
-; GFX8-NEXT:    v_mov_b32_e32 v1, s1
-; GFX8-NEXT:    v_mul_f32_e32 v3, s3, v2
-; GFX8-NEXT:    v_mul_f32_e32 v2, s2, v4
-; GFX8-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
+; GFX8-NEXT:    v_rcp_f32_e32 v0, s3
+; GFX8-NEXT:    v_rcp_f32_e32 v2, s2
+; GFX8-NEXT:    v_mul_f32_e32 v1, s1, v0
+; GFX8-NEXT:    v_mul_f32_e32 v0, s0, v2
+; GFX8-NEXT:    v_mov_b32_e32 v2, s4
+; GFX8-NEXT:    v_mov_b32_e32 v3, s5
+; GFX8-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
 ; GFX8-NEXT:    s_endpgm
 ;
 ; GFX10-LABEL: s_fdiv_v2f32_arcp_math:
 ; GFX10:       ; %bb.0: ; %entry
 ; GFX10-NEXT:    s_clause 0x1
-; GFX10-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
-; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x2c
+; GFX10-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x24
 ; GFX10-NEXT:    v_mov_b32_e32 v3, 0
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-NEXT:    v_rcp_f32_e32 v0, s7
-; GFX10-NEXT:    v_rcp_f32_e32 v2, s6
-; GFX10-NEXT:    v_mul_f32_e32 v1, s3, v0
-; GFX10-NEXT:    v_mul_f32_e32 v0, s2, v2
-; GFX10-NEXT:    global_store_dwordx2 v3, v[0:1], s[0:1]
+; GFX10-NEXT:    v_rcp_f32_e32 v0, s3
+; GFX10-NEXT:    v_rcp_f32_e32 v2, s2
+; GFX10-NEXT:    v_mul_f32_e32 v1, s1, v0
+; GFX10-NEXT:    v_mul_f32_e32 v0, s0, v2
+; GFX10-NEXT:    global_store_dwordx2 v3, v[0:1], s[6:7]
 ; GFX10-NEXT:    s_endpgm
 ;
 ; GFX11-LABEL: s_fdiv_v2f32_arcp_math:
 ; GFX11:       ; %bb.0: ; %entry
 ; GFX11-NEXT:    s_clause 0x1
-; GFX11-NEXT:    s_load_b64 s[6:7], s[4:5], 0x34
-; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x2c
+; GFX11-NEXT:    s_load_b64 s[4:5], s[4:5], 0x24
 ; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX11-NEXT:    v_rcp_f32_e32 v0, s7
-; GFX11-NEXT:    v_rcp_f32_e32 v2, s6
+; GFX11-NEXT:    v_rcp_f32_e32 v0, s3
+; GFX11-NEXT:    v_rcp_f32_e32 v2, s2
 ; GFX11-NEXT:    v_mov_b32_e32 v3, 0
 ; GFX11-NEXT:    s_waitcnt_depctr depctr_va_vdst(0)
-; GFX11-NEXT:    v_dual_mul_f32 v1, s3, v0 :: v_dual_mul_f32 v0, s2, v2
-; GFX11-NEXT:    global_store_b64 v3, v[0:1], s[0:1]
+; GFX11-NEXT:    v_dual_mul_f32 v1, s1, v0 :: v_dual_mul_f32 v0, s0, v2
+; GFX11-NEXT:    global_store_b64 v3, v[0:1], s[4:5]
 ; GFX11-NEXT:    s_endpgm
 ;
 ; EG-LABEL: s_fdiv_v2f32_arcp_math:
diff --git a/llvm/test/CodeGen/AMDGPU/fnearbyint.ll b/llvm/test/CodeGen/AMDGPU/fnearbyint.ll
index 3e450b785b57b..6c2ab5fb15a20 100644
--- a/llvm/test/CodeGen/AMDGPU/fnearbyint.ll
+++ b/llvm/test/CodeGen/AMDGPU/fnearbyint.ll
@@ -121,25 +121,24 @@ define amdgpu_kernel void @fnearbyint_v2f32(ptr addrspace(1) %out, <2 x float> %
 ; SICI-LABEL: fnearbyint_v2f32:
 ; SICI:       ; %bb.0: ; %entry
 ; SICI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
-; SICI-NEXT:    s_mov_b32 s7, 0xf000
-; SICI-NEXT:    s_mov_b32 s6, -1
 ; SICI-NEXT:    s_waitcnt lgkmcnt(0)
-; SICI-NEXT:    s_mov_b32 s4, s0
-; SICI-NEXT:    s_mov_b32 s5, s1
-; SICI-NEXT:    v_rndne_f32_e32 v1, s3
-; SICI-NEXT:    v_rndne_f32_e32 v0, s2
-; SICI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; SICI-NEXT:    s_mov_b64 s[4:5], s[2:3]
+; SICI-NEXT:    s_mov_b32 s3, 0xf000
+; SICI-NEXT:    s_mov_b32 s2, -1
+; SICI-NEXT:    v_rndne_f32_e32 v1, s5
+; SICI-NEXT:    v_rndne_f32_e32 v0, s4
+; SICI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
 ; SICI-NEXT:    s_endpgm
 ;
 ; VI-LABEL: fnearbyint_v2f32:
 ; VI:       ; %bb.0: ; %entry
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
-; VI-NEXT:    v_mov_b32_e32 v0, s0
-; VI-NEXT:    v_mov_b32_e32 v1, s1
-; VI-NEXT:    v_rndne_f32_e32 v3, s3
-; VI-NEXT:    v_rndne_f32_e32 v2, s2
-; VI-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
+; VI-NEXT:    v_mov_b32_e32 v3, s1
+; VI-NEXT:    v_rndne_f32_e32 v1, s3
+; VI-NEXT:    v_rndne_f32_e32 v0, s2
+; VI-NEXT:    v_mov_b32_e32 v2, s0
+; VI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
 ; VI-NEXT:    s_endpgm
 ;
 ; GFX11-LABEL: fnearbyint_v2f32:
diff --git a/llvm/test/CodeGen/AMDGPU/fneg-fabs.bf16.ll b/llvm/test/CodeGen/AMDGPU/fneg-fabs.bf16.ll
index 468df77f5c2aa..5424ebfcffcd1 100644
--- a/llvm/test/CodeGen/AMDGPU/fneg-fabs.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/fneg-fabs.bf16.ll
@@ -624,13 +624,13 @@ define amdgpu_kernel void @fneg_fabs_v4bf16(ptr addrspace(1) %out, <4 x bfloat>
 ; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
-; CI-NEXT:    v_mov_b32_e32 v0, s0
-; CI-NEXT:    v_mov_b32_e32 v1, s1
-; CI-NEXT:    s_or_b32 s0, s3, 0x80008000
-; CI-NEXT:    s_or_b32 s1, s2, 0x80008000
-; CI-NEXT:    v_mov_b32_e32 v2, s1
-; CI-NEXT:    v_mov_b32_e32 v3, s0
-; CI-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
+; CI-NEXT:    s_or_b32 s3, s3, 0x80008000
+; CI-NEXT:    s_or_b32 s2, s2, 0x80008000
+; CI-NEXT:    v_mov_b32_e32 v3, s1
+; CI-NEXT:    v_mov_b32_e32 v0, s2
+; CI-NEXT:    v_mov_b32_e32 v1, s3
+; CI-NEXT:    v_mov_b32_e32 v2, s0
+; CI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
 ; CI-NEXT:    s_endpgm
 ;
 ; VI-LABEL: fneg_fabs_v4bf16:
@@ -640,25 +640,23 @@ define amdgpu_kernel void @fneg_fabs_v4bf16(ptr addrspace(1) %out, <4 x bfloat>
 ; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
-; VI-NEXT:    v_mov_b32_e32 v0, s0
-; VI-NEXT:    v_mov_b32_e32 v1, s1
-; VI-NEXT:    s_and_b32 s0, s2, 0x7fff7fff
-; VI-NEXT:    s_and_b32 s1, s3, 0x7fff7fff
-; VI-NEXT:    s_bfe_u32 s3, s3, 0xf0010
-; VI-NEXT:    s_bfe_u32 s2, s2, 0xf0010
-; VI-NEXT:    s_xor_b32 s1, s1, 0x8000
-; VI-NEXT:    s_xor_b32 s3, s3, 0x8000
-; VI-NEXT:    s_xor_b32 s0, s0, 0x8000
-; VI-NEXT:    s_xor_b32 s2, s2, 0x8000
-; VI-NEXT:    s_and_b32 s1, 0xffff, s1
-; VI-NEXT:    s_lshl_b32 s3, s3, 16
-; VI-NEXT:    s_and_b32 s0, 0xffff, s0
-; VI-NEXT:    s_lshl_b32 s2, s2, 16
-; VI-NEXT:    s_or_b32 s1, s1, s3
-; VI-NEXT:    s_or_b32 s0, s0, s2
-; VI-NEXT:    v_mov_b32_e32 v2, s0
+; VI-NEXT:    s_lshr_b32 s4, s2, 16
+; VI-NEXT:    s_lshr_b32 s5, s3, 16
+; VI-NEXT:    s_bitset1_b32 s3, 15
+; VI-NEXT:    s_bitset1_b32 s2, 15
+; VI-NEXT:    s_bitset1_b32 s5, 15
+; VI-NEXT:    s_bitset1_b32 s4, 15
+; VI-NEXT:    s_and_b32 s3, 0xffff, s3
+; VI-NEXT:    s_lshl_b32 s5, s5, 16
+; VI-NEXT:    s_and_b32 s2, 0xffff, s2
+; VI-NEXT:    s_lshl_b32 s4, s4, 16
+; VI-NEXT:    s_or_b32 s3, s3, s5
+; VI-NEXT:    s_or_b32 s2, s2, s4
 ; VI-NEXT:    v_mov_b32_e32 v3, s1
-; VI-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
+; VI-NEXT:    v_mov_b32_e32 v0, s2
+; VI-NEXT:    v_mov_b32_e32 v1, s3
+; VI-NEXT:    v_mov_b32_e32 v2, s0
+; VI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
 ; VI-NEXT:    s_endpgm
 ;
 ; GFX9-LABEL: fneg_fabs_v4bf16:
@@ -666,16 +664,14 @@ define amdgpu_kernel void @fneg_fabs_v4bf16(ptr addrspace(1) %out, <4 x bfloat>
 ; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; GFX9-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    s_and_b32 s4, s2, 0x7fff7fff
-; GFX9-NEXT:    s_and_b32 s5, s3, 0x7fff7fff
-; GFX9-NEXT:    s_bfe_u32 s3, s3, 0xf0010
-; GFX9-NEXT:    s_bfe_u32 s2, s2, 0xf0010
-; GFX9-NEXT:    s_xor_b32 s3, s3, 0x8000
-; GFX9-NEXT:    s_xor_b32 s5, s5, 0x8000
-; GFX9-NEXT:    s_xor_b32 s2, s2, 0x8000
-; GFX9-NEXT:    s_xor_b32 s4, s4, 0x8000
-; GFX9-NEXT:    s_pack_ll_b32_b16 s3, s5, s3
-; GFX9-NEXT:    s_pack_ll_b32_b16 s2, s4, s2
+; GFX9-NEXT:    s_lshr_b32 s4, s2, 16
+; GFX9-NEXT:    s_lshr_b32 s5, s3, 16
+; GFX9-NEXT:    s_bitset1_b32 s3, 15
+; GFX9-NEXT:    s_bitset1_b32 s2, 15
+; GFX9-NEXT:    s_bitset1_b32 s5, 15
+; GFX9-NEXT:    s_bitset1_b32 s4, 15
+; GFX9-NEXT:    s_pack_ll_b32_b16 s3, s3, s5
+; GFX9-NEXT:    s_pack_ll_b32_b16 s2, s2, s4
 ; GFX9-NEXT:    v_mov_b32_e32 v0, s2
 ; GFX9-NEXT:    v_mov_b32_e32 v1, s3
 ; GFX9-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1]
@@ -685,16 +681,14 @@ define amdgpu_kernel void @fneg_fabs_v4bf16(ptr addrspace(1) %out, <4 x bfloat>
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x0
 ; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX11-NEXT:    s_and_b32 s4, s2, 0x7fff7fff
-; GFX11-NEXT:    s_and_b32 s5, s3, 0x7fff7fff
-; GFX11-NEXT:    s_bfe_u32 s3, s3, 0xf0010
-; GFX11-NEXT:    s_bfe_u32 s2, s2, 0xf0010
-; GFX11-NEXT:    s_xor_b32 s3, s3, 0x8000
-; GFX11-NEXT:    s_xor_b32 s2, s2, 0x8000
-; GFX11-NEXT:    s_xor_b32 s4, s4, 0x8000
-; GFX11-NEXT:    s_xor_b32 s5, s5, 0x8000
-; GFX11-NEXT:    s_pack_ll_b32_b16 s2, s4, s2
-; GFX11-NEXT:    s_pack_ll_b32_b16 s3, s5, s3
+; GFX11-NEXT:    s_lshr_b32 s4, s2, 16
+; GFX11-NEXT:    s_lshr_b32 s5, s3, 16
+; GFX11-NEXT:    s_bitset1_b32 s3, 15
+; GFX11-NEXT:    s_bitset1_b32 s2, 15
+; GFX11-NEXT:    s_bitset1_b32 s4, 15
+; GFX11-NEXT:    s_bitset1_b32 s5, 15
+; GFX11-NEXT:    s_pack_ll_b32_b16 s2, s2, s4
+; GFX11-NEXT:    s_pack_ll_b32_b16 s3, s3, s5
 ; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
 ; GFX11-NEXT:    v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s3
 ; GFX11-NEXT:    v_mov_b32_e32 v0, s2
diff --git a/llvm/test/CodeGen/AMDGPU/fneg-fabs.f16.ll b/llvm/test/CodeGen/AMDGPU/fneg-fabs.f16.ll
index 305f4e56184cc..9d9a851a5507e 100644
--- a/llvm/test/CodeGen/AMDGPU/fneg-fabs.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/fneg-fabs.f16.ll
@@ -516,13 +516,13 @@ define amdgpu_kernel void @fneg_fabs_v4f16(ptr addrspace(1) %out, <4 x half> %in
 ; CIVI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; CIVI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CIVI-NEXT:    s_waitcnt lgkmcnt(0)
-; CIVI-NEXT:    v_mov_b32_e32 v0, s0
-; CIVI-NEXT:    v_mov_b32_e32 v1, s1
-; CIVI-NEXT:    s_or_b32 s0, s3, 0x80008000
-; CIVI-NEXT:    s_or_b32 s1, s2, 0x80008000
-; CIVI-NEXT:    v_mov_b32_e32 v2, s1
-; CIVI-NEXT:    v_mov_b32_e32 v3, s0
-; CIVI-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
+; CIVI-NEXT:    s_or_b32 s3, s3, 0x80008000
+; CIVI-NEXT:    s_or_b32 s2, s2, 0x80008000
+; CIVI-NEXT:    v_mov_b32_e32 v3, s1
+; CIVI-NEXT:    v_mov_b32_e32 v0, s2
+; CIVI-NEXT:    v_mov_b32_e32 v1, s3
+; CIVI-NEXT:    v_mov_b32_e32 v2, s0
+; CIVI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
 ; CIVI-NEXT:    s_endpgm
 ;
 ; GFX9-LABEL: fneg_fabs_v4f16:
diff --git a/llvm/test/CodeGen/AMDGPU/fneg-fabs.ll b/llvm/test/CodeGen/AMDGPU/fneg-fabs.ll
index b93a598cb52ae..214ccedd75170 100644
--- a/llvm/test/CodeGen/AMDGPU/fneg-fabs.ll
+++ b/llvm/test/CodeGen/AMDGPU/fneg-fabs.ll
@@ -199,29 +199,28 @@ define amdgpu_kernel void @fneg_fabsf_v2f32(ptr addrspace(1) %out, <2 x float> %
 ; SI-LABEL: fneg_fabsf_v2f32:
 ; SI:       ; %bb.0:
 ; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
-; SI-NEXT:    s_mov_b32 s7, 0xf000
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
-; SI-NEXT:    s_bitset1_b32 s3, 31
-; SI-NEXT:    s_bitset1_b32 s2, 31
-; SI-NEXT:    s_mov_b32 s6, -1
-; SI-NEXT:    s_mov_b32 s4, s0
-; SI-NEXT:    s_mov_b32 s5, s1
-; SI-NEXT:    v_mov_b32_e32 v0, s2
-; SI-NEXT:    v_mov_b32_e32 v1, s3
-; SI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; SI-NEXT:    s_mov_b64 s[4:5], s[2:3]
+; SI-NEXT:    s_mov_b32 s3, 0xf000
+; SI-NEXT:    s_bitset1_b32 s5, 31
+; SI-NEXT:    s_bitset1_b32 s4, 31
+; SI-NEXT:    s_mov_b32 s2, -1
+; SI-NEXT:    v_mov_b32_e32 v0, s4
+; SI-NEXT:    v_mov_b32_e32 v1, s5
+; SI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
 ; SI-NEXT:    s_endpgm
 ;
 ; VI-LABEL: fneg_fabsf_v2f32:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
-; VI-NEXT:    v_mov_b32_e32 v0, s0
-; VI-NEXT:    v_mov_b32_e32 v1, s1
-; VI-NEXT:    s_or_b32 s0, s3, 0x80000000
-; VI-NEXT:    s_or_b32 s1, s2, 0x80000000
-; VI-NEXT:    v_mov_b32_e32 v2, s1
-; VI-NEXT:    v_mov_b32_e32 v3, s0
-; VI-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
+; VI-NEXT:    s_bitset1_b32 s3, 31
+; VI-NEXT:    s_bitset1_b32 s2, 31
+; VI-NEXT:    v_mov_b32_e32 v3, s1
+; VI-NEXT:    v_mov_b32_e32 v0, s2
+; VI-NEXT:    v_mov_b32_e32 v1, s3
+; VI-NEXT:    v_mov_b32_e32 v2, s0
+; VI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
 ; VI-NEXT:    s_endpgm
   %fabs = call <2 x float> @llvm.fabs.v2f32(<2 x float> %in)
   %fsub = fsub <2 x float> <float -0.000000e+00, float -0.000000e+00>, %fabs
diff --git a/llvm/test/CodeGen/AMDGPU/fneg.ll b/llvm/test/CodeGen/AMDGPU/fneg.ll
index 17225b7c39f4f..02235151a83e1 100644
--- a/llvm/test/CodeGen/AMDGPU/fneg.ll
+++ b/llvm/test/CodeGen/AMDGPU/fneg.ll
@@ -52,29 +52,28 @@ define amdgpu_kernel void @s_fneg_v2f32(ptr addrspace(1) nocapture %out, <2 x fl
 ; SI-LABEL: s_fneg_v2f32:
 ; SI:       ; %bb.0:
 ; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
-; SI-NEXT:    s_mov_b32 s7, 0xf000
-; SI-NEXT:    s_mov_b32 s6, -1
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
-; SI-NEXT:    s_mov_b32 s4, s0
-; SI-NEXT:    s_mov_b32 s5, s1
-; SI-NEXT:    s_xor_b32 s0, s3, 0x80000000
-; SI-NEXT:    s_xor_b32 s1, s2, 0x80000000
-; SI-NEXT:    v_mov_b32_e32 v0, s1
-; SI-NEXT:    v_mov_b32_e32 v1, s0
-; SI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; SI-NEXT:    s_mov_b64 s[4:5], s[2:3]
+; SI-NEXT:    s_xor_b32 s5, s5, 0x80000000
+; SI-NEXT:    s_xor_b32 s4, s4, 0x80000000
+; SI-NEXT:    s_mov_b32 s3, 0xf000
+; SI-NEXT:    s_mov_b32 s2, -1
+; SI-NEXT:    v_mov_b32_e32 v0, s4
+; SI-NEXT:    v_mov_b32_e32 v1, s5
+; SI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
 ; SI-NEXT:    s_endpgm
 ;
 ; VI-LABEL: s_fneg_v2f32:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
-; VI-NEXT:    v_mov_b32_e32 v0, s0
-; VI-NEXT:    v_mov_b32_e32 v1, s1
-; VI-NEXT:    s_xor_b32 s0, s3, 0x80000000
-; VI-NEXT:    s_xor_b32 s1, s2, 0x80000000
-; VI-NEXT:    v_mov_b32_e32 v2, s1
-; VI-NEXT:    v_mov_b32_e32 v3, s0
-; VI-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
+; VI-NEXT:    s_xor_b32 s3, s3, 0x80000000
+; VI-NEXT:    s_xor_b32 s2, s2, 0x80000000
+; VI-NEXT:    v_mov_b32_e32 v3, s1
+; VI-NEXT:    v_mov_b32_e32 v0, s2
+; VI-NEXT:    v_mov_b32_e32 v1, s3
+; VI-NEXT:    v_mov_b32_e32 v2, s0
+; VI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
 ; VI-NEXT:    s_endpgm
 ;
 ; GFX11-LABEL: s_fneg_v2f32:
diff --git a/llvm/test/CodeGen/AMDGPU/fp_to_sint.ll b/llvm/test/CodeGen/AMDGPU/fp_to_sint.ll
index 35de9ccd99739..a2cd6d28e96cb 100644
--- a/llvm/test/CodeGen/AMDGPU/fp_to_sint.ll
+++ b/llvm/test/CodeGen/AMDGPU/fp_to_sint.ll
@@ -134,27 +134,24 @@ define amdgpu_kernel void @fp_to_sint_v2i32(ptr addrspace(1) %out, <2 x float> %
 ; SI-LABEL: fp_to_sint_v2i32:
 ; SI:       ; %bb.0:
 ; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
-; SI-NEXT:    s_mov_b32 s7, 0xf000
-; SI-NEXT:    s_mov_b32 s6, -1
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
-; SI-NEXT:    s_mov_b32 s4, s0
-; SI-NEXT:    s_mov_b32 s5, s1
-; SI-NEXT:    v_cvt_i32_f32_e32 v1, s3
-; SI-NEXT:    v_cvt_i32_f32_e32 v0, s2
-; SI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; SI-NEXT:    s_mov_b64 s[4:5], s[2:3]
+; SI-NEXT:    s_mov_b32 s3, 0xf000
+; SI-NEXT:    s_mov_b32 s2, -1
+; SI-NEXT:    v_cvt_i32_f32_e32 v1, s5
+; SI-NEXT:    v_cvt_i32_f32_e32 v0, s4
+; SI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
 ; SI-NEXT:    s_endpgm
 ;
 ; VI-LABEL: fp_to_sint_v2i32:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
-; VI-NEXT:    s_mov_b32 s7, 0xf000
-; VI-NEXT:    s_mov_b32 s6, -1
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_cvt_i32_f32_e32 v1, s3
 ; VI-NEXT:    v_cvt_i32_f32_e32 v0, s2
-; VI-NEXT:    s_mov_b32 s4, s0
-; VI-NEXT:    s_mov_b32 s5, s1
-; VI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; VI-NEXT:    s_mov_b32 s3, 0xf000
+; VI-NEXT:    s_mov_b32 s2, -1
+; VI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
 ; VI-NEXT:    s_endpgm
 ;
 ; GFX11-SDAG-LABEL: fp_to_sint_v2i32:
@@ -438,26 +435,25 @@ entry:
 define amdgpu_kernel void @fp_to_sint_v2i64(ptr addrspace(1) %out, <2 x float> %x) {
 ; SI-LABEL: fp_to_sint_v2i64:
 ; SI:       ; %bb.0:
-; SI-NEXT:    s_load_dwordx4 s[4:7], s[4:5], 0x9
+; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT:    s_mov_b32 s6, 0x2f800000
+; SI-NEXT:    s_mov_b32 s7, 0xcf800000
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_mov_b64 s[4:5], s[2:3]
 ; SI-NEXT:    s_mov_b32 s3, 0xf000
 ; SI-NEXT:    s_mov_b32 s2, -1
-; SI-NEXT:    s_mov_b32 s8, 0x2f800000
-; SI-NEXT:    s_mov_b32 s9, 0xcf800000
-; SI-NEXT:    s_waitcnt lgkmcnt(0)
-; SI-NEXT:    s_mov_b32 s0, s4
-; SI-NEXT:    s_mov_b32 s1, s5
-; SI-NEXT:    v_trunc_f32_e32 v0, s7
-; SI-NEXT:    v_trunc_f32_e32 v1, s6
-; SI-NEXT:    v_mul_f32_e64 v2, |v0|, s8
+; SI-NEXT:    v_trunc_f32_e32 v0, s5
+; SI-NEXT:    v_trunc_f32_e32 v1, s4
+; SI-NEXT:    v_mul_f32_e64 v2, |v0|, s6
 ; SI-NEXT:    v_ashrrev_i32_e32 v3, 31, v0
-; SI-NEXT:    v_mul_f32_e64 v4, |v1|, s8
+; SI-NEXT:    v_mul_f32_e64 v4, |v1|, s6
 ; SI-NEXT:    v_ashrrev_i32_e32 v5, 31, v1
 ; SI-NEXT:    v_floor_f32_e32 v2, v2
 ; SI-NEXT:    v_floor_f32_e32 v4, v4
 ; SI-NEXT:    v_cvt_u32_f32_e32 v6, v2
-; SI-NEXT:    v_fma_f32 v0, v2, s9, |v0|
+; SI-NEXT:    v_fma_f32 v0, v2, s7, |v0|
 ; SI-NEXT:    v_cvt_u32_f32_e32 v2, v4
-; SI-NEXT:    v_fma_f32 v1, v4, s9, |v1|
+; SI-NEXT:    v_fma_f32 v1, v4, s7, |v1|
 ; SI-NEXT:    v_cvt_u32_f32_e32 v0, v0
 ; SI-NEXT:    v_xor_b32_e32 v4, v6, v3
 ; SI-NEXT:    v_cvt_u32_f32_e32 v1, v1
@@ -474,36 +470,35 @@ define amdgpu_kernel void @fp_to_sint_v2i64(ptr addrspace(1) %out, <2 x float> %
 ; VI-LABEL: fp_to_sint_v2i64:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
-; VI-NEXT:    s_mov_b32 s8, 0x2f800000
-; VI-NEXT:    s_mov_b32 s7, 0xf000
-; VI-NEXT:    s_mov_b32 s6, -1
+; VI-NEXT:    s_mov_b32 s6, 0x2f800000
+; VI-NEXT:    s_mov_b32 s7, 0xcf800000
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
-; VI-NEXT:    v_trunc_f32_e32 v0, s3
-; VI-NEXT:    v_mul_f32_e64 v1, |v0|, s8
-; VI-NEXT:    s_mov_b32 s4, s0
+; VI-NEXT:    s_mov_b64 s[4:5], s[2:3]
+; VI-NEXT:    v_trunc_f32_e32 v0, s5
+; VI-NEXT:    v_mul_f32_e64 v1, |v0|, s6
 ; VI-NEXT:    v_floor_f32_e32 v1, v1
-; VI-NEXT:    s_mov_b32 s0, 0xcf800000
-; VI-NEXT:    v_fma_f32 v2, v1, s0, |v0|
-; VI-NEXT:    v_trunc_f32_e32 v4, s2
-; VI-NEXT:    v_cvt_u32_f32_e32 v2, v2
-; VI-NEXT:    v_mul_f32_e64 v3, |v4|, s8
-; VI-NEXT:    v_cvt_u32_f32_e32 v1, v1
-; VI-NEXT:    v_floor_f32_e32 v3, v3
-; VI-NEXT:    v_cvt_u32_f32_e32 v5, v3
-; VI-NEXT:    v_fma_f32 v3, v3, s0, |v4|
+; VI-NEXT:    v_cvt_u32_f32_e32 v2, v1
+; VI-NEXT:    v_fma_f32 v1, v1, s7, |v0|
 ; VI-NEXT:    v_ashrrev_i32_e32 v0, 31, v0
-; VI-NEXT:    v_cvt_u32_f32_e32 v6, v3
-; VI-NEXT:    v_xor_b32_e32 v2, v2, v0
+; VI-NEXT:    v_trunc_f32_e32 v4, s4
+; VI-NEXT:    v_xor_b32_e32 v3, v2, v0
+; VI-NEXT:    v_mul_f32_e64 v2, |v4|, s6
+; VI-NEXT:    v_cvt_u32_f32_e32 v1, v1
+; VI-NEXT:    v_floor_f32_e32 v2, v2
+; VI-NEXT:    v_cvt_u32_f32_e32 v5, v2
+; VI-NEXT:    v_fma_f32 v2, v2, s7, |v4|
+; VI-NEXT:    v_cvt_u32_f32_e32 v6, v2
 ; VI-NEXT:    v_xor_b32_e32 v1, v1, v0
-; VI-NEXT:    v_sub_u32_e32 v2, vcc, v2, v0
-; VI-NEXT:    v_subb_u32_e32 v3, vcc, v1, v0, vcc
+; VI-NEXT:    v_sub_u32_e32 v2, vcc, v1, v0
 ; VI-NEXT:    v_ashrrev_i32_e32 v1, 31, v4
+; VI-NEXT:    v_subb_u32_e32 v3, vcc, v3, v0, vcc
 ; VI-NEXT:    v_xor_b32_e32 v0, v6, v1
 ; VI-NEXT:    v_xor_b32_e32 v4, v5, v1
 ; VI-NEXT:    v_sub_u32_e32 v0, vcc, v0, v1
-; VI-NEXT:    s_mov_b32 s5, s1
+; VI-NEXT:    s_mov_b32 s3, 0xf000
+; VI-NEXT:    s_mov_b32 s2, -1
 ; VI-NEXT:    v_subb_u32_e32 v1, vcc, v4, v1, vcc
-; VI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[4:7], 0
+; VI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0
 ; VI-NEXT:    s_endpgm
 ;
 ; GFX11-SDAG-LABEL: fp_to_sint_v2i64:
@@ -1298,32 +1293,29 @@ define amdgpu_kernel void @fp_to_sint_v2f32_to_v2i16(ptr addrspace(1) %out, <2 x
 ; SI-LABEL: fp_to_sint_v2f32_to_v2i16:
 ; SI:       ; %bb.0:
 ; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
-; SI-NEXT:    s_mov_b32 s7, 0xf000
-; SI-NEXT:    s_mov_b32 s6, -1
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
-; SI-NEXT:    s_mov_b32 s4, s0
-; SI-NEXT:    s_mov_b32 s5, s1
-; SI-NEXT:    v_cvt_i32_f32_e32 v0, s3
-; SI-NEXT:    v_cvt_i32_f32_e32 v1, s2
+; SI-NEXT:    s_mov_b64 s[4:5], s[2:3]
+; SI-NEXT:    s_mov_b32 s3, 0xf000
+; SI-NEXT:    s_mov_b32 s2, -1
+; SI-NEXT:    v_cvt_i32_f32_e32 v0, s5
+; SI-NEXT:    v_cvt_i32_f32_e32 v1, s4
 ; SI-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
 ; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v1
 ; SI-NEXT:    v_or_b32_e32 v0, v1, v0
-; SI-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; SI-NEXT:    s_endpgm
 ;
 ; VI-LABEL: fp_to_sint_v2f32_to_v2i16:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
-; VI-NEXT:    s_mov_b32 s7, 0xf000
-; VI-NEXT:    s_mov_b32 s6, -1
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v0, s3
 ; VI-NEXT:    v_cvt_i32_f32_sdwa v0, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD
 ; VI-NEXT:    v_cvt_i32_f32_e32 v1, s2
-; VI-NEXT:    s_mov_b32 s4, s0
-; VI-NEXT:    s_mov_b32 s5, s1
+; VI-NEXT:    s_mov_b32 s3, 0xf000
+; VI-NEXT:    s_mov_b32 s2, -1
 ; VI-NEXT:    v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; VI-NEXT:    s_endpgm
 ;
 ; GFX11-SDAG-LABEL: fp_to_sint_v2f32_to_v2i16:
diff --git a/llvm/test/CodeGen/AMDGPU/fp_to_uint.ll b/llvm/test/CodeGen/AMDGPU/fp_to_uint.ll
index 106d1116c2bc6..32f80ff6c22f8 100644
--- a/llvm/test/CodeGen/AMDGPU/fp_to_uint.ll
+++ b/llvm/test/CodeGen/AMDGPU/fp_to_uint.ll
@@ -72,27 +72,24 @@ define amdgpu_kernel void @fp_to_uint_v2f32_to_v2i32(ptr addrspace(1) %out, <2 x
 ; SI-LABEL: fp_to_uint_v2f32_to_v2i32:
 ; SI:       ; %bb.0:
 ; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
-; SI-NEXT:    s_mov_b32 s7, 0xf000
-; SI-NEXT:    s_mov_b32 s6, -1
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
-; SI-NEXT:    s_mov_b32 s4, s0
-; SI-NEXT:    s_mov_b32 s5, s1
-; SI-NEXT:    v_cvt_u32_f32_e32 v1, s3
-; SI-NEXT:    v_cvt_u32_f32_e32 v0, s2
-; SI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; SI-NEXT:    s_mov_b64 s[4:5], s[2:3]
+; SI-NEXT:    s_mov_b32 s3, 0xf000
+; SI-NEXT:    s_mov_b32 s2, -1
+; SI-NEXT:    v_cvt_u32_f32_e32 v1, s5
+; SI-NEXT:    v_cvt_u32_f32_e32 v0, s4
+; SI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
 ; SI-NEXT:    s_endpgm
 ;
 ; VI-LABEL: fp_to_uint_v2f32_to_v2i32:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
-; VI-NEXT:    s_mov_b32 s7, 0xf000
-; VI-NEXT:    s_mov_b32 s6, -1
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_cvt_u32_f32_e32 v1, s3
 ; VI-NEXT:    v_cvt_u32_f32_e32 v0, s2
-; VI-NEXT:    s_mov_b32 s4, s0
-; VI-NEXT:    s_mov_b32 s5, s1
-; VI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; VI-NEXT:    s_mov_b32 s3, 0xf000
+; VI-NEXT:    s_mov_b32 s2, -1
+; VI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
 ; VI-NEXT:    s_endpgm
 ;
 ; GFX11-SDAG-LABEL: fp_to_uint_v2f32_to_v2i32:
@@ -349,32 +346,29 @@ define amdgpu_kernel void @fp_to_uint_v2f32_to_v2i64(ptr addrspace(1) %out, <2 x
 ; SI-LABEL: fp_to_uint_v2f32_to_v2i64:
 ; SI:       ; %bb.0:
 ; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
-; SI-NEXT:    s_mov_b32 s7, 0xf000
-; SI-NEXT:    s_mov_b32 s6, -1
-; SI-NEXT:    s_mov_b32 s8, 0xcf800000
+; SI-NEXT:    s_mov_b32 s6, 0xcf800000
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
-; SI-NEXT:    s_mov_b32 s4, s0
-; SI-NEXT:    s_mov_b32 s5, s1
-; SI-NEXT:    v_trunc_f32_e32 v0, s3
-; SI-NEXT:    v_trunc_f32_e32 v2, s2
+; SI-NEXT:    s_mov_b64 s[4:5], s[2:3]
+; SI-NEXT:    s_mov_b32 s3, 0xf000
+; SI-NEXT:    s_mov_b32 s2, -1
+; SI-NEXT:    v_trunc_f32_e32 v0, s5
+; SI-NEXT:    v_trunc_f32_e32 v2, s4
 ; SI-NEXT:    v_mul_f32_e32 v1, 0x2f800000, v0
 ; SI-NEXT:    v_mul_f32_e32 v3, 0x2f800000, v2
 ; SI-NEXT:    v_floor_f32_e32 v4, v1
 ; SI-NEXT:    v_floor_f32_e32 v5, v3
 ; SI-NEXT:    v_cvt_u32_f32_e32 v3, v4
 ; SI-NEXT:    v_cvt_u32_f32_e32 v1, v5
-; SI-NEXT:    v_fma_f32 v0, v4, s8, v0
-; SI-NEXT:    v_fma_f32 v4, v5, s8, v2
+; SI-NEXT:    v_fma_f32 v0, v4, s6, v0
+; SI-NEXT:    v_fma_f32 v4, v5, s6, v2
 ; SI-NEXT:    v_cvt_u32_f32_e32 v2, v0
 ; SI-NEXT:    v_cvt_u32_f32_e32 v0, v4
-; SI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[4:7], 0
+; SI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0
 ; SI-NEXT:    s_endpgm
 ;
 ; VI-LABEL: fp_to_uint_v2f32_to_v2i64:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
-; VI-NEXT:    s_mov_b32 s7, 0xf000
-; VI-NEXT:    s_mov_b32 s6, -1
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_trunc_f32_e32 v0, s3
 ; VI-NEXT:    v_trunc_f32_e32 v4, s2
@@ -389,9 +383,9 @@ define amdgpu_kernel void @fp_to_uint_v2f32_to_v2i64(ptr addrspace(1) %out, <2 x
 ; VI-NEXT:    v_cvt_u32_f32_e32 v3, v5
 ; VI-NEXT:    v_cvt_u32_f32_e32 v1, v6
 ; VI-NEXT:    v_cvt_u32_f32_e32 v0, v0
-; VI-NEXT:    s_mov_b32 s4, s0
-; VI-NEXT:    s_mov_b32 s5, s1
-; VI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[4:7], 0
+; VI-NEXT:    s_mov_b32 s3, 0xf000
+; VI-NEXT:    s_mov_b32 s2, -1
+; VI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0
 ; VI-NEXT:    s_endpgm
 ;
 ; GFX11-SDAG-LABEL: fp_to_uint_v2f32_to_v2i64:
@@ -1078,31 +1072,28 @@ define amdgpu_kernel void @fp_to_uint_v2f32_to_v2i16(ptr addrspace(1) %out, <2 x
 ; SI-LABEL: fp_to_uint_v2f32_to_v2i16:
 ; SI:       ; %bb.0:
 ; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
-; SI-NEXT:    s_mov_b32 s7, 0xf000
-; SI-NEXT:    s_mov_b32 s6, -1
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
-; SI-NEXT:    s_mov_b32 s4, s0
-; SI-NEXT:    s_mov_b32 s5, s1
-; SI-NEXT:    v_cvt_u32_f32_e32 v0, s3
+; SI-NEXT:    s_mov_b64 s[4:5], s[2:3]
+; SI-NEXT:    s_mov_b32 s3, 0xf000
+; SI-NEXT:    s_mov_b32 s2, -1
+; SI-NEXT:    v_cvt_u32_f32_e32 v0, s5
 ; SI-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
-; SI-NEXT:    v_cvt_u32_f32_e32 v1, s2
+; SI-NEXT:    v_cvt_u32_f32_e32 v1, s4
 ; SI-NEXT:    v_or_b32_e32 v0, v1, v0
-; SI-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; SI-NEXT:    s_endpgm
 ;
 ; VI-LABEL: fp_to_uint_v2f32_to_v2i16:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
-; VI-NEXT:    s_mov_b32 s7, 0xf000
-; VI-NEXT:    s_mov_b32 s6, -1
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v0, s3
 ; VI-NEXT:    v_cvt_u32_f32_sdwa v0, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD
 ; VI-NEXT:    v_cvt_u32_f32_e32 v1, s2
-; VI-NEXT:    s_mov_b32 s4, s0
-; VI-NEXT:    s_mov_b32 s5, s1
+; VI-NEXT:    s_mov_b32 s3, 0xf000
+; VI-NEXT:    s_mov_b32 s2, -1
 ; VI-NEXT:    v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; VI-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; VI-NEXT:    s_endpgm
 ;
 ; GFX11-SDAG-LABEL: fp_to_uint_v2f32_to_v2i16:
diff --git a/llvm/test/CodeGen/AMDGPU/fshl.ll b/llvm/test/CodeGen/AMDGPU/fshl.ll
index 0366d618249df..72c2003058a01 100644
--- a/llvm/test/CodeGen/AMDGPU/fshl.ll
+++ b/llvm/test/CodeGen/AMDGPU/fshl.ll
@@ -209,81 +209,85 @@ entry:
 define amdgpu_kernel void @fshl_v2i32(ptr addrspace(1) %in, <2 x i32> %x, <2 x i32> %y, <2 x i32> %z) {
 ; SI-LABEL: fshl_v2i32:
 ; SI:       ; %bb.0: ; %entry
-; SI-NEXT:    s_load_dwordx8 s[0:7], s[4:5], 0x9
+; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0xb
+; SI-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x9
+; SI-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xf
 ; SI-NEXT:    s_mov_b32 s11, 0xf000
 ; SI-NEXT:    s_mov_b32 s10, -1
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
-; SI-NEXT:    s_mov_b32 s8, s0
-; SI-NEXT:    s_mov_b32 s9, s1
-; SI-NEXT:    s_mov_b32 s0, s5
-; SI-NEXT:    s_mov_b32 s1, s3
-; SI-NEXT:    s_lshr_b32 s12, s3, 1
-; SI-NEXT:    s_lshr_b64 s[0:1], s[0:1], 1
-; SI-NEXT:    s_not_b32 s3, s7
-; SI-NEXT:    s_mov_b32 s1, s12
-; SI-NEXT:    s_and_b32 s3, s3, 31
-; SI-NEXT:    s_lshr_b64 s[0:1], s[0:1], s3
-; SI-NEXT:    s_mov_b32 s5, s2
-; SI-NEXT:    s_lshr_b32 s1, s2, 1
-; SI-NEXT:    s_lshr_b64 s[2:3], s[4:5], 1
-; SI-NEXT:    s_mov_b32 s3, s1
-; SI-NEXT:    s_not_b32 s1, s6
+; SI-NEXT:    s_mov_b32 s6, s3
+; SI-NEXT:    s_mov_b32 s7, s1
+; SI-NEXT:    s_lshr_b32 s12, s1, 1
+; SI-NEXT:    s_lshr_b64 s[6:7], s[6:7], 1
+; SI-NEXT:    s_not_b32 s1, s5
+; SI-NEXT:    s_mov_b32 s7, s12
 ; SI-NEXT:    s_and_b32 s1, s1, 31
-; SI-NEXT:    s_lshr_b64 s[2:3], s[2:3], s1
-; SI-NEXT:    v_mov_b32_e32 v0, s2
-; SI-NEXT:    v_mov_b32_e32 v1, s0
+; SI-NEXT:    s_mov_b32 s3, s0
+; SI-NEXT:    s_lshr_b64 s[6:7], s[6:7], s1
+; SI-NEXT:    s_lshr_b32 s5, s0, 1
+; SI-NEXT:    s_lshr_b64 s[0:1], s[2:3], 1
+; SI-NEXT:    s_not_b32 s2, s4
+; SI-NEXT:    s_mov_b32 s1, s5
+; SI-NEXT:    s_and_b32 s2, s2, 31
+; SI-NEXT:    s_lshr_b64 s[0:1], s[0:1], s2
+; SI-NEXT:    v_mov_b32_e32 v0, s0
+; SI-NEXT:    v_mov_b32_e32 v1, s6
 ; SI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[8:11], 0
 ; SI-NEXT:    s_endpgm
 ;
 ; VI-LABEL: fshl_v2i32:
 ; VI:       ; %bb.0: ; %entry
-; VI-NEXT:    s_load_dwordx8 s[0:7], s[4:5], 0x24
+; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x2c
+; VI-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x24
+; VI-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x3c
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
-; VI-NEXT:    v_mov_b32_e32 v0, s0
-; VI-NEXT:    v_mov_b32_e32 v1, s1
-; VI-NEXT:    s_mov_b32 s0, s5
-; VI-NEXT:    s_mov_b32 s1, s3
-; VI-NEXT:    s_lshr_b32 s8, s3, 1
-; VI-NEXT:    s_lshr_b64 s[0:1], s[0:1], 1
-; VI-NEXT:    s_not_b32 s3, s7
-; VI-NEXT:    s_mov_b32 s1, s8
-; VI-NEXT:    s_and_b32 s3, s3, 31
-; VI-NEXT:    s_lshr_b64 s[0:1], s[0:1], s3
-; VI-NEXT:    s_mov_b32 s5, s2
-; VI-NEXT:    s_lshr_b32 s1, s2, 1
-; VI-NEXT:    s_lshr_b64 s[2:3], s[4:5], 1
-; VI-NEXT:    s_mov_b32 s3, s1
-; VI-NEXT:    s_not_b32 s1, s6
+; VI-NEXT:    s_mov_b32 s8, s3
+; VI-NEXT:    s_mov_b32 s9, s1
+; VI-NEXT:    s_lshr_b32 s10, s1, 1
+; VI-NEXT:    s_lshr_b64 s[8:9], s[8:9], 1
+; VI-NEXT:    s_not_b32 s1, s5
+; VI-NEXT:    s_mov_b32 s9, s10
 ; VI-NEXT:    s_and_b32 s1, s1, 31
-; VI-NEXT:    s_lshr_b64 s[2:3], s[2:3], s1
-; VI-NEXT:    v_mov_b32_e32 v2, s2
-; VI-NEXT:    v_mov_b32_e32 v3, s0
-; VI-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
+; VI-NEXT:    s_mov_b32 s3, s0
+; VI-NEXT:    s_lshr_b64 s[8:9], s[8:9], s1
+; VI-NEXT:    s_lshr_b32 s5, s0, 1
+; VI-NEXT:    s_lshr_b64 s[0:1], s[2:3], 1
+; VI-NEXT:    s_not_b32 s2, s4
+; VI-NEXT:    s_mov_b32 s1, s5
+; VI-NEXT:    s_and_b32 s2, s2, 31
+; VI-NEXT:    s_lshr_b64 s[0:1], s[0:1], s2
+; VI-NEXT:    v_mov_b32_e32 v2, s6
+; VI-NEXT:    v_mov_b32_e32 v0, s0
+; VI-NEXT:    v_mov_b32_e32 v1, s8
+; VI-NEXT:    v_mov_b32_e32 v3, s7
+; VI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
 ; VI-NEXT:    s_endpgm
 ;
 ; GFX9-LABEL: fshl_v2i32:
 ; GFX9:       ; %bb.0: ; %entry
-; GFX9-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x24
+; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x2c
+; GFX9-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x24
+; GFX9-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x3c
 ; GFX9-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    s_mov_b32 s0, s13
-; GFX9-NEXT:    s_mov_b32 s1, s11
-; GFX9-NEXT:    s_lshr_b32 s2, s11, 1
-; GFX9-NEXT:    s_not_b32 s3, s15
-; GFX9-NEXT:    s_lshr_b64 s[0:1], s[0:1], 1
-; GFX9-NEXT:    s_mov_b32 s1, s2
-; GFX9-NEXT:    s_and_b32 s2, s3, 31
-; GFX9-NEXT:    s_lshr_b64 s[0:1], s[0:1], s2
-; GFX9-NEXT:    s_mov_b32 s13, s10
-; GFX9-NEXT:    s_lshr_b32 s1, s10, 1
-; GFX9-NEXT:    s_lshr_b64 s[2:3], s[12:13], 1
-; GFX9-NEXT:    s_mov_b32 s3, s1
-; GFX9-NEXT:    s_not_b32 s1, s14
+; GFX9-NEXT:    s_mov_b32 s4, s3
+; GFX9-NEXT:    s_mov_b32 s5, s1
+; GFX9-NEXT:    s_lshr_b32 s10, s1, 1
+; GFX9-NEXT:    s_lshr_b64 s[4:5], s[4:5], 1
+; GFX9-NEXT:    s_not_b32 s1, s9
+; GFX9-NEXT:    s_mov_b32 s5, s10
 ; GFX9-NEXT:    s_and_b32 s1, s1, 31
-; GFX9-NEXT:    s_lshr_b64 s[2:3], s[2:3], s1
-; GFX9-NEXT:    v_mov_b32_e32 v0, s2
-; GFX9-NEXT:    v_mov_b32_e32 v1, s0
-; GFX9-NEXT:    global_store_dwordx2 v2, v[0:1], s[8:9]
+; GFX9-NEXT:    s_lshr_b64 s[4:5], s[4:5], s1
+; GFX9-NEXT:    s_mov_b32 s3, s0
+; GFX9-NEXT:    s_lshr_b32 s5, s0, 1
+; GFX9-NEXT:    s_lshr_b64 s[0:1], s[2:3], 1
+; GFX9-NEXT:    s_not_b32 s2, s8
+; GFX9-NEXT:    s_mov_b32 s1, s5
+; GFX9-NEXT:    s_and_b32 s2, s2, 31
+; GFX9-NEXT:    s_lshr_b64 s[0:1], s[0:1], s2
+; GFX9-NEXT:    v_mov_b32_e32 v0, s0
+; GFX9-NEXT:    v_mov_b32_e32 v1, s4
+; GFX9-NEXT:    global_store_dwordx2 v2, v[0:1], s[6:7]
 ; GFX9-NEXT:    s_endpgm
 ;
 ; R600-LABEL: fshl_v2i32:
@@ -306,24 +310,27 @@ define amdgpu_kernel void @fshl_v2i32(ptr addrspace(1) %in, <2 x i32> %x, <2 x i
 ;
 ; GFX10-LABEL: fshl_v2i32:
 ; GFX10:       ; %bb.0: ; %entry
-; GFX10-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x24
+; GFX10-NEXT:    s_clause 0x2
+; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x2c
+; GFX10-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x3c
+; GFX10-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x24
 ; GFX10-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-NEXT:    s_mov_b32 s0, s13
-; GFX10-NEXT:    s_mov_b32 s1, s11
-; GFX10-NEXT:    s_not_b32 s2, s15
-; GFX10-NEXT:    s_mov_b32 s13, s10
-; GFX10-NEXT:    s_lshr_b32 s4, s11, 1
-; GFX10-NEXT:    s_lshr_b32 s5, s10, 1
-; GFX10-NEXT:    s_not_b32 s6, s14
-; GFX10-NEXT:    s_lshr_b64 s[0:1], s[0:1], 1
-; GFX10-NEXT:    s_and_b32 s7, s2, 31
-; GFX10-NEXT:    s_lshr_b64 s[2:3], s[12:13], 1
-; GFX10-NEXT:    s_and_b32 s6, s6, 31
-; GFX10-NEXT:    s_mov_b32 s3, s5
-; GFX10-NEXT:    s_mov_b32 s1, s4
-; GFX10-NEXT:    s_lshr_b64 s[2:3], s[2:3], s6
-; GFX10-NEXT:    s_lshr_b64 s[0:1], s[0:1], s7
+; GFX10-NEXT:    s_mov_b32 s4, s3
+; GFX10-NEXT:    s_mov_b32 s5, s1
+; GFX10-NEXT:    s_mov_b32 s3, s0
+; GFX10-NEXT:    s_lshr_b32 s10, s1, 1
+; GFX10-NEXT:    s_not_b32 s7, s7
+; GFX10-NEXT:    s_lshr_b32 s11, s0, 1
+; GFX10-NEXT:    s_not_b32 s6, s6
+; GFX10-NEXT:    s_lshr_b64 s[0:1], s[4:5], 1
+; GFX10-NEXT:    s_lshr_b64 s[2:3], s[2:3], 1
+; GFX10-NEXT:    s_and_b32 s4, s7, 31
+; GFX10-NEXT:    s_and_b32 s5, s6, 31
+; GFX10-NEXT:    s_mov_b32 s3, s11
+; GFX10-NEXT:    s_mov_b32 s1, s10
+; GFX10-NEXT:    s_lshr_b64 s[2:3], s[2:3], s5
+; GFX10-NEXT:    s_lshr_b64 s[0:1], s[0:1], s4
 ; GFX10-NEXT:    v_mov_b32_e32 v0, s2
 ; GFX10-NEXT:    v_mov_b32_e32 v1, s0
 ; GFX10-NEXT:    global_store_dwordx2 v2, v[0:1], s[8:9]
@@ -331,27 +338,30 @@ define amdgpu_kernel void @fshl_v2i32(ptr addrspace(1) %in, <2 x i32> %x, <2 x i
 ;
 ; GFX11-LABEL: fshl_v2i32:
 ; GFX11:       ; %bb.0: ; %entry
-; GFX11-NEXT:    s_load_b256 s[0:7], s[4:5], 0x24
+; GFX11-NEXT:    s_clause 0x2
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x2c
+; GFX11-NEXT:    s_load_b64 s[6:7], s[4:5], 0x3c
+; GFX11-NEXT:    s_load_b64 s[4:5], s[4:5], 0x24
 ; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX11-NEXT:    s_mov_b32 s8, s5
-; GFX11-NEXT:    s_mov_b32 s9, s3
-; GFX11-NEXT:    s_mov_b32 s5, s2
-; GFX11-NEXT:    s_lshr_b32 s10, s3, 1
+; GFX11-NEXT:    s_mov_b32 s8, s3
+; GFX11-NEXT:    s_mov_b32 s9, s1
+; GFX11-NEXT:    s_mov_b32 s3, s0
+; GFX11-NEXT:    s_lshr_b32 s10, s1, 1
 ; GFX11-NEXT:    s_not_b32 s7, s7
-; GFX11-NEXT:    s_lshr_b32 s11, s2, 1
+; GFX11-NEXT:    s_lshr_b32 s11, s0, 1
 ; GFX11-NEXT:    s_not_b32 s6, s6
-; GFX11-NEXT:    s_lshr_b64 s[2:3], s[8:9], 1
-; GFX11-NEXT:    s_lshr_b64 s[4:5], s[4:5], 1
+; GFX11-NEXT:    s_lshr_b64 s[0:1], s[8:9], 1
+; GFX11-NEXT:    s_lshr_b64 s[2:3], s[2:3], 1
 ; GFX11-NEXT:    s_and_b32 s7, s7, 31
 ; GFX11-NEXT:    s_and_b32 s6, s6, 31
-; GFX11-NEXT:    s_mov_b32 s5, s11
-; GFX11-NEXT:    s_mov_b32 s3, s10
-; GFX11-NEXT:    s_lshr_b64 s[4:5], s[4:5], s6
-; GFX11-NEXT:    s_lshr_b64 s[2:3], s[2:3], s7
+; GFX11-NEXT:    s_mov_b32 s3, s11
+; GFX11-NEXT:    s_mov_b32 s1, s10
+; GFX11-NEXT:    s_lshr_b64 s[2:3], s[2:3], s6
+; GFX11-NEXT:    s_lshr_b64 s[0:1], s[0:1], s7
 ; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT:    v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s2
-; GFX11-NEXT:    v_mov_b32_e32 v0, s4
-; GFX11-NEXT:    global_store_b64 v2, v[0:1], s[0:1]
+; GFX11-NEXT:    v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s0
+; GFX11-NEXT:    v_mov_b32_e32 v0, s2
+; GFX11-NEXT:    global_store_b64 v2, v[0:1], s[4:5]
 ; GFX11-NEXT:    s_endpgm
 entry:
   %0 = call <2 x i32> @llvm.fshl.v2i32(<2 x i32> %x, <2 x i32> %y, <2 x i32> %z)
@@ -362,54 +372,52 @@ entry:
 define amdgpu_kernel void @fshl_v2i32_imm(ptr addrspace(1) %in, <2 x i32> %x, <2 x i32> %y) {
 ; SI-LABEL: fshl_v2i32_imm:
 ; SI:       ; %bb.0: ; %entry
-; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
-; SI-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0xd
+; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0xb
+; SI-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x9
 ; SI-NEXT:    s_mov_b32 s7, 0xf000
 ; SI-NEXT:    s_mov_b32 s6, -1
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
-; SI-NEXT:    s_mov_b32 s4, s0
-; SI-NEXT:    s_mov_b32 s5, s1
-; SI-NEXT:    s_mov_b32 s0, s9
-; SI-NEXT:    s_mov_b32 s1, s3
-; SI-NEXT:    s_mov_b32 s9, s2
-; SI-NEXT:    s_lshr_b64 s[0:1], s[0:1], 23
-; SI-NEXT:    s_lshr_b64 s[2:3], s[8:9], 25
-; SI-NEXT:    v_mov_b32_e32 v0, s2
-; SI-NEXT:    v_mov_b32_e32 v1, s0
+; SI-NEXT:    s_mov_b32 s8, s3
+; SI-NEXT:    s_mov_b32 s9, s1
+; SI-NEXT:    s_mov_b32 s3, s0
+; SI-NEXT:    s_lshr_b64 s[8:9], s[8:9], 23
+; SI-NEXT:    s_lshr_b64 s[0:1], s[2:3], 25
+; SI-NEXT:    v_mov_b32_e32 v0, s0
+; SI-NEXT:    v_mov_b32_e32 v1, s8
 ; SI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
 ; SI-NEXT:    s_endpgm
 ;
 ; VI-LABEL: fshl_v2i32_imm:
 ; VI:       ; %bb.0: ; %entry
-; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
-; VI-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x34
+; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x2c
+; VI-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x24
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
-; VI-NEXT:    v_mov_b32_e32 v0, s0
-; VI-NEXT:    v_mov_b32_e32 v1, s1
-; VI-NEXT:    s_mov_b32 s0, s5
-; VI-NEXT:    s_mov_b32 s1, s3
-; VI-NEXT:    s_mov_b32 s5, s2
-; VI-NEXT:    s_lshr_b64 s[0:1], s[0:1], 23
-; VI-NEXT:    s_lshr_b64 s[2:3], s[4:5], 25
-; VI-NEXT:    v_mov_b32_e32 v2, s2
-; VI-NEXT:    v_mov_b32_e32 v3, s0
-; VI-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
+; VI-NEXT:    s_mov_b32 s6, s3
+; VI-NEXT:    s_mov_b32 s7, s1
+; VI-NEXT:    s_mov_b32 s3, s0
+; VI-NEXT:    s_lshr_b64 s[0:1], s[6:7], 23
+; VI-NEXT:    s_lshr_b64 s[2:3], s[2:3], 25
+; VI-NEXT:    v_mov_b32_e32 v2, s4
+; VI-NEXT:    v_mov_b32_e32 v0, s2
+; VI-NEXT:    v_mov_b32_e32 v1, s0
+; VI-NEXT:    v_mov_b32_e32 v3, s5
+; VI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
 ; VI-NEXT:    s_endpgm
 ;
 ; GFX9-LABEL: fshl_v2i32_imm:
 ; GFX9:       ; %bb.0: ; %entry
-; GFX9-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
-; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x2c
+; GFX9-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x24
 ; GFX9-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    s_mov_b32 s4, s7
-; GFX9-NEXT:    s_mov_b32 s5, s3
-; GFX9-NEXT:    s_mov_b32 s7, s2
-; GFX9-NEXT:    s_lshr_b64 s[2:3], s[4:5], 23
-; GFX9-NEXT:    s_lshr_b64 s[4:5], s[6:7], 25
-; GFX9-NEXT:    v_mov_b32_e32 v0, s4
-; GFX9-NEXT:    v_mov_b32_e32 v1, s2
-; GFX9-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX9-NEXT:    s_mov_b32 s4, s3
+; GFX9-NEXT:    s_mov_b32 s5, s1
+; GFX9-NEXT:    s_mov_b32 s3, s0
+; GFX9-NEXT:    s_lshr_b64 s[0:1], s[4:5], 23
+; GFX9-NEXT:    s_lshr_b64 s[2:3], s[2:3], 25
+; GFX9-NEXT:    v_mov_b32_e32 v0, s2
+; GFX9-NEXT:    v_mov_b32_e32 v1, s0
+; GFX9-NEXT:    global_store_dwordx2 v2, v[0:1], s[6:7]
 ; GFX9-NEXT:    s_endpgm
 ;
 ; R600-LABEL: fshl_v2i32_imm:
@@ -429,35 +437,35 @@ define amdgpu_kernel void @fshl_v2i32_imm(ptr addrspace(1) %in, <2 x i32> %x, <2
 ; GFX10-LABEL: fshl_v2i32_imm:
 ; GFX10:       ; %bb.0: ; %entry
 ; GFX10-NEXT:    s_clause 0x1
-; GFX10-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
-; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x2c
+; GFX10-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x24
 ; GFX10-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-NEXT:    s_mov_b32 s4, s7
-; GFX10-NEXT:    s_mov_b32 s7, s2
-; GFX10-NEXT:    s_mov_b32 s5, s3
-; GFX10-NEXT:    s_lshr_b64 s[2:3], s[6:7], 25
-; GFX10-NEXT:    s_lshr_b64 s[4:5], s[4:5], 23
-; GFX10-NEXT:    v_mov_b32_e32 v0, s2
-; GFX10-NEXT:    v_mov_b32_e32 v1, s4
-; GFX10-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX10-NEXT:    s_mov_b32 s4, s3
+; GFX10-NEXT:    s_mov_b32 s3, s0
+; GFX10-NEXT:    s_mov_b32 s5, s1
+; GFX10-NEXT:    s_lshr_b64 s[0:1], s[2:3], 25
+; GFX10-NEXT:    s_lshr_b64 s[2:3], s[4:5], 23
+; GFX10-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-NEXT:    global_store_dwordx2 v2, v[0:1], s[6:7]
 ; GFX10-NEXT:    s_endpgm
 ;
 ; GFX11-LABEL: fshl_v2i32_imm:
 ; GFX11:       ; %bb.0: ; %entry
 ; GFX11-NEXT:    s_clause 0x1
-; GFX11-NEXT:    s_load_b64 s[6:7], s[4:5], 0x34
-; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x2c
+; GFX11-NEXT:    s_load_b64 s[4:5], s[4:5], 0x24
 ; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX11-NEXT:    s_mov_b32 s4, s7
-; GFX11-NEXT:    s_mov_b32 s7, s2
-; GFX11-NEXT:    s_mov_b32 s5, s3
-; GFX11-NEXT:    s_lshr_b64 s[2:3], s[6:7], 25
-; GFX11-NEXT:    s_lshr_b64 s[4:5], s[4:5], 23
+; GFX11-NEXT:    s_mov_b32 s6, s3
+; GFX11-NEXT:    s_mov_b32 s3, s0
+; GFX11-NEXT:    s_mov_b32 s7, s1
+; GFX11-NEXT:    s_lshr_b64 s[0:1], s[2:3], 25
+; GFX11-NEXT:    s_lshr_b64 s[2:3], s[6:7], 23
 ; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT:    v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s4
-; GFX11-NEXT:    v_mov_b32_e32 v0, s2
-; GFX11-NEXT:    global_store_b64 v2, v[0:1], s[0:1]
+; GFX11-NEXT:    v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s2
+; GFX11-NEXT:    v_mov_b32_e32 v0, s0
+; GFX11-NEXT:    global_store_b64 v2, v[0:1], s[4:5]
 ; GFX11-NEXT:    s_endpgm
 entry:
   %0 = call <2 x i32> @llvm.fshl.v2i32(<2 x i32> %x, <2 x i32> %y, <2 x i32> <i32 7, i32 9>)
diff --git a/llvm/test/CodeGen/AMDGPU/fshr.ll b/llvm/test/CodeGen/AMDGPU/fshr.ll
index d3ceaba111848..7afb2cf317869 100644
--- a/llvm/test/CodeGen/AMDGPU/fshr.ll
+++ b/llvm/test/CodeGen/AMDGPU/fshr.ll
@@ -325,56 +325,60 @@ entry:
 define amdgpu_kernel void @fshr_v2i32(ptr addrspace(1) %in, <2 x i32> %x, <2 x i32> %y, <2 x i32> %z) {
 ; SI-LABEL: fshr_v2i32:
 ; SI:       ; %bb.0: ; %entry
-; SI-NEXT:    s_load_dwordx8 s[0:7], s[4:5], 0x9
+; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0xb
+; SI-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x9
+; SI-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xf
 ; SI-NEXT:    s_mov_b32 s11, 0xf000
 ; SI-NEXT:    s_mov_b32 s10, -1
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
-; SI-NEXT:    s_mov_b32 s8, s0
-; SI-NEXT:    s_mov_b32 s9, s1
-; SI-NEXT:    s_mov_b32 s0, s5
-; SI-NEXT:    s_mov_b32 s1, s3
-; SI-NEXT:    s_and_b32 s3, s7, 31
-; SI-NEXT:    s_lshr_b64 s[0:1], s[0:1], s3
-; SI-NEXT:    s_mov_b32 s5, s2
-; SI-NEXT:    s_and_b32 s1, s6, 31
-; SI-NEXT:    s_lshr_b64 s[2:3], s[4:5], s1
-; SI-NEXT:    v_mov_b32_e32 v0, s2
-; SI-NEXT:    v_mov_b32_e32 v1, s0
+; SI-NEXT:    s_mov_b32 s6, s3
+; SI-NEXT:    s_mov_b32 s7, s1
+; SI-NEXT:    s_and_b32 s1, s5, 31
+; SI-NEXT:    s_mov_b32 s3, s0
+; SI-NEXT:    s_and_b32 s0, s4, 31
+; SI-NEXT:    s_lshr_b64 s[6:7], s[6:7], s1
+; SI-NEXT:    s_lshr_b64 s[0:1], s[2:3], s0
+; SI-NEXT:    v_mov_b32_e32 v0, s0
+; SI-NEXT:    v_mov_b32_e32 v1, s6
 ; SI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[8:11], 0
 ; SI-NEXT:    s_endpgm
 ;
 ; VI-LABEL: fshr_v2i32:
 ; VI:       ; %bb.0: ; %entry
-; VI-NEXT:    s_load_dwordx8 s[0:7], s[4:5], 0x24
+; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x2c
+; VI-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x3c
+; VI-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x24
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    s_mov_b32 s8, s3
+; VI-NEXT:    s_mov_b32 s9, s1
+; VI-NEXT:    s_and_b32 s1, s7, 31
+; VI-NEXT:    s_mov_b32 s3, s0
+; VI-NEXT:    s_and_b32 s0, s6, 31
+; VI-NEXT:    s_lshr_b64 s[8:9], s[8:9], s1
+; VI-NEXT:    s_lshr_b64 s[0:1], s[2:3], s0
+; VI-NEXT:    v_mov_b32_e32 v2, s4
 ; VI-NEXT:    v_mov_b32_e32 v0, s0
-; VI-NEXT:    v_mov_b32_e32 v1, s1
-; VI-NEXT:    s_mov_b32 s0, s5
-; VI-NEXT:    s_mov_b32 s1, s3
-; VI-NEXT:    s_and_b32 s3, s7, 31
-; VI-NEXT:    s_lshr_b64 s[0:1], s[0:1], s3
-; VI-NEXT:    s_mov_b32 s5, s2
-; VI-NEXT:    s_and_b32 s1, s6, 31
-; VI-NEXT:    s_lshr_b64 s[2:3], s[4:5], s1
-; VI-NEXT:    v_mov_b32_e32 v2, s2
-; VI-NEXT:    v_mov_b32_e32 v3, s0
-; VI-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
+; VI-NEXT:    v_mov_b32_e32 v1, s8
+; VI-NEXT:    v_mov_b32_e32 v3, s5
+; VI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
 ; VI-NEXT:    s_endpgm
 ;
 ; GFX9-LABEL: fshr_v2i32:
 ; GFX9:       ; %bb.0: ; %entry
-; GFX9-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x24
+; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x2c
+; GFX9-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x3c
+; GFX9-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x24
 ; GFX9-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    s_mov_b32 s0, s13
-; GFX9-NEXT:    s_mov_b32 s1, s11
-; GFX9-NEXT:    s_and_b32 s2, s15, 31
-; GFX9-NEXT:    s_lshr_b64 s[0:1], s[0:1], s2
-; GFX9-NEXT:    s_mov_b32 s13, s10
-; GFX9-NEXT:    s_and_b32 s1, s14, 31
-; GFX9-NEXT:    s_lshr_b64 s[2:3], s[12:13], s1
-; GFX9-NEXT:    v_mov_b32_e32 v0, s2
-; GFX9-NEXT:    v_mov_b32_e32 v1, s0
+; GFX9-NEXT:    s_mov_b32 s4, s3
+; GFX9-NEXT:    s_mov_b32 s5, s1
+; GFX9-NEXT:    s_and_b32 s1, s7, 31
+; GFX9-NEXT:    s_mov_b32 s3, s0
+; GFX9-NEXT:    s_and_b32 s0, s6, 31
+; GFX9-NEXT:    s_lshr_b64 s[4:5], s[4:5], s1
+; GFX9-NEXT:    s_lshr_b64 s[0:1], s[2:3], s0
+; GFX9-NEXT:    v_mov_b32_e32 v0, s0
+; GFX9-NEXT:    v_mov_b32_e32 v1, s4
 ; GFX9-NEXT:    global_store_dwordx2 v2, v[0:1], s[8:9]
 ; GFX9-NEXT:    s_endpgm
 ;
@@ -394,53 +398,62 @@ define amdgpu_kernel void @fshr_v2i32(ptr addrspace(1) %in, <2 x i32> %x, <2 x i
 ;
 ; GFX10-LABEL: fshr_v2i32:
 ; GFX10:       ; %bb.0: ; %entry
-; GFX10-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x24
+; GFX10-NEXT:    s_clause 0x2
+; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x2c
+; GFX10-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x3c
+; GFX10-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x24
 ; GFX10-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-NEXT:    s_mov_b32 s0, s13
-; GFX10-NEXT:    s_mov_b32 s1, s11
-; GFX10-NEXT:    s_mov_b32 s13, s10
-; GFX10-NEXT:    s_and_b32 s2, s14, 31
-; GFX10-NEXT:    s_and_b32 s4, s15, 31
-; GFX10-NEXT:    s_lshr_b64 s[2:3], s[12:13], s2
-; GFX10-NEXT:    s_lshr_b64 s[0:1], s[0:1], s4
-; GFX10-NEXT:    v_mov_b32_e32 v0, s2
-; GFX10-NEXT:    v_mov_b32_e32 v1, s0
+; GFX10-NEXT:    s_mov_b32 s4, s3
+; GFX10-NEXT:    s_mov_b32 s5, s1
+; GFX10-NEXT:    s_mov_b32 s3, s0
+; GFX10-NEXT:    s_and_b32 s0, s6, 31
+; GFX10-NEXT:    s_and_b32 s6, s7, 31
+; GFX10-NEXT:    s_lshr_b64 s[0:1], s[2:3], s0
+; GFX10-NEXT:    s_lshr_b64 s[2:3], s[4:5], s6
+; GFX10-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-NEXT:    v_mov_b32_e32 v1, s2
 ; GFX10-NEXT:    global_store_dwordx2 v2, v[0:1], s[8:9]
 ; GFX10-NEXT:    s_endpgm
 ;
 ; GFX11-LABEL: fshr_v2i32:
 ; GFX11:       ; %bb.0: ; %entry
-; GFX11-NEXT:    s_load_b256 s[0:7], s[4:5], 0x24
+; GFX11-NEXT:    s_clause 0x2
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x2c
+; GFX11-NEXT:    s_load_b64 s[6:7], s[4:5], 0x3c
+; GFX11-NEXT:    s_load_b64 s[4:5], s[4:5], 0x24
 ; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX11-NEXT:    s_mov_b32 s8, s5
-; GFX11-NEXT:    s_mov_b32 s9, s3
-; GFX11-NEXT:    s_mov_b32 s5, s2
-; GFX11-NEXT:    s_and_b32 s2, s6, 31
+; GFX11-NEXT:    s_mov_b32 s8, s3
+; GFX11-NEXT:    s_mov_b32 s9, s1
+; GFX11-NEXT:    s_mov_b32 s3, s0
+; GFX11-NEXT:    s_and_b32 s0, s6, 31
 ; GFX11-NEXT:    s_and_b32 s6, s7, 31
-; GFX11-NEXT:    s_lshr_b64 s[2:3], s[4:5], s2
-; GFX11-NEXT:    s_lshr_b64 s[4:5], s[8:9], s6
+; GFX11-NEXT:    s_lshr_b64 s[0:1], s[2:3], s0
+; GFX11-NEXT:    s_lshr_b64 s[2:3], s[8:9], s6
 ; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT:    v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s4
-; GFX11-NEXT:    v_mov_b32_e32 v0, s2
-; GFX11-NEXT:    global_store_b64 v2, v[0:1], s[0:1]
+; GFX11-NEXT:    v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s2
+; GFX11-NEXT:    v_mov_b32_e32 v0, s0
+; GFX11-NEXT:    global_store_b64 v2, v[0:1], s[4:5]
 ; GFX11-NEXT:    s_endpgm
 ;
 ; GFX12-LABEL: fshr_v2i32:
 ; GFX12:       ; %bb.0: ; %entry
-; GFX12-NEXT:    s_load_b256 s[0:7], s[4:5], 0x24
+; GFX12-NEXT:    s_clause 0x2
+; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x2c
+; GFX12-NEXT:    s_load_b64 s[6:7], s[4:5], 0x3c
+; GFX12-NEXT:    s_load_b64 s[4:5], s[4:5], 0x24
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
-; GFX12-NEXT:    s_mov_b32 s8, s5
-; GFX12-NEXT:    s_mov_b32 s9, s3
-; GFX12-NEXT:    s_mov_b32 s5, s2
-; GFX12-NEXT:    s_and_b32 s2, s6, 31
+; GFX12-NEXT:    s_mov_b32 s8, s3
+; GFX12-NEXT:    s_mov_b32 s9, s1
+; GFX12-NEXT:    s_mov_b32 s3, s0
+; GFX12-NEXT:    s_and_b32 s0, s6, 31
 ; GFX12-NEXT:    s_and_b32 s6, s7, 31
-; GFX12-NEXT:    s_lshr_b64 s[2:3], s[4:5], s2
-; GFX12-NEXT:    s_lshr_b64 s[4:5], s[8:9], s6
+; GFX12-NEXT:    s_lshr_b64 s[0:1], s[2:3], s0
+; GFX12-NEXT:    s_lshr_b64 s[2:3], s[8:9], s6
 ; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-NEXT:    v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s4
-; GFX12-NEXT:    v_mov_b32_e32 v0, s2
-; GFX12-NEXT:    global_store_b64 v2, v[0:1], s[0:1]
+; GFX12-NEXT:    v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-NEXT:    v_mov_b32_e32 v0, s0
+; GFX12-NEXT:    global_store_b64 v2, v[0:1], s[4:5]
 ; GFX12-NEXT:    s_endpgm
 entry:
   %0 = call <2 x i32> @llvm.fshr.v2i32(<2 x i32> %x, <2 x i32> %y, <2 x i32> %z)
@@ -451,54 +464,52 @@ entry:
 define amdgpu_kernel void @fshr_v2i32_imm(ptr addrspace(1) %in, <2 x i32> %x, <2 x i32> %y) {
 ; SI-LABEL: fshr_v2i32_imm:
 ; SI:       ; %bb.0: ; %entry
-; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
-; SI-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0xd
+; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0xb
+; SI-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x9
 ; SI-NEXT:    s_mov_b32 s7, 0xf000
 ; SI-NEXT:    s_mov_b32 s6, -1
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
-; SI-NEXT:    s_mov_b32 s4, s0
-; SI-NEXT:    s_mov_b32 s5, s1
-; SI-NEXT:    s_mov_b32 s0, s9
-; SI-NEXT:    s_mov_b32 s1, s3
-; SI-NEXT:    s_mov_b32 s9, s2
-; SI-NEXT:    s_lshr_b64 s[0:1], s[0:1], 9
-; SI-NEXT:    s_lshr_b64 s[2:3], s[8:9], 7
-; SI-NEXT:    v_mov_b32_e32 v0, s2
-; SI-NEXT:    v_mov_b32_e32 v1, s0
+; SI-NEXT:    s_mov_b32 s8, s3
+; SI-NEXT:    s_mov_b32 s9, s1
+; SI-NEXT:    s_mov_b32 s3, s0
+; SI-NEXT:    s_lshr_b64 s[8:9], s[8:9], 9
+; SI-NEXT:    s_lshr_b64 s[0:1], s[2:3], 7
+; SI-NEXT:    v_mov_b32_e32 v0, s0
+; SI-NEXT:    v_mov_b32_e32 v1, s8
 ; SI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
 ; SI-NEXT:    s_endpgm
 ;
 ; VI-LABEL: fshr_v2i32_imm:
 ; VI:       ; %bb.0: ; %entry
-; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
-; VI-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x34
+; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x2c
+; VI-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x24
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
-; VI-NEXT:    v_mov_b32_e32 v0, s0
-; VI-NEXT:    v_mov_b32_e32 v1, s1
-; VI-NEXT:    s_mov_b32 s0, s5
-; VI-NEXT:    s_mov_b32 s1, s3
-; VI-NEXT:    s_mov_b32 s5, s2
-; VI-NEXT:    s_lshr_b64 s[0:1], s[0:1], 9
-; VI-NEXT:    s_lshr_b64 s[2:3], s[4:5], 7
-; VI-NEXT:    v_mov_b32_e32 v2, s2
-; VI-NEXT:    v_mov_b32_e32 v3, s0
-; VI-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
+; VI-NEXT:    s_mov_b32 s6, s3
+; VI-NEXT:    s_mov_b32 s7, s1
+; VI-NEXT:    s_mov_b32 s3, s0
+; VI-NEXT:    s_lshr_b64 s[0:1], s[6:7], 9
+; VI-NEXT:    s_lshr_b64 s[2:3], s[2:3], 7
+; VI-NEXT:    v_mov_b32_e32 v2, s4
+; VI-NEXT:    v_mov_b32_e32 v0, s2
+; VI-NEXT:    v_mov_b32_e32 v1, s0
+; VI-NEXT:    v_mov_b32_e32 v3, s5
+; VI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
 ; VI-NEXT:    s_endpgm
 ;
 ; GFX9-LABEL: fshr_v2i32_imm:
 ; GFX9:       ; %bb.0: ; %entry
-; GFX9-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
-; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x2c
+; GFX9-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x24
 ; GFX9-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    s_mov_b32 s4, s7
-; GFX9-NEXT:    s_mov_b32 s5, s3
-; GFX9-NEXT:    s_mov_b32 s7, s2
-; GFX9-NEXT:    s_lshr_b64 s[2:3], s[4:5], 9
-; GFX9-NEXT:    s_lshr_b64 s[4:5], s[6:7], 7
-; GFX9-NEXT:    v_mov_b32_e32 v0, s4
-; GFX9-NEXT:    v_mov_b32_e32 v1, s2
-; GFX9-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX9-NEXT:    s_mov_b32 s4, s3
+; GFX9-NEXT:    s_mov_b32 s5, s1
+; GFX9-NEXT:    s_mov_b32 s3, s0
+; GFX9-NEXT:    s_lshr_b64 s[0:1], s[4:5], 9
+; GFX9-NEXT:    s_lshr_b64 s[2:3], s[2:3], 7
+; GFX9-NEXT:    v_mov_b32_e32 v0, s2
+; GFX9-NEXT:    v_mov_b32_e32 v1, s0
+; GFX9-NEXT:    global_store_dwordx2 v2, v[0:1], s[6:7]
 ; GFX9-NEXT:    s_endpgm
 ;
 ; R600-LABEL: fshr_v2i32_imm:
@@ -518,52 +529,52 @@ define amdgpu_kernel void @fshr_v2i32_imm(ptr addrspace(1) %in, <2 x i32> %x, <2
 ; GFX10-LABEL: fshr_v2i32_imm:
 ; GFX10:       ; %bb.0: ; %entry
 ; GFX10-NEXT:    s_clause 0x1
-; GFX10-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
-; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x2c
+; GFX10-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x24
 ; GFX10-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-NEXT:    s_mov_b32 s4, s7
-; GFX10-NEXT:    s_mov_b32 s7, s2
-; GFX10-NEXT:    s_mov_b32 s5, s3
-; GFX10-NEXT:    s_lshr_b64 s[2:3], s[6:7], 7
-; GFX10-NEXT:    s_lshr_b64 s[4:5], s[4:5], 9
-; GFX10-NEXT:    v_mov_b32_e32 v0, s2
-; GFX10-NEXT:    v_mov_b32_e32 v1, s4
-; GFX10-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX10-NEXT:    s_mov_b32 s4, s3
+; GFX10-NEXT:    s_mov_b32 s3, s0
+; GFX10-NEXT:    s_mov_b32 s5, s1
+; GFX10-NEXT:    s_lshr_b64 s[0:1], s[2:3], 7
+; GFX10-NEXT:    s_lshr_b64 s[2:3], s[4:5], 9
+; GFX10-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-NEXT:    global_store_dwordx2 v2, v[0:1], s[6:7]
 ; GFX10-NEXT:    s_endpgm
 ;
 ; GFX11-LABEL: fshr_v2i32_imm:
 ; GFX11:       ; %bb.0: ; %entry
 ; GFX11-NEXT:    s_clause 0x1
-; GFX11-NEXT:    s_load_b64 s[6:7], s[4:5], 0x34
-; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x2c
+; GFX11-NEXT:    s_load_b64 s[4:5], s[4:5], 0x24
 ; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX11-NEXT:    s_mov_b32 s4, s7
-; GFX11-NEXT:    s_mov_b32 s7, s2
-; GFX11-NEXT:    s_mov_b32 s5, s3
-; GFX11-NEXT:    s_lshr_b64 s[2:3], s[6:7], 7
-; GFX11-NEXT:    s_lshr_b64 s[4:5], s[4:5], 9
+; GFX11-NEXT:    s_mov_b32 s6, s3
+; GFX11-NEXT:    s_mov_b32 s3, s0
+; GFX11-NEXT:    s_mov_b32 s7, s1
+; GFX11-NEXT:    s_lshr_b64 s[0:1], s[2:3], 7
+; GFX11-NEXT:    s_lshr_b64 s[2:3], s[6:7], 9
 ; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT:    v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s4
-; GFX11-NEXT:    v_mov_b32_e32 v0, s2
-; GFX11-NEXT:    global_store_b64 v2, v[0:1], s[0:1]
+; GFX11-NEXT:    v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s2
+; GFX11-NEXT:    v_mov_b32_e32 v0, s0
+; GFX11-NEXT:    global_store_b64 v2, v[0:1], s[4:5]
 ; GFX11-NEXT:    s_endpgm
 ;
 ; GFX12-LABEL: fshr_v2i32_imm:
 ; GFX12:       ; %bb.0: ; %entry
 ; GFX12-NEXT:    s_clause 0x1
-; GFX12-NEXT:    s_load_b64 s[6:7], s[4:5], 0x34
-; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x2c
+; GFX12-NEXT:    s_load_b64 s[4:5], s[4:5], 0x24
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
-; GFX12-NEXT:    s_mov_b32 s4, s7
-; GFX12-NEXT:    s_mov_b32 s7, s2
-; GFX12-NEXT:    s_mov_b32 s5, s3
-; GFX12-NEXT:    s_lshr_b64 s[2:3], s[6:7], 7
-; GFX12-NEXT:    s_lshr_b64 s[4:5], s[4:5], 9
+; GFX12-NEXT:    s_mov_b32 s6, s3
+; GFX12-NEXT:    s_mov_b32 s3, s0
+; GFX12-NEXT:    s_mov_b32 s7, s1
+; GFX12-NEXT:    s_lshr_b64 s[0:1], s[2:3], 7
+; GFX12-NEXT:    s_lshr_b64 s[2:3], s[6:7], 9
 ; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-NEXT:    v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s4
-; GFX12-NEXT:    v_mov_b32_e32 v0, s2
-; GFX12-NEXT:    global_store_b64 v2, v[0:1], s[0:1]
+; GFX12-NEXT:    v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-NEXT:    v_mov_b32_e32 v0, s0
+; GFX12-NEXT:    global_store_b64 v2, v[0:1], s[4:5]
 ; GFX12-NEXT:    s_endpgm
 entry:
   %0 = call <2 x i32> @llvm.fshr.v2i32(<2 x i32> %x, <2 x i32> %y, <2 x i32> <i32 7, i32 9>)
@@ -574,63 +585,61 @@ entry:
 define amdgpu_kernel void @fshr_v2i32_imm_src1(ptr addrspace(1) %in, <2 x i32> %x, <2 x i32> %y) {
 ; SI-LABEL: fshr_v2i32_imm_src1:
 ; SI:       ; %bb.0: ; %entry
-; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
-; SI-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0xd
+; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0xb
+; SI-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x9
+; SI-NEXT:    s_mov_b32 s8, 9
 ; SI-NEXT:    s_mov_b32 s10, 7
 ; SI-NEXT:    s_mov_b32 s7, 0xf000
-; SI-NEXT:    s_mov_b32 s6, -1
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
-; SI-NEXT:    s_mov_b32 s4, s0
-; SI-NEXT:    s_mov_b32 s5, s1
-; SI-NEXT:    s_mov_b32 s0, 9
-; SI-NEXT:    s_mov_b32 s1, s3
-; SI-NEXT:    s_and_b32 s3, s9, 31
-; SI-NEXT:    s_lshr_b64 s[0:1], s[0:1], s3
-; SI-NEXT:    s_mov_b32 s11, s2
-; SI-NEXT:    s_and_b32 s1, s8, 31
-; SI-NEXT:    s_lshr_b64 s[2:3], s[10:11], s1
-; SI-NEXT:    v_mov_b32_e32 v0, s2
-; SI-NEXT:    v_mov_b32_e32 v1, s0
+; SI-NEXT:    s_mov_b32 s9, s1
+; SI-NEXT:    s_and_b32 s1, s3, 31
+; SI-NEXT:    s_mov_b32 s11, s0
+; SI-NEXT:    s_and_b32 s0, s2, 31
+; SI-NEXT:    s_lshr_b64 s[8:9], s[8:9], s1
+; SI-NEXT:    s_lshr_b64 s[0:1], s[10:11], s0
+; SI-NEXT:    s_mov_b32 s6, -1
+; SI-NEXT:    v_mov_b32_e32 v0, s0
+; SI-NEXT:    v_mov_b32_e32 v1, s8
 ; SI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
 ; SI-NEXT:    s_endpgm
 ;
 ; VI-LABEL: fshr_v2i32_imm_src1:
 ; VI:       ; %bb.0: ; %entry
-; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
-; VI-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x34
+; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x2c
+; VI-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x24
 ; VI-NEXT:    s_mov_b32 s6, 9
+; VI-NEXT:    s_mov_b32 s8, 7
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    s_mov_b32 s7, s1
+; VI-NEXT:    s_and_b32 s1, s3, 31
+; VI-NEXT:    s_mov_b32 s9, s0
+; VI-NEXT:    s_and_b32 s0, s2, 31
+; VI-NEXT:    s_lshr_b64 s[6:7], s[6:7], s1
+; VI-NEXT:    s_lshr_b64 s[0:1], s[8:9], s0
+; VI-NEXT:    v_mov_b32_e32 v2, s4
 ; VI-NEXT:    v_mov_b32_e32 v0, s0
-; VI-NEXT:    s_mov_b32 s7, s3
-; VI-NEXT:    s_and_b32 s0, s5, 31
-; VI-NEXT:    v_mov_b32_e32 v1, s1
-; VI-NEXT:    s_lshr_b64 s[0:1], s[6:7], s0
-; VI-NEXT:    s_mov_b32 s6, 7
-; VI-NEXT:    s_mov_b32 s7, s2
-; VI-NEXT:    s_and_b32 s1, s4, 31
-; VI-NEXT:    s_lshr_b64 s[2:3], s[6:7], s1
-; VI-NEXT:    v_mov_b32_e32 v2, s2
-; VI-NEXT:    v_mov_b32_e32 v3, s0
-; VI-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
+; VI-NEXT:    v_mov_b32_e32 v1, s6
+; VI-NEXT:    v_mov_b32_e32 v3, s5
+; VI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
 ; VI-NEXT:    s_endpgm
 ;
 ; GFX9-LABEL: fshr_v2i32_imm_src1:
 ; GFX9:       ; %bb.0: ; %entry
-; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX9-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x2c
+; GFX9-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x24
 ; GFX9-NEXT:    s_mov_b32 s4, 9
 ; GFX9-NEXT:    s_mov_b32 s8, 7
 ; GFX9-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    s_mov_b32 s5, s3
-; GFX9-NEXT:    s_and_b32 s3, s7, 31
-; GFX9-NEXT:    s_mov_b32 s9, s2
-; GFX9-NEXT:    s_and_b32 s2, s6, 31
-; GFX9-NEXT:    s_lshr_b64 s[4:5], s[4:5], s3
-; GFX9-NEXT:    s_lshr_b64 s[2:3], s[8:9], s2
-; GFX9-NEXT:    v_mov_b32_e32 v0, s2
+; GFX9-NEXT:    s_mov_b32 s5, s1
+; GFX9-NEXT:    s_and_b32 s1, s3, 31
+; GFX9-NEXT:    s_mov_b32 s9, s0
+; GFX9-NEXT:    s_and_b32 s0, s2, 31
+; GFX9-NEXT:    s_lshr_b64 s[4:5], s[4:5], s1
+; GFX9-NEXT:    s_lshr_b64 s[0:1], s[8:9], s0
+; GFX9-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX9-NEXT:    v_mov_b32_e32 v1, s4
-; GFX9-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX9-NEXT:    global_store_dwordx2 v2, v[0:1], s[6:7]
 ; GFX9-NEXT:    s_endpgm
 ;
 ; R600-LABEL: fshr_v2i32_imm_src1:
@@ -650,61 +659,61 @@ define amdgpu_kernel void @fshr_v2i32_imm_src1(ptr addrspace(1) %in, <2 x i32> %
 ; GFX10-LABEL: fshr_v2i32_imm_src1:
 ; GFX10:       ; %bb.0: ; %entry
 ; GFX10-NEXT:    s_clause 0x1
-; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX10-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x2c
+; GFX10-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x24
 ; GFX10-NEXT:    s_mov_b32 s4, 9
 ; GFX10-NEXT:    s_mov_b32 s8, 7
 ; GFX10-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-NEXT:    s_mov_b32 s5, s3
-; GFX10-NEXT:    s_mov_b32 s9, s2
-; GFX10-NEXT:    s_and_b32 s2, s6, 31
-; GFX10-NEXT:    s_and_b32 s6, s7, 31
-; GFX10-NEXT:    s_lshr_b64 s[2:3], s[8:9], s2
-; GFX10-NEXT:    s_lshr_b64 s[4:5], s[4:5], s6
-; GFX10-NEXT:    v_mov_b32_e32 v0, s2
-; GFX10-NEXT:    v_mov_b32_e32 v1, s4
-; GFX10-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX10-NEXT:    s_mov_b32 s5, s1
+; GFX10-NEXT:    s_mov_b32 s9, s0
+; GFX10-NEXT:    s_and_b32 s0, s2, 31
+; GFX10-NEXT:    s_and_b32 s2, s3, 31
+; GFX10-NEXT:    s_lshr_b64 s[0:1], s[8:9], s0
+; GFX10-NEXT:    s_lshr_b64 s[2:3], s[4:5], s2
+; GFX10-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-NEXT:    global_store_dwordx2 v2, v[0:1], s[6:7]
 ; GFX10-NEXT:    s_endpgm
 ;
 ; GFX11-LABEL: fshr_v2i32_imm_src1:
 ; GFX11:       ; %bb.0: ; %entry
 ; GFX11-NEXT:    s_clause 0x1
-; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x2c
+; GFX11-NEXT:    s_load_b64 s[4:5], s[4:5], 0x24
 ; GFX11-NEXT:    s_mov_b32 s6, 9
 ; GFX11-NEXT:    s_mov_b32 s8, 7
 ; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX11-NEXT:    s_mov_b32 s7, s3
-; GFX11-NEXT:    s_mov_b32 s9, s2
-; GFX11-NEXT:    s_and_b32 s2, s4, 31
-; GFX11-NEXT:    s_and_b32 s4, s5, 31
-; GFX11-NEXT:    s_lshr_b64 s[2:3], s[8:9], s2
-; GFX11-NEXT:    s_lshr_b64 s[4:5], s[6:7], s4
+; GFX11-NEXT:    s_mov_b32 s7, s1
+; GFX11-NEXT:    s_mov_b32 s9, s0
+; GFX11-NEXT:    s_and_b32 s0, s2, 31
+; GFX11-NEXT:    s_and_b32 s2, s3, 31
+; GFX11-NEXT:    s_lshr_b64 s[0:1], s[8:9], s0
+; GFX11-NEXT:    s_lshr_b64 s[2:3], s[6:7], s2
 ; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT:    v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s4
-; GFX11-NEXT:    v_mov_b32_e32 v0, s2
-; GFX11-NEXT:    global_store_b64 v2, v[0:1], s[0:1]
+; GFX11-NEXT:    v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s2
+; GFX11-NEXT:    v_mov_b32_e32 v0, s0
+; GFX11-NEXT:    global_store_b64 v2, v[0:1], s[4:5]
 ; GFX11-NEXT:    s_endpgm
 ;
 ; GFX12-LABEL: fshr_v2i32_imm_src1:
 ; GFX12:       ; %bb.0: ; %entry
 ; GFX12-NEXT:    s_clause 0x1
-; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
-; GFX12-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
+; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x2c
+; GFX12-NEXT:    s_load_b64 s[4:5], s[4:5], 0x24
 ; GFX12-NEXT:    s_mov_b32 s6, 9
 ; GFX12-NEXT:    s_mov_b32 s8, 7
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
-; GFX12-NEXT:    s_mov_b32 s7, s3
-; GFX12-NEXT:    s_mov_b32 s9, s2
-; GFX12-NEXT:    s_and_b32 s2, s4, 31
-; GFX12-NEXT:    s_and_b32 s4, s5, 31
-; GFX12-NEXT:    s_lshr_b64 s[2:3], s[8:9], s2
-; GFX12-NEXT:    s_lshr_b64 s[4:5], s[6:7], s4
+; GFX12-NEXT:    s_mov_b32 s7, s1
+; GFX12-NEXT:    s_mov_b32 s9, s0
+; GFX12-NEXT:    s_and_b32 s0, s2, 31
+; GFX12-NEXT:    s_and_b32 s2, s3, 31
+; GFX12-NEXT:    s_lshr_b64 s[0:1], s[8:9], s0
+; GFX12-NEXT:    s_lshr_b64 s[2:3], s[6:7], s2
 ; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-NEXT:    v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s4
-; GFX12-NEXT:    v_mov_b32_e32 v0, s2
-; GFX12-NEXT:    global_store_b64 v2, v[0:1], s[0:1]
+; GFX12-NEXT:    v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-NEXT:    v_mov_b32_e32 v0, s0
+; GFX12-NEXT:    global_store_b64 v2, v[0:1], s[4:5]
 ; GFX12-NEXT:    s_endpgm
 entry:
   %0 = call <2 x i32> @llvm.fshr.v2i32(<2 x i32> %x, <2 x i32> <i32 7, i32 9>, <2 x i32> %y)
diff --git a/llvm/test/CodeGen/AMDGPU/global_atomics.ll b/llvm/test/CodeGen/AMDGPU/global_atomics.ll
index 580eeda73781e..da132d0269e6b 100644
--- a/llvm/test/CodeGen/AMDGPU/global_atomics.ll
+++ b/llvm/test/CodeGen/AMDGPU/global_atomics.ll
@@ -9552,47 +9552,6 @@ define amdgpu_kernel void @atomic_sub_i16_soffset__amdgpu_no_remote_memory(ptr a
 ; GFX9-NEXT:    s_cbranch_execnz .LBB136_1
 ; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX9-NEXT:    s_endpgm
-;
-; GFX11-LABEL: atomic_sub_i16_soffset__amdgpu_no_remote_memory:
-; GFX11:       ; %bb.0:
-; GFX11-NEXT:    s_clause 0x1
-; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
-; GFX11-NEXT:    s_load_b32 s2, s[4:5], 0x2c
-; GFX11-NEXT:    v_mov_b32_e32 v2, 0
-; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX11-NEXT:    s_add_u32 s3, s0, 0x4650
-; GFX11-NEXT:    s_addc_u32 s1, s1, 0
-; GFX11-NEXT:    s_and_b32 s0, s3, -4
-; GFX11-NEXT:    s_and_b32 s3, s3, 3
-; GFX11-NEXT:    s_load_b32 s4, s[0:1], 0x0
-; GFX11-NEXT:    s_lshl_b32 s5, s3, 3
-; GFX11-NEXT:    s_and_b32 s6, s2, 0xffff
-; GFX11-NEXT:    s_lshl_b32 s2, 0xffff, s5
-; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT:    s_not_b32 s3, s2
-; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX11-NEXT:    v_mov_b32_e32 v1, s4
-; GFX11-NEXT:    s_lshl_b32 s4, s6, s5
-; GFX11-NEXT:    s_mov_b32 s5, 0
-; GFX11-NEXT:  .LBB136_1: ; %atomicrmw.start
-; GFX11-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT:    v_subrev_nc_u32_e32 v0, s4, v1
-; GFX11-NEXT:    v_and_b32_e32 v0, s2, v0
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT:    v_and_or_b32 v0, v1, s3, v0
-; GFX11-NEXT:    global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] glc
-; GFX11-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-NEXT:    buffer_gl1_inv
-; GFX11-NEXT:    buffer_gl0_inv
-; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v0, v1
-; GFX11-NEXT:    v_mov_b32_e32 v1, v0
-; GFX11-NEXT:    s_or_b32 s5, vcc_lo, s5
-; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s5
-; GFX11-NEXT:    s_cbranch_execnz .LBB136_1
-; GFX11-NEXT:  ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT:    s_endpgm
   %gep = getelementptr i16, ptr addrspace(1) %out, i64 9000
   %val = atomicrmw sub ptr addrspace(1) %gep, i16 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !0
   ret void
@@ -9712,47 +9671,6 @@ define amdgpu_kernel void @atomic_sub_i8_soffset__amdgpu_no_remote_memory(ptr ad
 ; GFX9-NEXT:    s_cbranch_execnz .LBB137_1
 ; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX9-NEXT:    s_endpgm
-;
-; GFX11-LABEL: atomic_sub_i8_soffset__amdgpu_no_remote_memory:
-; GFX11:       ; %bb.0:
-; GFX11-NEXT:    s_clause 0x1
-; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
-; GFX11-NEXT:    s_load_b32 s2, s[4:5], 0x2c
-; GFX11-NEXT:    v_mov_b32_e32 v2, 0
-; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX11-NEXT:    s_add_u32 s3, s0, 0x2328
-; GFX11-NEXT:    s_addc_u32 s1, s1, 0
-; GFX11-NEXT:    s_and_b32 s0, s3, -4
-; GFX11-NEXT:    s_and_b32 s3, s3, 3
-; GFX11-NEXT:    s_load_b32 s4, s[0:1], 0x0
-; GFX11-NEXT:    s_lshl_b32 s5, s3, 3
-; GFX11-NEXT:    s_and_b32 s6, s2, 0xff
-; GFX11-NEXT:    s_lshl_b32 s2, 0xff, s5
-; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT:    s_not_b32 s3, s2
-; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX11-NEXT:    v_mov_b32_e32 v1, s4
-; GFX11-NEXT:    s_lshl_b32 s4, s6, s5
-; GFX11-NEXT:    s_mov_b32 s5, 0
-; GFX11-NEXT:  .LBB137_1: ; %atomicrmw.start
-; GFX11-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT:    v_subrev_nc_u32_e32 v0, s4, v1
-; GFX11-NEXT:    v_and_b32_e32 v0, s2, v0
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT:    v_and_or_b32 v0, v1, s3, v0
-; GFX11-NEXT:    global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] glc
-; GFX11-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-NEXT:    buffer_gl1_inv
-; GFX11-NEXT:    buffer_gl0_inv
-; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v0, v1
-; GFX11-NEXT:    v_mov_b32_e32 v1, v0
-; GFX11-NEXT:    s_or_b32 s5, vcc_lo, s5
-; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s5
-; GFX11-NEXT:    s_cbranch_execnz .LBB137_1
-; GFX11-NEXT:  ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT:    s_endpgm
   %gep = getelementptr i8, ptr addrspace(1) %out, i64 9000
   %val = atomicrmw sub ptr addrspace(1) %gep, i8 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !0
   ret void
diff --git a/llvm/test/CodeGen/AMDGPU/half.ll b/llvm/test/CodeGen/AMDGPU/half.ll
index 87e57298f5dc6..8e427a6ef2023 100644
--- a/llvm/test/CodeGen/AMDGPU/half.ll
+++ b/llvm/test/CodeGen/AMDGPU/half.ll
@@ -98,16 +98,16 @@ define amdgpu_kernel void @load_v3f16_arg(ptr addrspace(1) %out, <3 x half> %arg
 ; CIVI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CIVI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; CIVI-NEXT:    s_waitcnt lgkmcnt(0)
+; CIVI-NEXT:    s_add_u32 s4, s0, 4
+; CIVI-NEXT:    s_addc_u32 s5, s1, 0
+; CIVI-NEXT:    v_mov_b32_e32 v2, s4
+; CIVI-NEXT:    v_mov_b32_e32 v4, s3
 ; CIVI-NEXT:    v_mov_b32_e32 v0, s0
+; CIVI-NEXT:    v_mov_b32_e32 v3, s5
 ; CIVI-NEXT:    v_mov_b32_e32 v1, s1
-; CIVI-NEXT:    v_mov_b32_e32 v2, s2
-; CIVI-NEXT:    s_add_u32 s0, s0, 4
-; CIVI-NEXT:    flat_store_dword v[0:1], v2
-; CIVI-NEXT:    s_addc_u32 s1, s1, 0
-; CIVI-NEXT:    v_mov_b32_e32 v0, s0
-; CIVI-NEXT:    v_mov_b32_e32 v1, s1
-; CIVI-NEXT:    v_mov_b32_e32 v2, s3
-; CIVI-NEXT:    flat_store_short v[0:1], v2
+; CIVI-NEXT:    v_mov_b32_e32 v5, s2
+; CIVI-NEXT:    flat_store_short v[2:3], v4
+; CIVI-NEXT:    flat_store_dword v[0:1], v5
 ; CIVI-NEXT:    s_endpgm
 ;
 ; GFX11-LABEL: load_v3f16_arg:
@@ -135,8 +135,8 @@ define amdgpu_kernel void @load_v4f16_arg(ptr addrspace(1) %out, <4 x half> %arg
 ; CIVI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CIVI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CIVI-NEXT:    v_mov_b32_e32 v0, s0
-; CIVI-NEXT:    v_mov_b32_e32 v1, s1
 ; CIVI-NEXT:    v_mov_b32_e32 v2, s2
+; CIVI-NEXT:    v_mov_b32_e32 v1, s1
 ; CIVI-NEXT:    v_mov_b32_e32 v3, s3
 ; CIVI-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
 ; CIVI-NEXT:    s_endpgm
@@ -144,9 +144,9 @@ define amdgpu_kernel void @load_v4f16_arg(ptr addrspace(1) %out, <4 x half> %arg
 ; GFX11-LABEL: load_v4f16_arg:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x0
+; GFX11-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX11-NEXT:    v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s3
-; GFX11-NEXT:    v_mov_b32_e32 v0, s2
+; GFX11-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-NEXT:    global_store_b64 v2, v[0:1], s[0:1]
 ; GFX11-NEXT:    s_endpgm
   store <4 x half> %arg, ptr addrspace(1) %out
@@ -348,21 +348,37 @@ define amdgpu_kernel void @extload_v2f16_to_v2f32_arg(ptr addrspace(1) %out, <2
 }
 
 define amdgpu_kernel void @extload_v3f16_to_v3f32_arg(ptr addrspace(1) %out, <3 x half> %arg) #0 {
-; CIVI-LABEL: extload_v3f16_to_v3f32_arg:
-; CIVI:       ; %bb.0:
-; CIVI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
-; CIVI-NEXT:    s_add_i32 s12, s12, s17
-; CIVI-NEXT:    s_mov_b32 flat_scratch_lo, s13
-; CIVI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
-; CIVI-NEXT:    s_waitcnt lgkmcnt(0)
-; CIVI-NEXT:    s_lshr_b32 s4, s2, 16
-; CIVI-NEXT:    v_cvt_f32_f16_e32 v2, s3
-; CIVI-NEXT:    v_cvt_f32_f16_e32 v1, s4
-; CIVI-NEXT:    v_cvt_f32_f16_e32 v0, s2
-; CIVI-NEXT:    v_mov_b32_e32 v3, s0
-; CIVI-NEXT:    v_mov_b32_e32 v4, s1
-; CIVI-NEXT:    flat_store_dwordx3 v[3:4], v[0:2]
-; CIVI-NEXT:    s_endpgm
+; CI-LABEL: extload_v3f16_to_v3f32_arg:
+; CI:       ; %bb.0:
+; CI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; CI-NEXT:    s_waitcnt lgkmcnt(0)
+; CI-NEXT:    s_lshr_b32 s4, s2, 16
+; CI-NEXT:    v_cvt_f32_f16_e32 v2, s3
+; CI-NEXT:    v_cvt_f32_f16_e32 v1, s4
+; CI-NEXT:    v_cvt_f32_f16_e32 v0, s2
+; CI-NEXT:    v_mov_b32_e32 v4, s1
+; CI-NEXT:    v_mov_b32_e32 v3, s0
+; CI-NEXT:    flat_store_dwordx3 v[3:4], v[0:2]
+; CI-NEXT:    s_endpgm
+;
+; VI-LABEL: extload_v3f16_to_v3f32_arg:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    s_lshr_b32 s4, s2, 16
+; VI-NEXT:    v_cvt_f32_f16_e32 v0, s2
+; VI-NEXT:    v_cvt_f32_f16_e32 v1, s4
+; VI-NEXT:    v_cvt_f32_f16_e32 v2, s3
+; VI-NEXT:    v_mov_b32_e32 v4, s1
+; VI-NEXT:    v_mov_b32_e32 v3, s0
+; VI-NEXT:    flat_store_dwordx3 v[3:4], v[0:2]
+; VI-NEXT:    s_endpgm
 ;
 ; GFX11-LABEL: extload_v3f16_to_v3f32_arg:
 ; GFX11:       ; %bb.0:
@@ -370,9 +386,9 @@ define amdgpu_kernel void @extload_v3f16_to_v3f32_arg(ptr addrspace(1) %out, <3
 ; GFX11-NEXT:    v_mov_b32_e32 v3, 0
 ; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-NEXT:    s_lshr_b32 s4, s2, 16
-; GFX11-NEXT:    v_cvt_f32_f16_e32 v2, s3
-; GFX11-NEXT:    v_cvt_f32_f16_e32 v1, s4
 ; GFX11-NEXT:    v_cvt_f32_f16_e32 v0, s2
+; GFX11-NEXT:    v_cvt_f32_f16_e32 v1, s4
+; GFX11-NEXT:    v_cvt_f32_f16_e32 v2, s3
 ; GFX11-NEXT:    global_store_b96 v3, v[0:2], s[0:1]
 ; GFX11-NEXT:    s_endpgm
   %ext = fpext <3 x half> %arg to <3 x float>
@@ -388,14 +404,14 @@ define amdgpu_kernel void @extload_v4f16_to_v4f32_arg(ptr addrspace(1) %out, <4
 ; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
-; CI-NEXT:    s_lshr_b32 s4, s2, 16
-; CI-NEXT:    s_lshr_b32 s5, s3, 16
+; CI-NEXT:    s_lshr_b32 s4, s3, 16
+; CI-NEXT:    s_lshr_b32 s5, s2, 16
 ; CI-NEXT:    v_cvt_f32_f16_e32 v2, s3
-; CI-NEXT:    v_cvt_f32_f16_e32 v3, s5
-; CI-NEXT:    v_cvt_f32_f16_e32 v1, s4
+; CI-NEXT:    v_cvt_f32_f16_e32 v3, s4
+; CI-NEXT:    v_cvt_f32_f16_e32 v1, s5
 ; CI-NEXT:    v_cvt_f32_f16_e32 v0, s2
-; CI-NEXT:    v_mov_b32_e32 v4, s0
 ; CI-NEXT:    v_mov_b32_e32 v5, s1
+; CI-NEXT:    v_mov_b32_e32 v4, s0
 ; CI-NEXT:    flat_store_dwordx4 v[4:5], v[0:3]
 ; CI-NEXT:    s_endpgm
 ;
@@ -408,12 +424,12 @@ define amdgpu_kernel void @extload_v4f16_to_v4f32_arg(ptr addrspace(1) %out, <4
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    s_lshr_b32 s4, s3, 16
 ; VI-NEXT:    s_lshr_b32 s5, s2, 16
-; VI-NEXT:    v_cvt_f32_f16_e32 v2, s3
+; VI-NEXT:    v_cvt_f32_f16_e32 v0, s2
 ; VI-NEXT:    v_cvt_f32_f16_e32 v3, s4
 ; VI-NEXT:    v_cvt_f32_f16_e32 v1, s5
-; VI-NEXT:    v_cvt_f32_f16_e32 v0, s2
-; VI-NEXT:    v_mov_b32_e32 v4, s0
+; VI-NEXT:    v_cvt_f32_f16_e32 v2, s3
 ; VI-NEXT:    v_mov_b32_e32 v5, s1
+; VI-NEXT:    v_mov_b32_e32 v4, s0
 ; VI-NEXT:    flat_store_dwordx4 v[4:5], v[0:3]
 ; VI-NEXT:    s_endpgm
 ;
@@ -424,10 +440,10 @@ define amdgpu_kernel void @extload_v4f16_to_v4f32_arg(ptr addrspace(1) %out, <4
 ; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-NEXT:    s_lshr_b32 s4, s3, 16
 ; GFX11-NEXT:    s_lshr_b32 s5, s2, 16
-; GFX11-NEXT:    v_cvt_f32_f16_e32 v2, s3
+; GFX11-NEXT:    v_cvt_f32_f16_e32 v0, s2
 ; GFX11-NEXT:    v_cvt_f32_f16_e32 v3, s4
 ; GFX11-NEXT:    v_cvt_f32_f16_e32 v1, s5
-; GFX11-NEXT:    v_cvt_f32_f16_e32 v0, s2
+; GFX11-NEXT:    v_cvt_f32_f16_e32 v2, s3
 ; GFX11-NEXT:    global_store_b128 v4, v[0:3], s[0:1]
 ; GFX11-NEXT:    s_endpgm
   %ext = fpext <4 x half> %arg to <4 x float>
@@ -708,33 +724,61 @@ define amdgpu_kernel void @extload_v3f16_to_v3f64_arg(ptr addrspace(1) %out, <3
 }
 
 define amdgpu_kernel void @extload_v4f16_to_v4f64_arg(ptr addrspace(1) %out, <4 x half> %arg) #0 {
-; CIVI-LABEL: extload_v4f16_to_v4f64_arg:
-; CIVI:       ; %bb.0:
-; CIVI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
-; CIVI-NEXT:    s_add_i32 s12, s12, s17
-; CIVI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
-; CIVI-NEXT:    s_mov_b32 flat_scratch_lo, s13
-; CIVI-NEXT:    s_waitcnt lgkmcnt(0)
-; CIVI-NEXT:    s_lshr_b32 s5, s3, 16
-; CIVI-NEXT:    v_cvt_f32_f16_e32 v0, s3
-; CIVI-NEXT:    v_cvt_f32_f16_e32 v2, s5
-; CIVI-NEXT:    s_lshr_b32 s4, s2, 16
-; CIVI-NEXT:    v_cvt_f32_f16_e32 v4, s2
-; CIVI-NEXT:    v_cvt_f32_f16_e32 v6, s4
-; CIVI-NEXT:    s_add_u32 s2, s0, 16
-; CIVI-NEXT:    v_cvt_f64_f32_e32 v[0:1], v0
-; CIVI-NEXT:    v_cvt_f64_f32_e32 v[2:3], v2
-; CIVI-NEXT:    s_addc_u32 s3, s1, 0
-; CIVI-NEXT:    v_cvt_f64_f32_e32 v[4:5], v4
-; CIVI-NEXT:    v_cvt_f64_f32_e32 v[6:7], v6
-; CIVI-NEXT:    v_mov_b32_e32 v9, s3
-; CIVI-NEXT:    v_mov_b32_e32 v8, s2
-; CIVI-NEXT:    flat_store_dwordx4 v[8:9], v[0:3]
-; CIVI-NEXT:    s_nop 0
-; CIVI-NEXT:    v_mov_b32_e32 v0, s0
-; CIVI-NEXT:    v_mov_b32_e32 v1, s1
-; CIVI-NEXT:    flat_store_dwordx4 v[0:1], v[4:7]
-; CIVI-NEXT:    s_endpgm
+; CI-LABEL: extload_v4f16_to_v4f64_arg:
+; CI:       ; %bb.0:
+; CI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; CI-NEXT:    s_add_i32 s12, s12, s17
+; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; CI-NEXT:    s_waitcnt lgkmcnt(0)
+; CI-NEXT:    s_lshr_b32 s4, s3, 16
+; CI-NEXT:    v_cvt_f32_f16_e32 v0, s3
+; CI-NEXT:    v_cvt_f32_f16_e32 v2, s4
+; CI-NEXT:    s_lshr_b32 s5, s2, 16
+; CI-NEXT:    v_cvt_f32_f16_e32 v4, s2
+; CI-NEXT:    v_cvt_f32_f16_e32 v6, s5
+; CI-NEXT:    s_add_u32 s2, s0, 16
+; CI-NEXT:    v_cvt_f64_f32_e32 v[0:1], v0
+; CI-NEXT:    v_cvt_f64_f32_e32 v[2:3], v2
+; CI-NEXT:    s_addc_u32 s3, s1, 0
+; CI-NEXT:    v_cvt_f64_f32_e32 v[4:5], v4
+; CI-NEXT:    v_cvt_f64_f32_e32 v[6:7], v6
+; CI-NEXT:    v_mov_b32_e32 v9, s3
+; CI-NEXT:    v_mov_b32_e32 v8, s2
+; CI-NEXT:    flat_store_dwordx4 v[8:9], v[0:3]
+; CI-NEXT:    s_nop 0
+; CI-NEXT:    v_mov_b32_e32 v0, s0
+; CI-NEXT:    v_mov_b32_e32 v1, s1
+; CI-NEXT:    flat_store_dwordx4 v[0:1], v[4:7]
+; CI-NEXT:    s_endpgm
+;
+; VI-LABEL: extload_v4f16_to_v4f64_arg:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; VI-NEXT:    s_add_i32 s12, s12, s17
+; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
+; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    s_lshr_b32 s5, s3, 16
+; VI-NEXT:    v_cvt_f32_f16_e32 v0, s3
+; VI-NEXT:    v_cvt_f32_f16_e32 v2, s5
+; VI-NEXT:    s_lshr_b32 s4, s2, 16
+; VI-NEXT:    v_cvt_f32_f16_e32 v4, s2
+; VI-NEXT:    v_cvt_f32_f16_e32 v6, s4
+; VI-NEXT:    s_add_u32 s2, s0, 16
+; VI-NEXT:    v_cvt_f64_f32_e32 v[0:1], v0
+; VI-NEXT:    v_cvt_f64_f32_e32 v[2:3], v2
+; VI-NEXT:    s_addc_u32 s3, s1, 0
+; VI-NEXT:    v_cvt_f64_f32_e32 v[4:5], v4
+; VI-NEXT:    v_cvt_f64_f32_e32 v[6:7], v6
+; VI-NEXT:    v_mov_b32_e32 v9, s3
+; VI-NEXT:    v_mov_b32_e32 v8, s2
+; VI-NEXT:    flat_store_dwordx4 v[8:9], v[0:3]
+; VI-NEXT:    s_nop 0
+; VI-NEXT:    v_mov_b32_e32 v0, s0
+; VI-NEXT:    v_mov_b32_e32 v1, s1
+; VI-NEXT:    flat_store_dwordx4 v[0:1], v[4:7]
+; VI-NEXT:    s_endpgm
 ;
 ; GFX11-LABEL: extload_v4f16_to_v4f64_arg:
 ; GFX11:       ; %bb.0:
diff --git a/llvm/test/CodeGen/AMDGPU/hazard-gfx1250-flat-scr-hi.mir b/llvm/test/CodeGen/AMDGPU/hazard-gfx1250-flat-scr-hi.mir
new file mode 100644
index 0000000000000..e3b28c5518695
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/hazard-gfx1250-flat-scr-hi.mir
@@ -0,0 +1,158 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
+# RUN: llc -mtriple=amdgcn -mcpu=gfx1250 -run-pass si-fold-operands %s -o - | FileCheck -check-prefix=GCN %s
+
+---
+name:            s_ashr_i64
+tracksRegLiveness: true
+body:             |
+  bb.0:
+
+    ; GCN-LABEL: name: s_ashr_i64
+    ; GCN: [[COPY:%[0-9]+]]:sreg_32 = COPY $src_flat_scratch_base_hi
+    ; GCN-NEXT: [[S_ASHR_I64_:%[0-9]+]]:sreg_64 = S_ASHR_I64 undef %2:sreg_64, [[COPY]], implicit-def $scc
+    %0:sreg_32 = COPY $src_flat_scratch_base_hi
+    %2:sreg_64 = S_ASHR_I64 undef %1:sreg_64, %0, implicit-def $scc
+...
+
+---
+name:            s_lshl_b64
+tracksRegLiveness: true
+body:             |
+  bb.0:
+
+    ; GCN-LABEL: name: s_lshl_b64
+    ; GCN: [[COPY:%[0-9]+]]:sreg_32 = COPY $src_flat_scratch_base_hi
+    ; GCN-NEXT: [[S_LSHL_B64_:%[0-9]+]]:sreg_64 = S_LSHL_B64 undef %2:sreg_64, [[COPY]], implicit-def $scc
+    %0:sreg_32 = COPY $src_flat_scratch_base_hi
+    %2:sreg_64 = S_LSHL_B64 undef %1:sreg_64, %0, implicit-def $scc
+...
+
+---
+name:            s_lshr_b64
+tracksRegLiveness: true
+body:             |
+  bb.0:
+
+    ; GCN-LABEL: name: s_lshr_b64
+    ; GCN: [[COPY:%[0-9]+]]:sreg_32 = COPY $src_flat_scratch_base_hi
+    ; GCN-NEXT: [[S_LSHR_B64_:%[0-9]+]]:sreg_64 = S_LSHR_B64 undef %2:sreg_64, [[COPY]], implicit-def $scc
+    %0:sreg_32 = COPY $src_flat_scratch_base_hi
+    %2:sreg_64 = S_LSHR_B64 undef %1:sreg_64, %0, implicit-def $scc
+...
+
+---
+name:            s_bfe_i64
+tracksRegLiveness: true
+body:             |
+  bb.0:
+
+    ; GCN-LABEL: name: s_bfe_i64
+    ; GCN: [[COPY:%[0-9]+]]:sreg_32 = COPY $src_flat_scratch_base_hi
+    ; GCN-NEXT: [[S_BFE_I64_:%[0-9]+]]:sreg_64 = S_BFE_I64 undef %2:sreg_64, [[COPY]], implicit-def $scc
+    %0:sreg_32 = COPY $src_flat_scratch_base_hi
+    %2:sreg_64 = S_BFE_I64 undef %1:sreg_64, %0, implicit-def $scc
+...
+
+---
+name:            s_bfe_u64
+tracksRegLiveness: true
+body:             |
+  bb.0:
+
+    ; GCN-LABEL: name: s_bfe_u64
+    ; GCN: [[COPY:%[0-9]+]]:sreg_32 = COPY $src_flat_scratch_base_hi
+    ; GCN-NEXT: [[S_BFE_U64_:%[0-9]+]]:sreg_64 = S_BFE_U64 undef %2:sreg_64, [[COPY]], implicit-def $scc
+    %0:sreg_32 = COPY $src_flat_scratch_base_hi
+    %2:sreg_64 = S_BFE_U64 undef %1:sreg_64, %0, implicit-def $scc
+...
+
+---
+name:            s_bfm_b64
+tracksRegLiveness: true
+body:             |
+  bb.0:
+
+    ; GCN-LABEL: name: s_bfm_b64
+    ; GCN: [[COPY:%[0-9]+]]:sreg_32 = COPY $src_flat_scratch_base_hi
+    ; GCN-NEXT: [[S_BFM_B64_:%[0-9]+]]:sreg_64 = S_BFM_B64 [[COPY]], 1, implicit-def $scc
+    %0:sreg_32 = COPY $src_flat_scratch_base_hi
+    %1:sreg_64 = S_BFM_B64 %0, 1, implicit-def $scc
+...
+
+---
+name:            s_bitcmp0_b64
+tracksRegLiveness: true
+body:             |
+  bb.0:
+
+    ; GCN-LABEL: name: s_bitcmp0_b64
+    ; GCN: [[COPY:%[0-9]+]]:sreg_32 = COPY $src_flat_scratch_base_hi
+    ; GCN-NEXT: S_BITCMP0_B64 undef %1:sreg_64, [[COPY]], implicit undef $scc, implicit-def $scc
+    %0:sreg_32 = COPY $src_flat_scratch_base_hi
+    S_BITCMP0_B64 undef %1:sreg_64, %0, implicit undef $scc, implicit-def $scc
+...
+
+---
+name:            s_bitcmp1_b64
+tracksRegLiveness: true
+body:             |
+  bb.0:
+
+    ; GCN-LABEL: name: s_bitcmp1_b64
+    ; GCN: [[COPY:%[0-9]+]]:sreg_32 = COPY $src_flat_scratch_base_hi
+    ; GCN-NEXT: S_BITCMP1_B64 undef %1:sreg_64, [[COPY]], implicit undef $scc, implicit-def $scc
+    %0:sreg_32 = COPY $src_flat_scratch_base_hi
+    S_BITCMP1_B64 undef %1:sreg_64, %0, implicit undef $scc, implicit-def $scc
+...
+
+---
+name:            s_bitreplicate_b64_b32
+tracksRegLiveness: true
+body:             |
+  bb.0:
+
+    ; GCN-LABEL: name: s_bitreplicate_b64_b32
+    ; GCN: [[COPY:%[0-9]+]]:sreg_32 = COPY $src_flat_scratch_base_hi
+    ; GCN-NEXT: [[S_BITREPLICATE_B64_B32_:%[0-9]+]]:sreg_64 = S_BITREPLICATE_B64_B32 [[COPY]], implicit-def $scc
+    %0:sreg_32 = COPY $src_flat_scratch_base_hi
+    %2:sreg_64 = S_BITREPLICATE_B64_B32 %0, implicit-def $scc
+...
+
+---
+name:            s_bitset0_b64
+tracksRegLiveness: true
+body:             |
+  bb.0:
+
+    ; GCN-LABEL: name: s_bitset0_b64
+    ; GCN: [[COPY:%[0-9]+]]:sreg_32 = COPY $src_flat_scratch_base_hi
+    ; GCN-NEXT: [[S_BITSET0_B64_:%[0-9]+]]:sreg_64 = S_BITSET0_B64 [[COPY]], undef [[S_BITSET0_B64_]], implicit-def $scc
+    %0:sreg_32 = COPY $src_flat_scratch_base_hi
+    %1:sreg_64 = S_BITSET0_B64 %0, undef %1:sreg_64, implicit-def $scc
+...
+
+---
+name:            s_bitset1_b64
+tracksRegLiveness: true
+body:             |
+  bb.0:
+
+    ; GCN-LABEL: name: s_bitset1_b64
+    ; GCN: [[COPY:%[0-9]+]]:sreg_32 = COPY $src_flat_scratch_base_hi
+    ; GCN-NEXT: [[S_BITSET1_B64_:%[0-9]+]]:sreg_64 = S_BITSET1_B64 [[COPY]], undef [[S_BITSET1_B64_]], implicit-def $scc
+    %0:sreg_32 = COPY $src_flat_scratch_base_hi
+    %1:sreg_64 = S_BITSET1_B64 %0, undef %1:sreg_64, implicit-def $scc
+...
+
+---
+name:            s_ashr_i64_phys_dst
+tracksRegLiveness: true
+body:             |
+  bb.0:
+
+    ; GCN-LABEL: name: s_ashr_i64_phys_dst
+    ; GCN: [[COPY:%[0-9]+]]:sreg_32 = COPY $src_flat_scratch_base_hi
+    ; GCN-NEXT: $sgpr0_sgpr1 = S_ASHR_I64 undef %1:sreg_64, [[COPY]], implicit-def $scc
+    %0:sreg_32 = COPY $src_flat_scratch_base_hi
+    $sgpr0_sgpr1 = S_ASHR_I64 undef %1:sreg_64, %0, implicit-def $scc
+...
diff --git a/llvm/test/CodeGen/AMDGPU/insert_vector_dynelt.ll b/llvm/test/CodeGen/AMDGPU/insert_vector_dynelt.ll
index 11826aa0b360d..e1b4cad370f96 100644
--- a/llvm/test/CodeGen/AMDGPU/insert_vector_dynelt.ll
+++ b/llvm/test/CodeGen/AMDGPU/insert_vector_dynelt.ll
@@ -290,19 +290,19 @@ define amdgpu_kernel void @half4_inselt(ptr addrspace(1) %out, <4 x half> %vec,
 ; GCN-LABEL: half4_inselt:
 ; GCN:       ; %bb.0: ; %entry
 ; GCN-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GCN-NEXT:    s_load_dword s5, s[4:5], 0x34
+; GCN-NEXT:    s_load_dword s6, s[4:5], 0x34
 ; GCN-NEXT:    s_mov_b32 s4, 0x3c003c00
+; GCN-NEXT:    s_mov_b32 s5, s4
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_xor_b64 s[4:5], s[2:3], s[4:5]
+; GCN-NEXT:    s_lshl_b32 s6, s6, 4
+; GCN-NEXT:    s_lshl_b64 s[6:7], 0xffff, s6
+; GCN-NEXT:    s_and_b64 s[4:5], s[4:5], s[6:7]
+; GCN-NEXT:    s_xor_b64 s[2:3], s[4:5], s[2:3]
 ; GCN-NEXT:    v_mov_b32_e32 v0, s0
-; GCN-NEXT:    s_lshl_b32 s0, s5, 4
-; GCN-NEXT:    s_mov_b32 s5, s4
+; GCN-NEXT:    v_mov_b32_e32 v2, s2
 ; GCN-NEXT:    v_mov_b32_e32 v1, s1
-; GCN-NEXT:    s_lshl_b64 s[0:1], 0xffff, s0
-; GCN-NEXT:    s_xor_b64 s[4:5], s[2:3], s[4:5]
-; GCN-NEXT:    s_and_b64 s[0:1], s[4:5], s[0:1]
-; GCN-NEXT:    s_xor_b64 s[0:1], s[0:1], s[2:3]
-; GCN-NEXT:    v_mov_b32_e32 v3, s1
-; GCN-NEXT:    v_mov_b32_e32 v2, s0
+; GCN-NEXT:    v_mov_b32_e32 v3, s3
 ; GCN-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
 ; GCN-NEXT:    s_endpgm
 entry:
@@ -418,19 +418,19 @@ define amdgpu_kernel void @short4_inselt(ptr addrspace(1) %out, <4 x i16> %vec,
 ; GCN-LABEL: short4_inselt:
 ; GCN:       ; %bb.0: ; %entry
 ; GCN-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GCN-NEXT:    s_load_dword s5, s[4:5], 0x34
+; GCN-NEXT:    s_load_dword s6, s[4:5], 0x34
 ; GCN-NEXT:    s_mov_b32 s4, 0x10001
+; GCN-NEXT:    s_mov_b32 s5, s4
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_xor_b64 s[4:5], s[2:3], s[4:5]
+; GCN-NEXT:    s_lshl_b32 s6, s6, 4
+; GCN-NEXT:    s_lshl_b64 s[6:7], 0xffff, s6
+; GCN-NEXT:    s_and_b64 s[4:5], s[4:5], s[6:7]
+; GCN-NEXT:    s_xor_b64 s[2:3], s[4:5], s[2:3]
 ; GCN-NEXT:    v_mov_b32_e32 v0, s0
-; GCN-NEXT:    s_lshl_b32 s0, s5, 4
-; GCN-NEXT:    s_mov_b32 s5, s4
+; GCN-NEXT:    v_mov_b32_e32 v2, s2
 ; GCN-NEXT:    v_mov_b32_e32 v1, s1
-; GCN-NEXT:    s_lshl_b64 s[0:1], 0xffff, s0
-; GCN-NEXT:    s_xor_b64 s[4:5], s[2:3], s[4:5]
-; GCN-NEXT:    s_and_b64 s[0:1], s[4:5], s[0:1]
-; GCN-NEXT:    s_xor_b64 s[0:1], s[0:1], s[2:3]
-; GCN-NEXT:    v_mov_b32_e32 v3, s1
-; GCN-NEXT:    v_mov_b32_e32 v2, s0
+; GCN-NEXT:    v_mov_b32_e32 v3, s3
 ; GCN-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
 ; GCN-NEXT:    s_endpgm
 entry:
@@ -443,18 +443,18 @@ define amdgpu_kernel void @byte8_inselt(ptr addrspace(1) %out, <8 x i8> %vec, i3
 ; GCN-LABEL: byte8_inselt:
 ; GCN:       ; %bb.0: ; %entry
 ; GCN-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GCN-NEXT:    s_load_dword s4, s[4:5], 0x34
+; GCN-NEXT:    s_load_dword s6, s[4:5], 0x34
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_xor_b32 s5, s3, 0x1010101
+; GCN-NEXT:    s_lshl_b32 s6, s6, 3
+; GCN-NEXT:    s_xor_b32 s4, s2, 0x1010101
+; GCN-NEXT:    s_lshl_b64 s[6:7], 0xff, s6
+; GCN-NEXT:    s_and_b64 s[4:5], s[4:5], s[6:7]
+; GCN-NEXT:    s_xor_b64 s[2:3], s[4:5], s[2:3]
 ; GCN-NEXT:    v_mov_b32_e32 v0, s0
-; GCN-NEXT:    s_lshl_b32 s4, s4, 3
+; GCN-NEXT:    v_mov_b32_e32 v2, s2
 ; GCN-NEXT:    v_mov_b32_e32 v1, s1
-; GCN-NEXT:    s_xor_b32 s1, s3, 0x1010101
-; GCN-NEXT:    s_xor_b32 s0, s2, 0x1010101
-; GCN-NEXT:    s_lshl_b64 s[4:5], 0xff, s4
-; GCN-NEXT:    s_and_b64 s[0:1], s[0:1], s[4:5]
-; GCN-NEXT:    s_xor_b64 s[0:1], s[0:1], s[2:3]
-; GCN-NEXT:    v_mov_b32_e32 v3, s1
-; GCN-NEXT:    v_mov_b32_e32 v2, s0
+; GCN-NEXT:    v_mov_b32_e32 v3, s3
 ; GCN-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
 ; GCN-NEXT:    s_endpgm
 entry:
diff --git a/llvm/test/CodeGen/AMDGPU/insert_vector_elt.ll b/llvm/test/CodeGen/AMDGPU/insert_vector_elt.ll
index e98d04556649a..7cbf9aeacfe48 100644
--- a/llvm/test/CodeGen/AMDGPU/insert_vector_elt.ll
+++ b/llvm/test/CodeGen/AMDGPU/insert_vector_elt.ll
@@ -1571,13 +1571,13 @@ define amdgpu_kernel void @dynamic_insertelement_v3i16(ptr addrspace(1) %out, <3
 ; VI-NEXT:    s_mov_b32 s6, -1
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    s_mov_b32 s4, s0
-; VI-NEXT:    s_lshl_b32 s0, s8, 4
-; VI-NEXT:    s_mov_b32 s8, 0x50005
-; VI-NEXT:    s_mov_b32 s9, s8
+; VI-NEXT:    s_mov_b32 s0, 0x50005
 ; VI-NEXT:    s_mov_b32 s5, s1
-; VI-NEXT:    s_lshl_b64 s[0:1], 0xffff, s0
-; VI-NEXT:    s_xor_b64 s[8:9], s[2:3], s[8:9]
-; VI-NEXT:    s_and_b64 s[0:1], s[8:9], s[0:1]
+; VI-NEXT:    s_mov_b32 s1, s0
+; VI-NEXT:    s_lshl_b32 s8, s8, 4
+; VI-NEXT:    s_xor_b64 s[0:1], s[2:3], s[0:1]
+; VI-NEXT:    s_lshl_b64 s[8:9], 0xffff, s8
+; VI-NEXT:    s_and_b64 s[0:1], s[0:1], s[8:9]
 ; VI-NEXT:    s_xor_b64 s[0:1], s[0:1], s[2:3]
 ; VI-NEXT:    v_mov_b32_e32 v0, s1
 ; VI-NEXT:    buffer_store_short v0, off, s[4:7], 0 offset:4
diff --git a/llvm/test/CodeGen/AMDGPU/kernel-args.ll b/llvm/test/CodeGen/AMDGPU/kernel-args.ll
index 41b5103b38e50..a2da8876472ab 100644
--- a/llvm/test/CodeGen/AMDGPU/kernel-args.ll
+++ b/llvm/test/CodeGen/AMDGPU/kernel-args.ll
@@ -734,8 +734,8 @@ define amdgpu_kernel void @v2i32_arg(ptr addrspace(1) nocapture %out, <2 x i32>
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v0, s0
-; VI-NEXT:    v_mov_b32_e32 v1, s1
 ; VI-NEXT:    v_mov_b32_e32 v2, s2
+; VI-NEXT:    v_mov_b32_e32 v1, s1
 ; VI-NEXT:    v_mov_b32_e32 v3, s3
 ; VI-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
 ; VI-NEXT:    s_endpgm
@@ -797,8 +797,8 @@ define amdgpu_kernel void @v2f32_arg(ptr addrspace(1) nocapture %out, <2 x float
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v0, s0
-; VI-NEXT:    v_mov_b32_e32 v1, s1
 ; VI-NEXT:    v_mov_b32_e32 v2, s2
+; VI-NEXT:    v_mov_b32_e32 v1, s1
 ; VI-NEXT:    v_mov_b32_e32 v3, s3
 ; VI-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
 ; VI-NEXT:    s_endpgm
@@ -1000,16 +1000,16 @@ define amdgpu_kernel void @v3i16_arg(ptr addrspace(1) nocapture %out, <3 x i16>
 ; VI:       ; %bb.0: ; %entry
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    s_add_u32 s4, s0, 4
+; VI-NEXT:    s_addc_u32 s5, s1, 0
+; VI-NEXT:    v_mov_b32_e32 v2, s4
+; VI-NEXT:    v_mov_b32_e32 v4, s3
 ; VI-NEXT:    v_mov_b32_e32 v0, s0
+; VI-NEXT:    v_mov_b32_e32 v3, s5
 ; VI-NEXT:    v_mov_b32_e32 v1, s1
-; VI-NEXT:    v_mov_b32_e32 v2, s2
-; VI-NEXT:    s_add_u32 s0, s0, 4
-; VI-NEXT:    flat_store_dword v[0:1], v2
-; VI-NEXT:    s_addc_u32 s1, s1, 0
-; VI-NEXT:    v_mov_b32_e32 v0, s0
-; VI-NEXT:    v_mov_b32_e32 v1, s1
-; VI-NEXT:    v_mov_b32_e32 v2, s3
-; VI-NEXT:    flat_store_short v[0:1], v2
+; VI-NEXT:    v_mov_b32_e32 v5, s2
+; VI-NEXT:    flat_store_short v[2:3], v4
+; VI-NEXT:    flat_store_dword v[0:1], v5
 ; VI-NEXT:    s_endpgm
 ;
 ; GFX9-LABEL: v3i16_arg:
@@ -1335,8 +1335,8 @@ define amdgpu_kernel void @v4i16_arg(ptr addrspace(1) %out, <4 x i16> %in) {
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v0, s0
-; VI-NEXT:    v_mov_b32_e32 v1, s1
 ; VI-NEXT:    v_mov_b32_e32 v2, s2
+; VI-NEXT:    v_mov_b32_e32 v1, s1
 ; VI-NEXT:    v_mov_b32_e32 v3, s3
 ; VI-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
 ; VI-NEXT:    s_endpgm
@@ -2400,8 +2400,8 @@ define amdgpu_kernel void @v8i8_arg(ptr addrspace(1) %out, <8 x i8> %in) {
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v0, s0
-; VI-NEXT:    v_mov_b32_e32 v1, s1
 ; VI-NEXT:    v_mov_b32_e32 v2, s2
+; VI-NEXT:    v_mov_b32_e32 v1, s1
 ; VI-NEXT:    v_mov_b32_e32 v3, s3
 ; VI-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
 ; VI-NEXT:    s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx950.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx950.ll
index 23c5a079c5c6e..ab0000f6831b6 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx950.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx950.ll
@@ -139,26 +139,20 @@ define amdgpu_kernel void @test_mfma_f32_16x16x32_f16_no_agpr__vgprcd(ptr addrsp
 ; SDAG-LABEL: test_mfma_f32_16x16x32_f16_no_agpr__vgprcd:
 ; SDAG:       ; %bb.0:
 ; SDAG-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x34
-; SDAG-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x24
 ; SDAG-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x54
-; SDAG-NEXT:    v_mov_b32_e32 v12, 0
+; SDAG-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x24
+; SDAG-NEXT:    v_mov_b32_e32 v4, 0
 ; SDAG-NEXT:    s_waitcnt lgkmcnt(0)
-; SDAG-NEXT:    v_mov_b32_e32 v0, s8
-; SDAG-NEXT:    v_mov_b32_e32 v1, s9
-; SDAG-NEXT:    v_mov_b32_e32 v2, s10
-; SDAG-NEXT:    v_mov_b32_e32 v3, s11
-; SDAG-NEXT:    v_mov_b32_e32 v4, s12
-; SDAG-NEXT:    v_mov_b32_e32 v5, s13
-; SDAG-NEXT:    v_mov_b32_e32 v6, s14
-; SDAG-NEXT:    v_mov_b32_e32 v7, s15
-; SDAG-NEXT:    v_mov_b32_e32 v8, s0
-; SDAG-NEXT:    v_mov_b32_e32 v9, s1
-; SDAG-NEXT:    v_mov_b32_e32 v10, s2
-; SDAG-NEXT:    v_mov_b32_e32 v11, s3
+; SDAG-NEXT:    v_mov_b64_e32 v[6:7], s[8:9]
+; SDAG-NEXT:    v_mov_b64_e32 v[8:9], s[10:11]
+; SDAG-NEXT:    v_mov_b64_e32 v[10:11], s[12:13]
+; SDAG-NEXT:    v_mov_b64_e32 v[0:1], s[0:1]
+; SDAG-NEXT:    v_mov_b64_e32 v[12:13], s[14:15]
+; SDAG-NEXT:    v_mov_b64_e32 v[2:3], s[2:3]
 ; SDAG-NEXT:    s_nop 1
-; SDAG-NEXT:    v_mfma_f32_16x16x32_f16 v[0:3], v[0:3], v[4:7], v[8:11]
+; SDAG-NEXT:    v_mfma_f32_16x16x32_f16 v[0:3], v[6:9], v[10:13], v[0:3]
 ; SDAG-NEXT:    s_nop 7
-; SDAG-NEXT:    global_store_dwordx4 v12, v[0:3], s[6:7]
+; SDAG-NEXT:    global_store_dwordx4 v4, v[0:3], s[6:7]
 ; SDAG-NEXT:    s_endpgm
 ;
 ; GISEL-LABEL: test_mfma_f32_16x16x32_f16_no_agpr__vgprcd:
@@ -183,51 +177,39 @@ define amdgpu_kernel void @test_mfma_f32_16x16x32_f16_no_agpr__vgprcd(ptr addrsp
 ; HEURRC-LABEL: test_mfma_f32_16x16x32_f16_no_agpr__vgprcd:
 ; HEURRC:       ; %bb.0:
 ; HEURRC-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x34
-; HEURRC-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x24
 ; HEURRC-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x54
-; HEURRC-NEXT:    v_mov_b32_e32 v12, 0
+; HEURRC-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x24
+; HEURRC-NEXT:    v_mov_b32_e32 v4, 0
 ; HEURRC-NEXT:    s_waitcnt lgkmcnt(0)
-; HEURRC-NEXT:    v_mov_b32_e32 v0, s8
-; HEURRC-NEXT:    v_mov_b32_e32 v1, s9
-; HEURRC-NEXT:    v_mov_b32_e32 v2, s10
-; HEURRC-NEXT:    v_mov_b32_e32 v3, s11
-; HEURRC-NEXT:    v_mov_b32_e32 v4, s12
-; HEURRC-NEXT:    v_mov_b32_e32 v5, s13
-; HEURRC-NEXT:    v_mov_b32_e32 v6, s14
-; HEURRC-NEXT:    v_mov_b32_e32 v7, s15
-; HEURRC-NEXT:    v_mov_b32_e32 v8, s0
-; HEURRC-NEXT:    v_mov_b32_e32 v9, s1
-; HEURRC-NEXT:    v_mov_b32_e32 v10, s2
-; HEURRC-NEXT:    v_mov_b32_e32 v11, s3
+; HEURRC-NEXT:    v_mov_b64_e32 v[6:7], s[8:9]
+; HEURRC-NEXT:    v_mov_b64_e32 v[8:9], s[10:11]
+; HEURRC-NEXT:    v_mov_b64_e32 v[10:11], s[12:13]
+; HEURRC-NEXT:    v_mov_b64_e32 v[0:1], s[0:1]
+; HEURRC-NEXT:    v_mov_b64_e32 v[12:13], s[14:15]
+; HEURRC-NEXT:    v_mov_b64_e32 v[2:3], s[2:3]
 ; HEURRC-NEXT:    s_nop 1
-; HEURRC-NEXT:    v_mfma_f32_16x16x32_f16 v[0:3], v[0:3], v[4:7], v[8:11]
+; HEURRC-NEXT:    v_mfma_f32_16x16x32_f16 v[0:3], v[6:9], v[10:13], v[0:3]
 ; HEURRC-NEXT:    s_nop 7
-; HEURRC-NEXT:    global_store_dwordx4 v12, v[0:3], s[6:7]
+; HEURRC-NEXT:    global_store_dwordx4 v4, v[0:3], s[6:7]
 ; HEURRC-NEXT:    s_endpgm
 ;
 ; VGPRRC-LABEL: test_mfma_f32_16x16x32_f16_no_agpr__vgprcd:
 ; VGPRRC:       ; %bb.0:
 ; VGPRRC-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x34
-; VGPRRC-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x24
 ; VGPRRC-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x54
-; VGPRRC-NEXT:    v_mov_b32_e32 v12, 0
+; VGPRRC-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x24
+; VGPRRC-NEXT:    v_mov_b32_e32 v4, 0
 ; VGPRRC-NEXT:    s_waitcnt lgkmcnt(0)
-; VGPRRC-NEXT:    v_mov_b32_e32 v0, s8
-; VGPRRC-NEXT:    v_mov_b32_e32 v1, s9
-; VGPRRC-NEXT:    v_mov_b32_e32 v2, s10
-; VGPRRC-NEXT:    v_mov_b32_e32 v3, s11
-; VGPRRC-NEXT:    v_mov_b32_e32 v4, s12
-; VGPRRC-NEXT:    v_mov_b32_e32 v5, s13
-; VGPRRC-NEXT:    v_mov_b32_e32 v6, s14
-; VGPRRC-NEXT:    v_mov_b32_e32 v7, s15
-; VGPRRC-NEXT:    v_mov_b32_e32 v8, s0
-; VGPRRC-NEXT:    v_mov_b32_e32 v9, s1
-; VGPRRC-NEXT:    v_mov_b32_e32 v10, s2
-; VGPRRC-NEXT:    v_mov_b32_e32 v11, s3
+; VGPRRC-NEXT:    v_mov_b64_e32 v[6:7], s[8:9]
+; VGPRRC-NEXT:    v_mov_b64_e32 v[8:9], s[10:11]
+; VGPRRC-NEXT:    v_mov_b64_e32 v[10:11], s[12:13]
+; VGPRRC-NEXT:    v_mov_b64_e32 v[0:1], s[0:1]
+; VGPRRC-NEXT:    v_mov_b64_e32 v[12:13], s[14:15]
+; VGPRRC-NEXT:    v_mov_b64_e32 v[2:3], s[2:3]
 ; VGPRRC-NEXT:    s_nop 1
-; VGPRRC-NEXT:    v_mfma_f32_16x16x32_f16 v[0:3], v[0:3], v[4:7], v[8:11]
+; VGPRRC-NEXT:    v_mfma_f32_16x16x32_f16 v[0:3], v[6:9], v[10:13], v[0:3]
 ; VGPRRC-NEXT:    s_nop 7
-; VGPRRC-NEXT:    global_store_dwordx4 v12, v[0:3], s[6:7]
+; VGPRRC-NEXT:    global_store_dwordx4 v4, v[0:3], s[6:7]
 ; VGPRRC-NEXT:    s_endpgm
 ; AGPR-LABEL: test_mfma_f32_16x16x32_f16_no_agpr__vgprcd:
 ; AGPR:       ; %bb.0:
@@ -276,26 +258,20 @@ define amdgpu_kernel void @test_mfma_f32_16x16x32_f16_no_agpr__vgprcd__flags(ptr
 ; SDAG-LABEL: test_mfma_f32_16x16x32_f16_no_agpr__vgprcd__flags:
 ; SDAG:       ; %bb.0:
 ; SDAG-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x34
-; SDAG-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x24
 ; SDAG-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x54
-; SDAG-NEXT:    v_mov_b32_e32 v12, 0
+; SDAG-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x24
+; SDAG-NEXT:    v_mov_b32_e32 v4, 0
 ; SDAG-NEXT:    s_waitcnt lgkmcnt(0)
-; SDAG-NEXT:    v_mov_b32_e32 v0, s8
-; SDAG-NEXT:    v_mov_b32_e32 v1, s9
-; SDAG-NEXT:    v_mov_b32_e32 v2, s10
-; SDAG-NEXT:    v_mov_b32_e32 v3, s11
-; SDAG-NEXT:    v_mov_b32_e32 v4, s12
-; SDAG-NEXT:    v_mov_b32_e32 v5, s13
-; SDAG-NEXT:    v_mov_b32_e32 v6, s14
-; SDAG-NEXT:    v_mov_b32_e32 v7, s15
-; SDAG-NEXT:    v_mov_b32_e32 v8, s0
-; SDAG-NEXT:    v_mov_b32_e32 v9, s1
-; SDAG-NEXT:    v_mov_b32_e32 v10, s2
-; SDAG-NEXT:    v_mov_b32_e32 v11, s3
+; SDAG-NEXT:    v_mov_b64_e32 v[6:7], s[8:9]
+; SDAG-NEXT:    v_mov_b64_e32 v[8:9], s[10:11]
+; SDAG-NEXT:    v_mov_b64_e32 v[10:11], s[12:13]
+; SDAG-NEXT:    v_mov_b64_e32 v[0:1], s[0:1]
+; SDAG-NEXT:    v_mov_b64_e32 v[12:13], s[14:15]
+; SDAG-NEXT:    v_mov_b64_e32 v[2:3], s[2:3]
 ; SDAG-NEXT:    s_nop 1
-; SDAG-NEXT:    v_mfma_f32_16x16x32_f16 v[0:3], v[0:3], v[4:7], v[8:11] cbsz:3 abid:2 blgp:1
+; SDAG-NEXT:    v_mfma_f32_16x16x32_f16 v[0:3], v[6:9], v[10:13], v[0:3] cbsz:3 abid:2 blgp:1
 ; SDAG-NEXT:    s_nop 7
-; SDAG-NEXT:    global_store_dwordx4 v12, v[0:3], s[6:7]
+; SDAG-NEXT:    global_store_dwordx4 v4, v[0:3], s[6:7]
 ; SDAG-NEXT:    s_endpgm
 ;
 ; GISEL-LABEL: test_mfma_f32_16x16x32_f16_no_agpr__vgprcd__flags:
@@ -320,51 +296,39 @@ define amdgpu_kernel void @test_mfma_f32_16x16x32_f16_no_agpr__vgprcd__flags(ptr
 ; HEURRC-LABEL: test_mfma_f32_16x16x32_f16_no_agpr__vgprcd__flags:
 ; HEURRC:       ; %bb.0:
 ; HEURRC-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x34
-; HEURRC-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x24
 ; HEURRC-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x54
-; HEURRC-NEXT:    v_mov_b32_e32 v12, 0
+; HEURRC-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x24
+; HEURRC-NEXT:    v_mov_b32_e32 v4, 0
 ; HEURRC-NEXT:    s_waitcnt lgkmcnt(0)
-; HEURRC-NEXT:    v_mov_b32_e32 v0, s8
-; HEURRC-NEXT:    v_mov_b32_e32 v1, s9
-; HEURRC-NEXT:    v_mov_b32_e32 v2, s10
-; HEURRC-NEXT:    v_mov_b32_e32 v3, s11
-; HEURRC-NEXT:    v_mov_b32_e32 v4, s12
-; HEURRC-NEXT:    v_mov_b32_e32 v5, s13
-; HEURRC-NEXT:    v_mov_b32_e32 v6, s14
-; HEURRC-NEXT:    v_mov_b32_e32 v7, s15
-; HEURRC-NEXT:    v_mov_b32_e32 v8, s0
-; HEURRC-NEXT:    v_mov_b32_e32 v9, s1
-; HEURRC-NEXT:    v_mov_b32_e32 v10, s2
-; HEURRC-NEXT:    v_mov_b32_e32 v11, s3
+; HEURRC-NEXT:    v_mov_b64_e32 v[6:7], s[8:9]
+; HEURRC-NEXT:    v_mov_b64_e32 v[8:9], s[10:11]
+; HEURRC-NEXT:    v_mov_b64_e32 v[10:11], s[12:13]
+; HEURRC-NEXT:    v_mov_b64_e32 v[0:1], s[0:1]
+; HEURRC-NEXT:    v_mov_b64_e32 v[12:13], s[14:15]
+; HEURRC-NEXT:    v_mov_b64_e32 v[2:3], s[2:3]
 ; HEURRC-NEXT:    s_nop 1
-; HEURRC-NEXT:    v_mfma_f32_16x16x32_f16 v[0:3], v[0:3], v[4:7], v[8:11] cbsz:3 abid:2 blgp:1
+; HEURRC-NEXT:    v_mfma_f32_16x16x32_f16 v[0:3], v[6:9], v[10:13], v[0:3] cbsz:3 abid:2 blgp:1
 ; HEURRC-NEXT:    s_nop 7
-; HEURRC-NEXT:    global_store_dwordx4 v12, v[0:3], s[6:7]
+; HEURRC-NEXT:    global_store_dwordx4 v4, v[0:3], s[6:7]
 ; HEURRC-NEXT:    s_endpgm
 ;
 ; VGPRRC-LABEL: test_mfma_f32_16x16x32_f16_no_agpr__vgprcd__flags:
 ; VGPRRC:       ; %bb.0:
 ; VGPRRC-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x34
-; VGPRRC-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x24
 ; VGPRRC-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x54
-; VGPRRC-NEXT:    v_mov_b32_e32 v12, 0
+; VGPRRC-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x24
+; VGPRRC-NEXT:    v_mov_b32_e32 v4, 0
 ; VGPRRC-NEXT:    s_waitcnt lgkmcnt(0)
-; VGPRRC-NEXT:    v_mov_b32_e32 v0, s8
-; VGPRRC-NEXT:    v_mov_b32_e32 v1, s9
-; VGPRRC-NEXT:    v_mov_b32_e32 v2, s10
-; VGPRRC-NEXT:    v_mov_b32_e32 v3, s11
-; VGPRRC-NEXT:    v_mov_b32_e32 v4, s12
-; VGPRRC-NEXT:    v_mov_b32_e32 v5, s13
-; VGPRRC-NEXT:    v_mov_b32_e32 v6, s14
-; VGPRRC-NEXT:    v_mov_b32_e32 v7, s15
-; VGPRRC-NEXT:    v_mov_b32_e32 v8, s0
-; VGPRRC-NEXT:    v_mov_b32_e32 v9, s1
-; VGPRRC-NEXT:    v_mov_b32_e32 v10, s2
-; VGPRRC-NEXT:    v_mov_b32_e32 v11, s3
+; VGPRRC-NEXT:    v_mov_b64_e32 v[6:7], s[8:9]
+; VGPRRC-NEXT:    v_mov_b64_e32 v[8:9], s[10:11]
+; VGPRRC-NEXT:    v_mov_b64_e32 v[10:11], s[12:13]
+; VGPRRC-NEXT:    v_mov_b64_e32 v[0:1], s[0:1]
+; VGPRRC-NEXT:    v_mov_b64_e32 v[12:13], s[14:15]
+; VGPRRC-NEXT:    v_mov_b64_e32 v[2:3], s[2:3]
 ; VGPRRC-NEXT:    s_nop 1
-; VGPRRC-NEXT:    v_mfma_f32_16x16x32_f16 v[0:3], v[0:3], v[4:7], v[8:11] cbsz:3 abid:2 blgp:1
+; VGPRRC-NEXT:    v_mfma_f32_16x16x32_f16 v[0:3], v[6:9], v[10:13], v[0:3] cbsz:3 abid:2 blgp:1
 ; VGPRRC-NEXT:    s_nop 7
-; VGPRRC-NEXT:    global_store_dwordx4 v12, v[0:3], s[6:7]
+; VGPRRC-NEXT:    global_store_dwordx4 v4, v[0:3], s[6:7]
 ; VGPRRC-NEXT:    s_endpgm
 ; AGPR-LABEL: test_mfma_f32_16x16x32_f16_no_agpr__vgprcd__flags:
 ; AGPR:       ; %bb.0:
@@ -5455,76 +5419,58 @@ define amdgpu_kernel void @test_mfma_f32_16x16x32_bf16_no_agpr__vgprcd(ptr addrs
 ; GCN-LABEL: test_mfma_f32_16x16x32_bf16_no_agpr__vgprcd:
 ; GCN:       ; %bb.0:
 ; GCN-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x34
-; GCN-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x24
 ; GCN-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x54
-; GCN-NEXT:    v_mov_b32_e32 v12, 0
+; GCN-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x24
+; GCN-NEXT:    v_mov_b32_e32 v4, 0
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
-; GCN-NEXT:    v_mov_b32_e32 v0, s8
-; GCN-NEXT:    v_mov_b32_e32 v1, s9
-; GCN-NEXT:    v_mov_b32_e32 v2, s10
-; GCN-NEXT:    v_mov_b32_e32 v3, s11
-; GCN-NEXT:    v_mov_b32_e32 v4, s12
-; GCN-NEXT:    v_mov_b32_e32 v5, s13
-; GCN-NEXT:    v_mov_b32_e32 v6, s14
-; GCN-NEXT:    v_mov_b32_e32 v7, s15
-; GCN-NEXT:    v_mov_b32_e32 v8, s0
-; GCN-NEXT:    v_mov_b32_e32 v9, s1
-; GCN-NEXT:    v_mov_b32_e32 v10, s2
-; GCN-NEXT:    v_mov_b32_e32 v11, s3
+; GCN-NEXT:    v_mov_b64_e32 v[6:7], s[8:9]
+; GCN-NEXT:    v_mov_b64_e32 v[8:9], s[10:11]
+; GCN-NEXT:    v_mov_b64_e32 v[10:11], s[12:13]
+; GCN-NEXT:    v_mov_b64_e32 v[0:1], s[0:1]
+; GCN-NEXT:    v_mov_b64_e32 v[12:13], s[14:15]
+; GCN-NEXT:    v_mov_b64_e32 v[2:3], s[2:3]
 ; GCN-NEXT:    s_nop 1
-; GCN-NEXT:    v_mfma_f32_16x16x32_bf16 v[0:3], v[0:3], v[4:7], v[8:11]
+; GCN-NEXT:    v_mfma_f32_16x16x32_bf16 v[0:3], v[6:9], v[10:13], v[0:3]
 ; GCN-NEXT:    s_nop 7
-; GCN-NEXT:    global_store_dwordx4 v12, v[0:3], s[6:7]
+; GCN-NEXT:    global_store_dwordx4 v4, v[0:3], s[6:7]
 ; GCN-NEXT:    s_endpgm
 ;
 ; HEURRC-LABEL: test_mfma_f32_16x16x32_bf16_no_agpr__vgprcd:
 ; HEURRC:       ; %bb.0:
 ; HEURRC-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x34
-; HEURRC-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x24
 ; HEURRC-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x54
-; HEURRC-NEXT:    v_mov_b32_e32 v12, 0
+; HEURRC-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x24
+; HEURRC-NEXT:    v_mov_b32_e32 v4, 0
 ; HEURRC-NEXT:    s_waitcnt lgkmcnt(0)
-; HEURRC-NEXT:    v_mov_b32_e32 v0, s8
-; HEURRC-NEXT:    v_mov_b32_e32 v1, s9
-; HEURRC-NEXT:    v_mov_b32_e32 v2, s10
-; HEURRC-NEXT:    v_mov_b32_e32 v3, s11
-; HEURRC-NEXT:    v_mov_b32_e32 v4, s12
-; HEURRC-NEXT:    v_mov_b32_e32 v5, s13
-; HEURRC-NEXT:    v_mov_b32_e32 v6, s14
-; HEURRC-NEXT:    v_mov_b32_e32 v7, s15
-; HEURRC-NEXT:    v_mov_b32_e32 v8, s0
-; HEURRC-NEXT:    v_mov_b32_e32 v9, s1
-; HEURRC-NEXT:    v_mov_b32_e32 v10, s2
-; HEURRC-NEXT:    v_mov_b32_e32 v11, s3
+; HEURRC-NEXT:    v_mov_b64_e32 v[6:7], s[8:9]
+; HEURRC-NEXT:    v_mov_b64_e32 v[8:9], s[10:11]
+; HEURRC-NEXT:    v_mov_b64_e32 v[10:11], s[12:13]
+; HEURRC-NEXT:    v_mov_b64_e32 v[0:1], s[0:1]
+; HEURRC-NEXT:    v_mov_b64_e32 v[12:13], s[14:15]
+; HEURRC-NEXT:    v_mov_b64_e32 v[2:3], s[2:3]
 ; HEURRC-NEXT:    s_nop 1
-; HEURRC-NEXT:    v_mfma_f32_16x16x32_bf16 v[0:3], v[0:3], v[4:7], v[8:11]
+; HEURRC-NEXT:    v_mfma_f32_16x16x32_bf16 v[0:3], v[6:9], v[10:13], v[0:3]
 ; HEURRC-NEXT:    s_nop 7
-; HEURRC-NEXT:    global_store_dwordx4 v12, v[0:3], s[6:7]
+; HEURRC-NEXT:    global_store_dwordx4 v4, v[0:3], s[6:7]
 ; HEURRC-NEXT:    s_endpgm
 ;
 ; VGPRRC-LABEL: test_mfma_f32_16x16x32_bf16_no_agpr__vgprcd:
 ; VGPRRC:       ; %bb.0:
 ; VGPRRC-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x34
-; VGPRRC-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x24
 ; VGPRRC-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x54
-; VGPRRC-NEXT:    v_mov_b32_e32 v12, 0
+; VGPRRC-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x24
+; VGPRRC-NEXT:    v_mov_b32_e32 v4, 0
 ; VGPRRC-NEXT:    s_waitcnt lgkmcnt(0)
-; VGPRRC-NEXT:    v_mov_b32_e32 v0, s8
-; VGPRRC-NEXT:    v_mov_b32_e32 v1, s9
-; VGPRRC-NEXT:    v_mov_b32_e32 v2, s10
-; VGPRRC-NEXT:    v_mov_b32_e32 v3, s11
-; VGPRRC-NEXT:    v_mov_b32_e32 v4, s12
-; VGPRRC-NEXT:    v_mov_b32_e32 v5, s13
-; VGPRRC-NEXT:    v_mov_b32_e32 v6, s14
-; VGPRRC-NEXT:    v_mov_b32_e32 v7, s15
-; VGPRRC-NEXT:    v_mov_b32_e32 v8, s0
-; VGPRRC-NEXT:    v_mov_b32_e32 v9, s1
-; VGPRRC-NEXT:    v_mov_b32_e32 v10, s2
-; VGPRRC-NEXT:    v_mov_b32_e32 v11, s3
+; VGPRRC-NEXT:    v_mov_b64_e32 v[6:7], s[8:9]
+; VGPRRC-NEXT:    v_mov_b64_e32 v[8:9], s[10:11]
+; VGPRRC-NEXT:    v_mov_b64_e32 v[10:11], s[12:13]
+; VGPRRC-NEXT:    v_mov_b64_e32 v[0:1], s[0:1]
+; VGPRRC-NEXT:    v_mov_b64_e32 v[12:13], s[14:15]
+; VGPRRC-NEXT:    v_mov_b64_e32 v[2:3], s[2:3]
 ; VGPRRC-NEXT:    s_nop 1
-; VGPRRC-NEXT:    v_mfma_f32_16x16x32_bf16 v[0:3], v[0:3], v[4:7], v[8:11]
+; VGPRRC-NEXT:    v_mfma_f32_16x16x32_bf16 v[0:3], v[6:9], v[10:13], v[0:3]
 ; VGPRRC-NEXT:    s_nop 7
-; VGPRRC-NEXT:    global_store_dwordx4 v12, v[0:3], s[6:7]
+; VGPRRC-NEXT:    global_store_dwordx4 v4, v[0:3], s[6:7]
 ; VGPRRC-NEXT:    s_endpgm
 ; AGPR-LABEL: test_mfma_f32_16x16x32_bf16_no_agpr__vgprcd:
 ; AGPR:       ; %bb.0:
@@ -5573,76 +5519,58 @@ define amdgpu_kernel void @test_mfma_f32_16x16x32_bf16_no_agpr__vgprcd__flags(pt
 ; GCN-LABEL: test_mfma_f32_16x16x32_bf16_no_agpr__vgprcd__flags:
 ; GCN:       ; %bb.0:
 ; GCN-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x34
-; GCN-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x24
 ; GCN-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x54
-; GCN-NEXT:    v_mov_b32_e32 v12, 0
+; GCN-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x24
+; GCN-NEXT:    v_mov_b32_e32 v4, 0
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
-; GCN-NEXT:    v_mov_b32_e32 v0, s8
-; GCN-NEXT:    v_mov_b32_e32 v1, s9
-; GCN-NEXT:    v_mov_b32_e32 v2, s10
-; GCN-NEXT:    v_mov_b32_e32 v3, s11
-; GCN-NEXT:    v_mov_b32_e32 v4, s12
-; GCN-NEXT:    v_mov_b32_e32 v5, s13
-; GCN-NEXT:    v_mov_b32_e32 v6, s14
-; GCN-NEXT:    v_mov_b32_e32 v7, s15
-; GCN-NEXT:    v_mov_b32_e32 v8, s0
-; GCN-NEXT:    v_mov_b32_e32 v9, s1
-; GCN-NEXT:    v_mov_b32_e32 v10, s2
-; GCN-NEXT:    v_mov_b32_e32 v11, s3
+; GCN-NEXT:    v_mov_b64_e32 v[6:7], s[8:9]
+; GCN-NEXT:    v_mov_b64_e32 v[8:9], s[10:11]
+; GCN-NEXT:    v_mov_b64_e32 v[10:11], s[12:13]
+; GCN-NEXT:    v_mov_b64_e32 v[0:1], s[0:1]
+; GCN-NEXT:    v_mov_b64_e32 v[12:13], s[14:15]
+; GCN-NEXT:    v_mov_b64_e32 v[2:3], s[2:3]
 ; GCN-NEXT:    s_nop 1
-; GCN-NEXT:    v_mfma_f32_16x16x32_bf16 v[0:3], v[0:3], v[4:7], v[8:11] cbsz:3 abid:2 blgp:1
+; GCN-NEXT:    v_mfma_f32_16x16x32_bf16 v[0:3], v[6:9], v[10:13], v[0:3] cbsz:3 abid:2 blgp:1
 ; GCN-NEXT:    s_nop 7
-; GCN-NEXT:    global_store_dwordx4 v12, v[0:3], s[6:7]
+; GCN-NEXT:    global_store_dwordx4 v4, v[0:3], s[6:7]
 ; GCN-NEXT:    s_endpgm
 ;
 ; HEURRC-LABEL: test_mfma_f32_16x16x32_bf16_no_agpr__vgprcd__flags:
 ; HEURRC:       ; %bb.0:
 ; HEURRC-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x34
-; HEURRC-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x24
 ; HEURRC-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x54
-; HEURRC-NEXT:    v_mov_b32_e32 v12, 0
+; HEURRC-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x24
+; HEURRC-NEXT:    v_mov_b32_e32 v4, 0
 ; HEURRC-NEXT:    s_waitcnt lgkmcnt(0)
-; HEURRC-NEXT:    v_mov_b32_e32 v0, s8
-; HEURRC-NEXT:    v_mov_b32_e32 v1, s9
-; HEURRC-NEXT:    v_mov_b32_e32 v2, s10
-; HEURRC-NEXT:    v_mov_b32_e32 v3, s11
-; HEURRC-NEXT:    v_mov_b32_e32 v4, s12
-; HEURRC-NEXT:    v_mov_b32_e32 v5, s13
-; HEURRC-NEXT:    v_mov_b32_e32 v6, s14
-; HEURRC-NEXT:    v_mov_b32_e32 v7, s15
-; HEURRC-NEXT:    v_mov_b32_e32 v8, s0
-; HEURRC-NEXT:    v_mov_b32_e32 v9, s1
-; HEURRC-NEXT:    v_mov_b32_e32 v10, s2
-; HEURRC-NEXT:    v_mov_b32_e32 v11, s3
+; HEURRC-NEXT:    v_mov_b64_e32 v[6:7], s[8:9]
+; HEURRC-NEXT:    v_mov_b64_e32 v[8:9], s[10:11]
+; HEURRC-NEXT:    v_mov_b64_e32 v[10:11], s[12:13]
+; HEURRC-NEXT:    v_mov_b64_e32 v[0:1], s[0:1]
+; HEURRC-NEXT:    v_mov_b64_e32 v[12:13], s[14:15]
+; HEURRC-NEXT:    v_mov_b64_e32 v[2:3], s[2:3]
 ; HEURRC-NEXT:    s_nop 1
-; HEURRC-NEXT:    v_mfma_f32_16x16x32_bf16 v[0:3], v[0:3], v[4:7], v[8:11] cbsz:3 abid:2 blgp:1
+; HEURRC-NEXT:    v_mfma_f32_16x16x32_bf16 v[0:3], v[6:9], v[10:13], v[0:3] cbsz:3 abid:2 blgp:1
 ; HEURRC-NEXT:    s_nop 7
-; HEURRC-NEXT:    global_store_dwordx4 v12, v[0:3], s[6:7]
+; HEURRC-NEXT:    global_store_dwordx4 v4, v[0:3], s[6:7]
 ; HEURRC-NEXT:    s_endpgm
 ;
 ; VGPRRC-LABEL: test_mfma_f32_16x16x32_bf16_no_agpr__vgprcd__flags:
 ; VGPRRC:       ; %bb.0:
 ; VGPRRC-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x34
-; VGPRRC-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x24
 ; VGPRRC-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x54
-; VGPRRC-NEXT:    v_mov_b32_e32 v12, 0
+; VGPRRC-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x24
+; VGPRRC-NEXT:    v_mov_b32_e32 v4, 0
 ; VGPRRC-NEXT:    s_waitcnt lgkmcnt(0)
-; VGPRRC-NEXT:    v_mov_b32_e32 v0, s8
-; VGPRRC-NEXT:    v_mov_b32_e32 v1, s9
-; VGPRRC-NEXT:    v_mov_b32_e32 v2, s10
-; VGPRRC-NEXT:    v_mov_b32_e32 v3, s11
-; VGPRRC-NEXT:    v_mov_b32_e32 v4, s12
-; VGPRRC-NEXT:    v_mov_b32_e32 v5, s13
-; VGPRRC-NEXT:    v_mov_b32_e32 v6, s14
-; VGPRRC-NEXT:    v_mov_b32_e32 v7, s15
-; VGPRRC-NEXT:    v_mov_b32_e32 v8, s0
-; VGPRRC-NEXT:    v_mov_b32_e32 v9, s1
-; VGPRRC-NEXT:    v_mov_b32_e32 v10, s2
-; VGPRRC-NEXT:    v_mov_b32_e32 v11, s3
+; VGPRRC-NEXT:    v_mov_b64_e32 v[6:7], s[8:9]
+; VGPRRC-NEXT:    v_mov_b64_e32 v[8:9], s[10:11]
+; VGPRRC-NEXT:    v_mov_b64_e32 v[10:11], s[12:13]
+; VGPRRC-NEXT:    v_mov_b64_e32 v[0:1], s[0:1]
+; VGPRRC-NEXT:    v_mov_b64_e32 v[12:13], s[14:15]
+; VGPRRC-NEXT:    v_mov_b64_e32 v[2:3], s[2:3]
 ; VGPRRC-NEXT:    s_nop 1
-; VGPRRC-NEXT:    v_mfma_f32_16x16x32_bf16 v[0:3], v[0:3], v[4:7], v[8:11] cbsz:3 abid:2 blgp:1
+; VGPRRC-NEXT:    v_mfma_f32_16x16x32_bf16 v[0:3], v[6:9], v[10:13], v[0:3] cbsz:3 abid:2 blgp:1
 ; VGPRRC-NEXT:    s_nop 7
-; VGPRRC-NEXT:    global_store_dwordx4 v12, v[0:3], s[6:7]
+; VGPRRC-NEXT:    global_store_dwordx4 v4, v[0:3], s[6:7]
 ; VGPRRC-NEXT:    s_endpgm
 ; AGPR-LABEL: test_mfma_f32_16x16x32_bf16_no_agpr__vgprcd__flags:
 ; AGPR:       ; %bb.0:
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.exp.ll b/llvm/test/CodeGen/AMDGPU/llvm.exp.ll
index 9ea8771506aa2..3897a0e028334 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.exp.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.exp.ll
@@ -339,53 +339,53 @@ define amdgpu_kernel void @s_exp_v2f32(ptr addrspace(1) %out, <2 x float> %in) {
 ; VI-SDAG-LABEL: s_exp_v2f32:
 ; VI-SDAG:       ; %bb.0:
 ; VI-SDAG-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
-; VI-SDAG-NEXT:    v_mov_b32_e32 v2, 0x3fb8a000
-; VI-SDAG-NEXT:    v_mov_b32_e32 v6, 0x42b17218
+; VI-SDAG-NEXT:    v_mov_b32_e32 v0, 0x3fb8a000
 ; VI-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-SDAG-NEXT:    s_and_b32 s4, s3, 0xfffff000
-; VI-SDAG-NEXT:    v_mov_b32_e32 v1, s4
-; VI-SDAG-NEXT:    v_sub_f32_e32 v1, s3, v1
-; VI-SDAG-NEXT:    v_mul_f32_e32 v4, 0x39a3b295, v1
-; VI-SDAG-NEXT:    v_mul_f32_e32 v1, 0x3fb8a000, v1
-; VI-SDAG-NEXT:    v_mul_f32_e32 v0, s4, v2
-; VI-SDAG-NEXT:    v_add_f32_e32 v1, v1, v4
+; VI-SDAG-NEXT:    v_mov_b32_e32 v2, s4
+; VI-SDAG-NEXT:    v_sub_f32_e32 v2, s3, v2
+; VI-SDAG-NEXT:    v_mul_f32_e32 v4, 0x39a3b295, v2
+; VI-SDAG-NEXT:    v_mul_f32_e32 v2, 0x3fb8a000, v2
+; VI-SDAG-NEXT:    v_add_f32_e32 v2, v2, v4
 ; VI-SDAG-NEXT:    v_mov_b32_e32 v4, 0x39a3b295
-; VI-SDAG-NEXT:    v_rndne_f32_e32 v3, v0
+; VI-SDAG-NEXT:    v_mul_f32_e32 v1, s4, v0
 ; VI-SDAG-NEXT:    v_mul_f32_e32 v5, s4, v4
-; VI-SDAG-NEXT:    v_sub_f32_e32 v0, v0, v3
-; VI-SDAG-NEXT:    v_add_f32_e32 v1, v5, v1
-; VI-SDAG-NEXT:    v_add_f32_e32 v0, v0, v1
-; VI-SDAG-NEXT:    v_exp_f32_e32 v5, v0
-; VI-SDAG-NEXT:    v_mov_b32_e32 v0, s0
-; VI-SDAG-NEXT:    s_and_b32 s0, s2, 0xfffff000
-; VI-SDAG-NEXT:    v_mov_b32_e32 v8, s0
-; VI-SDAG-NEXT:    v_sub_f32_e32 v8, s2, v8
-; VI-SDAG-NEXT:    v_mul_f32_e32 v2, s0, v2
-; VI-SDAG-NEXT:    v_mul_f32_e32 v9, 0x39a3b295, v8
-; VI-SDAG-NEXT:    v_mul_f32_e32 v8, 0x3fb8a000, v8
-; VI-SDAG-NEXT:    v_cvt_i32_f32_e32 v3, v3
-; VI-SDAG-NEXT:    v_rndne_f32_e32 v7, v2
-; VI-SDAG-NEXT:    v_add_f32_e32 v8, v8, v9
-; VI-SDAG-NEXT:    v_mul_f32_e32 v4, s0, v4
-; VI-SDAG-NEXT:    v_sub_f32_e32 v2, v2, v7
-; VI-SDAG-NEXT:    v_add_f32_e32 v4, v4, v8
-; VI-SDAG-NEXT:    v_add_f32_e32 v2, v2, v4
-; VI-SDAG-NEXT:    v_exp_f32_e32 v2, v2
-; VI-SDAG-NEXT:    v_cvt_i32_f32_e32 v4, v7
-; VI-SDAG-NEXT:    v_ldexp_f32 v3, v5, v3
-; VI-SDAG-NEXT:    v_mov_b32_e32 v5, 0xc2ce8ed0
-; VI-SDAG-NEXT:    v_cmp_nlt_f32_e32 vcc, s3, v5
-; VI-SDAG-NEXT:    v_cndmask_b32_e32 v3, 0, v3, vcc
-; VI-SDAG-NEXT:    v_mov_b32_e32 v7, 0x7f800000
-; VI-SDAG-NEXT:    v_cmp_ngt_f32_e32 vcc, s3, v6
-; VI-SDAG-NEXT:    v_cndmask_b32_e32 v3, v7, v3, vcc
-; VI-SDAG-NEXT:    v_ldexp_f32 v2, v2, v4
-; VI-SDAG-NEXT:    v_cmp_nlt_f32_e32 vcc, s2, v5
-; VI-SDAG-NEXT:    v_cndmask_b32_e32 v2, 0, v2, vcc
-; VI-SDAG-NEXT:    v_cmp_ngt_f32_e32 vcc, s2, v6
-; VI-SDAG-NEXT:    v_mov_b32_e32 v1, s1
-; VI-SDAG-NEXT:    v_cndmask_b32_e32 v2, v7, v2, vcc
-; VI-SDAG-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
+; VI-SDAG-NEXT:    s_and_b32 s4, s2, 0xfffff000
+; VI-SDAG-NEXT:    v_rndne_f32_e32 v3, v1
+; VI-SDAG-NEXT:    v_mov_b32_e32 v6, s4
+; VI-SDAG-NEXT:    v_sub_f32_e32 v1, v1, v3
+; VI-SDAG-NEXT:    v_add_f32_e32 v2, v5, v2
+; VI-SDAG-NEXT:    v_sub_f32_e32 v6, s2, v6
+; VI-SDAG-NEXT:    v_add_f32_e32 v1, v1, v2
+; VI-SDAG-NEXT:    v_mul_f32_e32 v0, s4, v0
+; VI-SDAG-NEXT:    v_mul_f32_e32 v7, 0x39a3b295, v6
+; VI-SDAG-NEXT:    v_mul_f32_e32 v6, 0x3fb8a000, v6
+; VI-SDAG-NEXT:    v_exp_f32_e32 v1, v1
+; VI-SDAG-NEXT:    v_cvt_i32_f32_e32 v2, v3
+; VI-SDAG-NEXT:    v_rndne_f32_e32 v5, v0
+; VI-SDAG-NEXT:    v_add_f32_e32 v6, v6, v7
+; VI-SDAG-NEXT:    v_mul_f32_e32 v4, s4, v4
+; VI-SDAG-NEXT:    v_sub_f32_e32 v0, v0, v5
+; VI-SDAG-NEXT:    v_add_f32_e32 v4, v4, v6
+; VI-SDAG-NEXT:    v_add_f32_e32 v0, v0, v4
+; VI-SDAG-NEXT:    v_exp_f32_e32 v0, v0
+; VI-SDAG-NEXT:    v_cvt_i32_f32_e32 v4, v5
+; VI-SDAG-NEXT:    v_ldexp_f32 v1, v1, v2
+; VI-SDAG-NEXT:    v_mov_b32_e32 v2, 0xc2ce8ed0
+; VI-SDAG-NEXT:    v_cmp_nlt_f32_e32 vcc, s3, v2
+; VI-SDAG-NEXT:    v_mov_b32_e32 v3, 0x42b17218
+; VI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; VI-SDAG-NEXT:    v_mov_b32_e32 v5, 0x7f800000
+; VI-SDAG-NEXT:    v_cmp_ngt_f32_e32 vcc, s3, v3
+; VI-SDAG-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
+; VI-SDAG-NEXT:    v_ldexp_f32 v0, v0, v4
+; VI-SDAG-NEXT:    v_cmp_nlt_f32_e32 vcc, s2, v2
+; VI-SDAG-NEXT:    v_cndmask_b32_e32 v0, 0, v0, vcc
+; VI-SDAG-NEXT:    v_cmp_ngt_f32_e32 vcc, s2, v3
+; VI-SDAG-NEXT:    v_mov_b32_e32 v3, s1
+; VI-SDAG-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc
+; VI-SDAG-NEXT:    v_mov_b32_e32 v2, s0
+; VI-SDAG-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
 ; VI-SDAG-NEXT:    s_endpgm
 ;
 ; VI-GISEL-LABEL: s_exp_v2f32:
@@ -520,42 +520,41 @@ define amdgpu_kernel void @s_exp_v2f32(ptr addrspace(1) %out, <2 x float> %in) {
 ;
 ; SI-SDAG-LABEL: s_exp_v2f32:
 ; SI-SDAG:       ; %bb.0:
-; SI-SDAG-NEXT:    s_load_dwordx4 s[4:7], s[4:5], 0x9
+; SI-SDAG-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
 ; SI-SDAG-NEXT:    v_mov_b32_e32 v0, 0x3fb8aa3b
 ; SI-SDAG-NEXT:    v_mov_b32_e32 v1, 0x32a5705f
-; SI-SDAG-NEXT:    s_mov_b32 s3, 0xf000
-; SI-SDAG-NEXT:    s_mov_b32 s2, -1
 ; SI-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
-; SI-SDAG-NEXT:    v_mul_f32_e32 v2, s7, v0
+; SI-SDAG-NEXT:    s_mov_b64 s[4:5], s[2:3]
+; SI-SDAG-NEXT:    v_mul_f32_e32 v2, s5, v0
 ; SI-SDAG-NEXT:    v_rndne_f32_e32 v3, v2
-; SI-SDAG-NEXT:    v_fma_f32 v4, s7, v0, -v2
+; SI-SDAG-NEXT:    v_fma_f32 v4, s5, v0, -v2
 ; SI-SDAG-NEXT:    v_sub_f32_e32 v2, v2, v3
-; SI-SDAG-NEXT:    v_fma_f32 v4, s7, v1, v4
+; SI-SDAG-NEXT:    v_fma_f32 v4, s5, v1, v4
 ; SI-SDAG-NEXT:    v_add_f32_e32 v2, v2, v4
-; SI-SDAG-NEXT:    v_mul_f32_e32 v5, s6, v0
-; SI-SDAG-NEXT:    v_exp_f32_e32 v2, v2
+; SI-SDAG-NEXT:    v_mul_f32_e32 v5, s4, v0
 ; SI-SDAG-NEXT:    v_cvt_i32_f32_e32 v3, v3
+; SI-SDAG-NEXT:    v_exp_f32_e32 v2, v2
 ; SI-SDAG-NEXT:    v_rndne_f32_e32 v6, v5
-; SI-SDAG-NEXT:    v_fma_f32 v0, s6, v0, -v5
+; SI-SDAG-NEXT:    v_fma_f32 v0, s4, v0, -v5
 ; SI-SDAG-NEXT:    v_sub_f32_e32 v7, v5, v6
-; SI-SDAG-NEXT:    v_fma_f32 v0, s6, v1, v0
+; SI-SDAG-NEXT:    v_fma_f32 v0, s4, v1, v0
 ; SI-SDAG-NEXT:    v_add_f32_e32 v0, v7, v0
 ; SI-SDAG-NEXT:    v_exp_f32_e32 v0, v0
 ; SI-SDAG-NEXT:    v_cvt_i32_f32_e32 v5, v6
 ; SI-SDAG-NEXT:    v_ldexp_f32_e32 v2, v2, v3
 ; SI-SDAG-NEXT:    v_mov_b32_e32 v3, 0xc2ce8ed0
-; SI-SDAG-NEXT:    v_cmp_nlt_f32_e32 vcc, s7, v3
+; SI-SDAG-NEXT:    v_cmp_nlt_f32_e32 vcc, s5, v3
 ; SI-SDAG-NEXT:    v_mov_b32_e32 v4, 0x42b17218
 ; SI-SDAG-NEXT:    v_cndmask_b32_e32 v2, 0, v2, vcc
 ; SI-SDAG-NEXT:    v_mov_b32_e32 v6, 0x7f800000
-; SI-SDAG-NEXT:    v_cmp_ngt_f32_e32 vcc, s7, v4
+; SI-SDAG-NEXT:    v_cmp_ngt_f32_e32 vcc, s5, v4
 ; SI-SDAG-NEXT:    v_cndmask_b32_e32 v1, v6, v2, vcc
 ; SI-SDAG-NEXT:    v_ldexp_f32_e32 v0, v0, v5
-; SI-SDAG-NEXT:    v_cmp_nlt_f32_e32 vcc, s6, v3
+; SI-SDAG-NEXT:    v_cmp_nlt_f32_e32 vcc, s4, v3
 ; SI-SDAG-NEXT:    v_cndmask_b32_e32 v0, 0, v0, vcc
-; SI-SDAG-NEXT:    v_cmp_ngt_f32_e32 vcc, s6, v4
-; SI-SDAG-NEXT:    s_mov_b32 s0, s4
-; SI-SDAG-NEXT:    s_mov_b32 s1, s5
+; SI-SDAG-NEXT:    v_cmp_ngt_f32_e32 vcc, s4, v4
+; SI-SDAG-NEXT:    s_mov_b32 s3, 0xf000
+; SI-SDAG-NEXT:    s_mov_b32 s2, -1
 ; SI-SDAG-NEXT:    v_cndmask_b32_e32 v0, v6, v0, vcc
 ; SI-SDAG-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
 ; SI-SDAG-NEXT:    s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.exp10.ll b/llvm/test/CodeGen/AMDGPU/llvm.exp10.ll
index 268e1e25f766f..3928ec2dd76d3 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.exp10.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.exp10.ll
@@ -341,53 +341,53 @@ define amdgpu_kernel void @s_exp10_v2f32(ptr addrspace(1) %out, <2 x float> %in)
 ; VI-SDAG-LABEL: s_exp10_v2f32:
 ; VI-SDAG:       ; %bb.0:
 ; VI-SDAG-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
-; VI-SDAG-NEXT:    v_mov_b32_e32 v2, 0x40549000
-; VI-SDAG-NEXT:    v_mov_b32_e32 v6, 0x421a209b
+; VI-SDAG-NEXT:    v_mov_b32_e32 v0, 0x40549000
 ; VI-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-SDAG-NEXT:    s_and_b32 s4, s3, 0xfffff000
-; VI-SDAG-NEXT:    v_mov_b32_e32 v1, s4
-; VI-SDAG-NEXT:    v_sub_f32_e32 v1, s3, v1
-; VI-SDAG-NEXT:    v_mul_f32_e32 v4, 0x3a2784bc, v1
-; VI-SDAG-NEXT:    v_mul_f32_e32 v1, 0x40549000, v1
-; VI-SDAG-NEXT:    v_mul_f32_e32 v0, s4, v2
-; VI-SDAG-NEXT:    v_add_f32_e32 v1, v1, v4
+; VI-SDAG-NEXT:    v_mov_b32_e32 v2, s4
+; VI-SDAG-NEXT:    v_sub_f32_e32 v2, s3, v2
+; VI-SDAG-NEXT:    v_mul_f32_e32 v4, 0x3a2784bc, v2
+; VI-SDAG-NEXT:    v_mul_f32_e32 v2, 0x40549000, v2
+; VI-SDAG-NEXT:    v_add_f32_e32 v2, v2, v4
 ; VI-SDAG-NEXT:    v_mov_b32_e32 v4, 0x3a2784bc
-; VI-SDAG-NEXT:    v_rndne_f32_e32 v3, v0
+; VI-SDAG-NEXT:    v_mul_f32_e32 v1, s4, v0
 ; VI-SDAG-NEXT:    v_mul_f32_e32 v5, s4, v4
-; VI-SDAG-NEXT:    v_sub_f32_e32 v0, v0, v3
-; VI-SDAG-NEXT:    v_add_f32_e32 v1, v5, v1
-; VI-SDAG-NEXT:    v_add_f32_e32 v0, v0, v1
-; VI-SDAG-NEXT:    v_exp_f32_e32 v5, v0
-; VI-SDAG-NEXT:    v_mov_b32_e32 v0, s0
-; VI-SDAG-NEXT:    s_and_b32 s0, s2, 0xfffff000
-; VI-SDAG-NEXT:    v_mov_b32_e32 v8, s0
-; VI-SDAG-NEXT:    v_sub_f32_e32 v8, s2, v8
-; VI-SDAG-NEXT:    v_mul_f32_e32 v2, s0, v2
-; VI-SDAG-NEXT:    v_mul_f32_e32 v9, 0x3a2784bc, v8
-; VI-SDAG-NEXT:    v_mul_f32_e32 v8, 0x40549000, v8
-; VI-SDAG-NEXT:    v_cvt_i32_f32_e32 v3, v3
-; VI-SDAG-NEXT:    v_rndne_f32_e32 v7, v2
-; VI-SDAG-NEXT:    v_add_f32_e32 v8, v8, v9
-; VI-SDAG-NEXT:    v_mul_f32_e32 v4, s0, v4
-; VI-SDAG-NEXT:    v_sub_f32_e32 v2, v2, v7
-; VI-SDAG-NEXT:    v_add_f32_e32 v4, v4, v8
-; VI-SDAG-NEXT:    v_add_f32_e32 v2, v2, v4
-; VI-SDAG-NEXT:    v_exp_f32_e32 v2, v2
-; VI-SDAG-NEXT:    v_cvt_i32_f32_e32 v4, v7
-; VI-SDAG-NEXT:    v_ldexp_f32 v3, v5, v3
-; VI-SDAG-NEXT:    v_mov_b32_e32 v5, 0xc23369f4
-; VI-SDAG-NEXT:    v_cmp_nlt_f32_e32 vcc, s3, v5
-; VI-SDAG-NEXT:    v_cndmask_b32_e32 v3, 0, v3, vcc
-; VI-SDAG-NEXT:    v_mov_b32_e32 v7, 0x7f800000
-; VI-SDAG-NEXT:    v_cmp_ngt_f32_e32 vcc, s3, v6
-; VI-SDAG-NEXT:    v_cndmask_b32_e32 v3, v7, v3, vcc
-; VI-SDAG-NEXT:    v_ldexp_f32 v2, v2, v4
-; VI-SDAG-NEXT:    v_cmp_nlt_f32_e32 vcc, s2, v5
-; VI-SDAG-NEXT:    v_cndmask_b32_e32 v2, 0, v2, vcc
-; VI-SDAG-NEXT:    v_cmp_ngt_f32_e32 vcc, s2, v6
-; VI-SDAG-NEXT:    v_mov_b32_e32 v1, s1
-; VI-SDAG-NEXT:    v_cndmask_b32_e32 v2, v7, v2, vcc
-; VI-SDAG-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
+; VI-SDAG-NEXT:    s_and_b32 s4, s2, 0xfffff000
+; VI-SDAG-NEXT:    v_rndne_f32_e32 v3, v1
+; VI-SDAG-NEXT:    v_mov_b32_e32 v6, s4
+; VI-SDAG-NEXT:    v_sub_f32_e32 v1, v1, v3
+; VI-SDAG-NEXT:    v_add_f32_e32 v2, v5, v2
+; VI-SDAG-NEXT:    v_sub_f32_e32 v6, s2, v6
+; VI-SDAG-NEXT:    v_add_f32_e32 v1, v1, v2
+; VI-SDAG-NEXT:    v_mul_f32_e32 v0, s4, v0
+; VI-SDAG-NEXT:    v_mul_f32_e32 v7, 0x3a2784bc, v6
+; VI-SDAG-NEXT:    v_mul_f32_e32 v6, 0x40549000, v6
+; VI-SDAG-NEXT:    v_exp_f32_e32 v1, v1
+; VI-SDAG-NEXT:    v_cvt_i32_f32_e32 v2, v3
+; VI-SDAG-NEXT:    v_rndne_f32_e32 v5, v0
+; VI-SDAG-NEXT:    v_add_f32_e32 v6, v6, v7
+; VI-SDAG-NEXT:    v_mul_f32_e32 v4, s4, v4
+; VI-SDAG-NEXT:    v_sub_f32_e32 v0, v0, v5
+; VI-SDAG-NEXT:    v_add_f32_e32 v4, v4, v6
+; VI-SDAG-NEXT:    v_add_f32_e32 v0, v0, v4
+; VI-SDAG-NEXT:    v_exp_f32_e32 v0, v0
+; VI-SDAG-NEXT:    v_cvt_i32_f32_e32 v4, v5
+; VI-SDAG-NEXT:    v_ldexp_f32 v1, v1, v2
+; VI-SDAG-NEXT:    v_mov_b32_e32 v2, 0xc23369f4
+; VI-SDAG-NEXT:    v_cmp_nlt_f32_e32 vcc, s3, v2
+; VI-SDAG-NEXT:    v_mov_b32_e32 v3, 0x421a209b
+; VI-SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; VI-SDAG-NEXT:    v_mov_b32_e32 v5, 0x7f800000
+; VI-SDAG-NEXT:    v_cmp_ngt_f32_e32 vcc, s3, v3
+; VI-SDAG-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
+; VI-SDAG-NEXT:    v_ldexp_f32 v0, v0, v4
+; VI-SDAG-NEXT:    v_cmp_nlt_f32_e32 vcc, s2, v2
+; VI-SDAG-NEXT:    v_cndmask_b32_e32 v0, 0, v0, vcc
+; VI-SDAG-NEXT:    v_cmp_ngt_f32_e32 vcc, s2, v3
+; VI-SDAG-NEXT:    v_mov_b32_e32 v3, s1
+; VI-SDAG-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc
+; VI-SDAG-NEXT:    v_mov_b32_e32 v2, s0
+; VI-SDAG-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
 ; VI-SDAG-NEXT:    s_endpgm
 ;
 ; VI-GISEL-LABEL: s_exp10_v2f32:
@@ -522,42 +522,41 @@ define amdgpu_kernel void @s_exp10_v2f32(ptr addrspace(1) %out, <2 x float> %in)
 ;
 ; SI-SDAG-LABEL: s_exp10_v2f32:
 ; SI-SDAG:       ; %bb.0:
-; SI-SDAG-NEXT:    s_load_dwordx4 s[4:7], s[4:5], 0x9
+; SI-SDAG-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
 ; SI-SDAG-NEXT:    v_mov_b32_e32 v0, 0x40549a78
 ; SI-SDAG-NEXT:    v_mov_b32_e32 v1, 0x33979a37
-; SI-SDAG-NEXT:    s_mov_b32 s3, 0xf000
-; SI-SDAG-NEXT:    s_mov_b32 s2, -1
 ; SI-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
-; SI-SDAG-NEXT:    v_mul_f32_e32 v2, s7, v0
+; SI-SDAG-NEXT:    s_mov_b64 s[4:5], s[2:3]
+; SI-SDAG-NEXT:    v_mul_f32_e32 v2, s5, v0
 ; SI-SDAG-NEXT:    v_rndne_f32_e32 v3, v2
-; SI-SDAG-NEXT:    v_fma_f32 v4, s7, v0, -v2
+; SI-SDAG-NEXT:    v_fma_f32 v4, s5, v0, -v2
 ; SI-SDAG-NEXT:    v_sub_f32_e32 v2, v2, v3
-; SI-SDAG-NEXT:    v_fma_f32 v4, s7, v1, v4
+; SI-SDAG-NEXT:    v_fma_f32 v4, s5, v1, v4
 ; SI-SDAG-NEXT:    v_add_f32_e32 v2, v2, v4
-; SI-SDAG-NEXT:    v_mul_f32_e32 v5, s6, v0
-; SI-SDAG-NEXT:    v_exp_f32_e32 v2, v2
+; SI-SDAG-NEXT:    v_mul_f32_e32 v5, s4, v0
 ; SI-SDAG-NEXT:    v_cvt_i32_f32_e32 v3, v3
+; SI-SDAG-NEXT:    v_exp_f32_e32 v2, v2
 ; SI-SDAG-NEXT:    v_rndne_f32_e32 v6, v5
-; SI-SDAG-NEXT:    v_fma_f32 v0, s6, v0, -v5
+; SI-SDAG-NEXT:    v_fma_f32 v0, s4, v0, -v5
 ; SI-SDAG-NEXT:    v_sub_f32_e32 v7, v5, v6
-; SI-SDAG-NEXT:    v_fma_f32 v0, s6, v1, v0
+; SI-SDAG-NEXT:    v_fma_f32 v0, s4, v1, v0
 ; SI-SDAG-NEXT:    v_add_f32_e32 v0, v7, v0
 ; SI-SDAG-NEXT:    v_exp_f32_e32 v0, v0
 ; SI-SDAG-NEXT:    v_cvt_i32_f32_e32 v5, v6
 ; SI-SDAG-NEXT:    v_ldexp_f32_e32 v2, v2, v3
 ; SI-SDAG-NEXT:    v_mov_b32_e32 v3, 0xc23369f4
-; SI-SDAG-NEXT:    v_cmp_nlt_f32_e32 vcc, s7, v3
+; SI-SDAG-NEXT:    v_cmp_nlt_f32_e32 vcc, s5, v3
 ; SI-SDAG-NEXT:    v_mov_b32_e32 v4, 0x421a209b
 ; SI-SDAG-NEXT:    v_cndmask_b32_e32 v2, 0, v2, vcc
 ; SI-SDAG-NEXT:    v_mov_b32_e32 v6, 0x7f800000
-; SI-SDAG-NEXT:    v_cmp_ngt_f32_e32 vcc, s7, v4
+; SI-SDAG-NEXT:    v_cmp_ngt_f32_e32 vcc, s5, v4
 ; SI-SDAG-NEXT:    v_cndmask_b32_e32 v1, v6, v2, vcc
 ; SI-SDAG-NEXT:    v_ldexp_f32_e32 v0, v0, v5
-; SI-SDAG-NEXT:    v_cmp_nlt_f32_e32 vcc, s6, v3
+; SI-SDAG-NEXT:    v_cmp_nlt_f32_e32 vcc, s4, v3
 ; SI-SDAG-NEXT:    v_cndmask_b32_e32 v0, 0, v0, vcc
-; SI-SDAG-NEXT:    v_cmp_ngt_f32_e32 vcc, s6, v4
-; SI-SDAG-NEXT:    s_mov_b32 s0, s4
-; SI-SDAG-NEXT:    s_mov_b32 s1, s5
+; SI-SDAG-NEXT:    v_cmp_ngt_f32_e32 vcc, s4, v4
+; SI-SDAG-NEXT:    s_mov_b32 s3, 0xf000
+; SI-SDAG-NEXT:    s_mov_b32 s2, -1
 ; SI-SDAG-NEXT:    v_cndmask_b32_e32 v0, v6, v0, vcc
 ; SI-SDAG-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
 ; SI-SDAG-NEXT:    s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.exp2.ll b/llvm/test/CodeGen/AMDGPU/llvm.exp2.ll
index c3f5146168033..dd44a1a35067e 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.exp2.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.exp2.ll
@@ -176,26 +176,25 @@ define amdgpu_kernel void @s_exp2_v2f32(ptr addrspace(1) %out, <2 x float> %in)
 ; SI-SDAG-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
 ; SI-SDAG-NEXT:    v_mov_b32_e32 v0, 0xc2fc0000
 ; SI-SDAG-NEXT:    v_mov_b32_e32 v1, 0x42800000
-; SI-SDAG-NEXT:    s_mov_b32 s7, 0xf000
-; SI-SDAG-NEXT:    s_mov_b32 s6, -1
 ; SI-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
-; SI-SDAG-NEXT:    v_cmp_lt_f32_e32 vcc, s3, v0
+; SI-SDAG-NEXT:    s_mov_b64 s[4:5], s[2:3]
+; SI-SDAG-NEXT:    v_cmp_lt_f32_e32 vcc, s5, v0
 ; SI-SDAG-NEXT:    v_cndmask_b32_e32 v2, 0, v1, vcc
-; SI-SDAG-NEXT:    s_mov_b32 s4, s0
-; SI-SDAG-NEXT:    s_mov_b32 s5, s1
-; SI-SDAG-NEXT:    s_and_b64 s[0:1], vcc, exec
-; SI-SDAG-NEXT:    v_add_f32_e32 v2, s3, v2
-; SI-SDAG-NEXT:    v_cmp_lt_f32_e32 vcc, s2, v0
-; SI-SDAG-NEXT:    v_exp_f32_e32 v2, v2
+; SI-SDAG-NEXT:    s_and_b64 s[6:7], vcc, exec
+; SI-SDAG-NEXT:    v_cmp_lt_f32_e32 vcc, s4, v0
 ; SI-SDAG-NEXT:    v_cndmask_b32_e32 v0, 0, v1, vcc
-; SI-SDAG-NEXT:    v_add_f32_e32 v0, s2, v0
+; SI-SDAG-NEXT:    v_add_f32_e32 v2, s5, v2
+; SI-SDAG-NEXT:    v_add_f32_e32 v0, s4, v0
+; SI-SDAG-NEXT:    v_exp_f32_e32 v2, v2
 ; SI-SDAG-NEXT:    v_exp_f32_e32 v0, v0
-; SI-SDAG-NEXT:    s_cselect_b32 s0, 0xffffffc0, 0
-; SI-SDAG-NEXT:    v_ldexp_f32_e64 v1, v2, s0
-; SI-SDAG-NEXT:    s_and_b64 s[0:1], vcc, exec
-; SI-SDAG-NEXT:    s_cselect_b32 s0, 0xffffffc0, 0
-; SI-SDAG-NEXT:    v_ldexp_f32_e64 v0, v0, s0
-; SI-SDAG-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; SI-SDAG-NEXT:    s_cselect_b32 s6, 0xffffffc0, 0
+; SI-SDAG-NEXT:    s_and_b64 s[4:5], vcc, exec
+; SI-SDAG-NEXT:    s_cselect_b32 s4, 0xffffffc0, 0
+; SI-SDAG-NEXT:    s_mov_b32 s3, 0xf000
+; SI-SDAG-NEXT:    s_mov_b32 s2, -1
+; SI-SDAG-NEXT:    v_ldexp_f32_e64 v1, v2, s6
+; SI-SDAG-NEXT:    v_ldexp_f32_e64 v0, v0, s4
+; SI-SDAG-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
 ; SI-SDAG-NEXT:    s_endpgm
 ;
 ; SI-GISEL-LABEL: s_exp2_v2f32:
@@ -225,26 +224,26 @@ define amdgpu_kernel void @s_exp2_v2f32(ptr addrspace(1) %out, <2 x float> %in)
 ; VI-SDAG-LABEL: s_exp2_v2f32:
 ; VI-SDAG:       ; %bb.0:
 ; VI-SDAG-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
-; VI-SDAG-NEXT:    v_mov_b32_e32 v2, 0xc2fc0000
-; VI-SDAG-NEXT:    v_mov_b32_e32 v3, 0x42800000
+; VI-SDAG-NEXT:    v_mov_b32_e32 v0, 0xc2fc0000
+; VI-SDAG-NEXT:    v_mov_b32_e32 v1, 0x42800000
 ; VI-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
-; VI-SDAG-NEXT:    v_cmp_lt_f32_e32 vcc, s3, v2
-; VI-SDAG-NEXT:    v_cndmask_b32_e32 v4, 0, v3, vcc
-; VI-SDAG-NEXT:    v_mov_b32_e32 v0, s0
-; VI-SDAG-NEXT:    v_mov_b32_e32 v1, s1
-; VI-SDAG-NEXT:    s_and_b64 s[0:1], vcc, exec
-; VI-SDAG-NEXT:    v_add_f32_e32 v4, s3, v4
-; VI-SDAG-NEXT:    v_cmp_lt_f32_e32 vcc, s2, v2
-; VI-SDAG-NEXT:    v_exp_f32_e32 v4, v4
-; VI-SDAG-NEXT:    v_cndmask_b32_e32 v2, 0, v3, vcc
-; VI-SDAG-NEXT:    v_add_f32_e32 v2, s2, v2
+; VI-SDAG-NEXT:    v_cmp_lt_f32_e32 vcc, s3, v0
+; VI-SDAG-NEXT:    v_cndmask_b32_e32 v2, 0, v1, vcc
+; VI-SDAG-NEXT:    s_and_b64 s[4:5], vcc, exec
+; VI-SDAG-NEXT:    v_add_f32_e32 v2, s3, v2
+; VI-SDAG-NEXT:    v_cmp_lt_f32_e32 vcc, s2, v0
 ; VI-SDAG-NEXT:    v_exp_f32_e32 v2, v2
-; VI-SDAG-NEXT:    s_cselect_b32 s0, 0xffffffc0, 0
-; VI-SDAG-NEXT:    v_ldexp_f32 v3, v4, s0
-; VI-SDAG-NEXT:    s_and_b64 s[0:1], vcc, exec
-; VI-SDAG-NEXT:    s_cselect_b32 s0, 0xffffffc0, 0
-; VI-SDAG-NEXT:    v_ldexp_f32 v2, v2, s0
-; VI-SDAG-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
+; VI-SDAG-NEXT:    v_cndmask_b32_e32 v0, 0, v1, vcc
+; VI-SDAG-NEXT:    v_add_f32_e32 v0, s2, v0
+; VI-SDAG-NEXT:    v_exp_f32_e32 v0, v0
+; VI-SDAG-NEXT:    s_cselect_b32 s3, 0xffffffc0, 0
+; VI-SDAG-NEXT:    v_ldexp_f32 v1, v2, s3
+; VI-SDAG-NEXT:    s_and_b64 s[2:3], vcc, exec
+; VI-SDAG-NEXT:    s_cselect_b32 s2, 0xffffffc0, 0
+; VI-SDAG-NEXT:    v_mov_b32_e32 v3, s1
+; VI-SDAG-NEXT:    v_ldexp_f32 v0, v0, s2
+; VI-SDAG-NEXT:    v_mov_b32_e32 v2, s0
+; VI-SDAG-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
 ; VI-SDAG-NEXT:    s_endpgm
 ;
 ; VI-GISEL-LABEL: s_exp2_v2f32:
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.log.ll b/llvm/test/CodeGen/AMDGPU/llvm.log.ll
index 74b6c75ac4948..7c06ae2f39f45 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.log.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.log.ll
@@ -321,39 +321,38 @@ define amdgpu_kernel void @s_log_f32(ptr addrspace(1) %out, float %in) {
 define amdgpu_kernel void @s_log_v2f32(ptr addrspace(1) %out, <2 x float> %in) {
 ; SI-SDAG-LABEL: s_log_v2f32:
 ; SI-SDAG:       ; %bb.0:
-; SI-SDAG-NEXT:    s_load_dwordx4 s[4:7], s[4:5], 0x9
+; SI-SDAG-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
 ; SI-SDAG-NEXT:    v_mov_b32_e32 v0, 0x800000
 ; SI-SDAG-NEXT:    v_mov_b32_e32 v1, 0x41b17218
-; SI-SDAG-NEXT:    s_mov_b32 s8, 0x3377d1cf
+; SI-SDAG-NEXT:    s_mov_b32 s8, 0x3f317217
 ; SI-SDAG-NEXT:    s_mov_b32 s9, 0x7f800000
 ; SI-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
-; SI-SDAG-NEXT:    v_cmp_lt_f32_e32 vcc, s7, v0
-; SI-SDAG-NEXT:    s_and_b64 s[0:1], vcc, exec
-; SI-SDAG-NEXT:    s_cselect_b32 s0, 32, 0
-; SI-SDAG-NEXT:    v_mov_b32_e32 v3, s0
-; SI-SDAG-NEXT:    v_ldexp_f32_e32 v3, s7, v3
+; SI-SDAG-NEXT:    s_mov_b64 s[4:5], s[2:3]
+; SI-SDAG-NEXT:    v_cmp_lt_f32_e32 vcc, s5, v0
+; SI-SDAG-NEXT:    s_and_b64 s[2:3], vcc, exec
+; SI-SDAG-NEXT:    s_cselect_b32 s2, 32, 0
+; SI-SDAG-NEXT:    v_mov_b32_e32 v3, s2
+; SI-SDAG-NEXT:    v_ldexp_f32_e32 v3, s5, v3
 ; SI-SDAG-NEXT:    v_log_f32_e32 v3, v3
 ; SI-SDAG-NEXT:    v_cndmask_b32_e32 v2, 0, v1, vcc
-; SI-SDAG-NEXT:    v_cmp_lt_f32_e32 vcc, s6, v0
-; SI-SDAG-NEXT:    s_mov_b32 s0, s4
-; SI-SDAG-NEXT:    s_mov_b32 s1, s5
-; SI-SDAG-NEXT:    s_and_b64 s[4:5], vcc, exec
-; SI-SDAG-NEXT:    s_mov_b32 s7, 0x3f317217
+; SI-SDAG-NEXT:    v_cmp_lt_f32_e32 vcc, s4, v0
+; SI-SDAG-NEXT:    s_and_b64 s[6:7], vcc, exec
 ; SI-SDAG-NEXT:    v_mul_f32_e32 v4, 0x3f317217, v3
-; SI-SDAG-NEXT:    s_cselect_b32 s4, 32, 0
-; SI-SDAG-NEXT:    v_fma_f32 v5, v3, s7, -v4
+; SI-SDAG-NEXT:    s_cselect_b32 s6, 32, 0
+; SI-SDAG-NEXT:    s_mov_b32 s5, 0x3377d1cf
+; SI-SDAG-NEXT:    v_fma_f32 v5, v3, s8, -v4
 ; SI-SDAG-NEXT:    v_cndmask_b32_e32 v0, 0, v1, vcc
-; SI-SDAG-NEXT:    v_mov_b32_e32 v1, s4
-; SI-SDAG-NEXT:    v_fma_f32 v5, v3, s8, v5
-; SI-SDAG-NEXT:    v_ldexp_f32_e32 v1, s6, v1
+; SI-SDAG-NEXT:    v_mov_b32_e32 v1, s6
+; SI-SDAG-NEXT:    v_fma_f32 v5, v3, s5, v5
+; SI-SDAG-NEXT:    v_ldexp_f32_e32 v1, s4, v1
 ; SI-SDAG-NEXT:    v_add_f32_e32 v4, v4, v5
 ; SI-SDAG-NEXT:    v_log_f32_e32 v5, v1
 ; SI-SDAG-NEXT:    v_cmp_lt_f32_e64 vcc, |v3|, s9
 ; SI-SDAG-NEXT:    v_cndmask_b32_e32 v1, v3, v4, vcc
 ; SI-SDAG-NEXT:    v_sub_f32_e32 v1, v1, v2
 ; SI-SDAG-NEXT:    v_mul_f32_e32 v2, 0x3f317217, v5
-; SI-SDAG-NEXT:    v_fma_f32 v3, v5, s7, -v2
-; SI-SDAG-NEXT:    v_fma_f32 v3, v5, s8, v3
+; SI-SDAG-NEXT:    v_fma_f32 v3, v5, s8, -v2
+; SI-SDAG-NEXT:    v_fma_f32 v3, v5, s5, v3
 ; SI-SDAG-NEXT:    v_add_f32_e32 v2, v2, v3
 ; SI-SDAG-NEXT:    v_cmp_lt_f32_e64 vcc, |v5|, s9
 ; SI-SDAG-NEXT:    v_cndmask_b32_e32 v2, v5, v2, vcc
@@ -406,51 +405,51 @@ define amdgpu_kernel void @s_log_v2f32(ptr addrspace(1) %out, <2 x float> %in) {
 ; VI-SDAG-LABEL: s_log_v2f32:
 ; VI-SDAG:       ; %bb.0:
 ; VI-SDAG-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
-; VI-SDAG-NEXT:    v_mov_b32_e32 v2, 0x800000
-; VI-SDAG-NEXT:    v_mov_b32_e32 v3, 0x41b17218
+; VI-SDAG-NEXT:    v_mov_b32_e32 v0, 0x800000
+; VI-SDAG-NEXT:    v_mov_b32_e32 v1, 0x41b17218
 ; VI-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
-; VI-SDAG-NEXT:    v_cmp_lt_f32_e32 vcc, s3, v2
+; VI-SDAG-NEXT:    v_cmp_lt_f32_e32 vcc, s3, v0
 ; VI-SDAG-NEXT:    s_and_b64 s[4:5], vcc, exec
 ; VI-SDAG-NEXT:    s_cselect_b32 s4, 32, 0
-; VI-SDAG-NEXT:    v_mov_b32_e32 v0, s4
-; VI-SDAG-NEXT:    v_ldexp_f32 v0, s3, v0
-; VI-SDAG-NEXT:    v_log_f32_e32 v5, v0
-; VI-SDAG-NEXT:    v_cndmask_b32_e32 v4, 0, v3, vcc
-; VI-SDAG-NEXT:    v_cmp_lt_f32_e32 vcc, s2, v2
-; VI-SDAG-NEXT:    v_mov_b32_e32 v0, s0
-; VI-SDAG-NEXT:    v_and_b32_e32 v6, 0xfffff000, v5
-; VI-SDAG-NEXT:    v_mov_b32_e32 v1, s1
-; VI-SDAG-NEXT:    v_sub_f32_e32 v7, v5, v6
-; VI-SDAG-NEXT:    s_and_b64 s[0:1], vcc, exec
-; VI-SDAG-NEXT:    v_mul_f32_e32 v8, 0x3805fdf4, v6
-; VI-SDAG-NEXT:    v_mul_f32_e32 v9, 0x3f317000, v7
-; VI-SDAG-NEXT:    v_mul_f32_e32 v7, 0x3805fdf4, v7
-; VI-SDAG-NEXT:    s_cselect_b32 s0, 32, 0
-; VI-SDAG-NEXT:    v_add_f32_e32 v7, v8, v7
-; VI-SDAG-NEXT:    v_cndmask_b32_e32 v2, 0, v3, vcc
-; VI-SDAG-NEXT:    v_mov_b32_e32 v3, s0
-; VI-SDAG-NEXT:    v_add_f32_e32 v7, v9, v7
-; VI-SDAG-NEXT:    v_mul_f32_e32 v6, 0x3f317000, v6
-; VI-SDAG-NEXT:    v_ldexp_f32 v3, s2, v3
-; VI-SDAG-NEXT:    v_add_f32_e32 v6, v6, v7
-; VI-SDAG-NEXT:    v_log_f32_e32 v7, v3
-; VI-SDAG-NEXT:    s_mov_b32 s3, 0x7f800000
-; VI-SDAG-NEXT:    v_cmp_lt_f32_e64 vcc, |v5|, s3
-; VI-SDAG-NEXT:    v_cndmask_b32_e32 v3, v5, v6, vcc
-; VI-SDAG-NEXT:    v_sub_f32_e32 v3, v3, v4
-; VI-SDAG-NEXT:    v_and_b32_e32 v4, 0xfffff000, v7
-; VI-SDAG-NEXT:    v_sub_f32_e32 v5, v7, v4
-; VI-SDAG-NEXT:    v_mul_f32_e32 v6, 0x3f317000, v5
+; VI-SDAG-NEXT:    v_mov_b32_e32 v3, s4
+; VI-SDAG-NEXT:    v_ldexp_f32 v3, s3, v3
+; VI-SDAG-NEXT:    v_log_f32_e32 v3, v3
+; VI-SDAG-NEXT:    v_cndmask_b32_e32 v2, 0, v1, vcc
+; VI-SDAG-NEXT:    v_cmp_lt_f32_e32 vcc, s2, v0
+; VI-SDAG-NEXT:    s_and_b64 s[4:5], vcc, exec
+; VI-SDAG-NEXT:    v_and_b32_e32 v4, 0xfffff000, v3
+; VI-SDAG-NEXT:    v_sub_f32_e32 v5, v3, v4
+; VI-SDAG-NEXT:    v_mul_f32_e32 v6, 0x3805fdf4, v4
+; VI-SDAG-NEXT:    v_mul_f32_e32 v7, 0x3f317000, v5
 ; VI-SDAG-NEXT:    v_mul_f32_e32 v5, 0x3805fdf4, v5
-; VI-SDAG-NEXT:    v_mul_f32_e32 v8, 0x3805fdf4, v4
-; VI-SDAG-NEXT:    v_add_f32_e32 v5, v8, v5
+; VI-SDAG-NEXT:    s_cselect_b32 s4, 32, 0
 ; VI-SDAG-NEXT:    v_add_f32_e32 v5, v6, v5
+; VI-SDAG-NEXT:    v_cndmask_b32_e32 v0, 0, v1, vcc
+; VI-SDAG-NEXT:    v_mov_b32_e32 v1, s4
 ; VI-SDAG-NEXT:    v_mul_f32_e32 v4, 0x3f317000, v4
+; VI-SDAG-NEXT:    v_add_f32_e32 v5, v7, v5
+; VI-SDAG-NEXT:    v_ldexp_f32 v1, s2, v1
 ; VI-SDAG-NEXT:    v_add_f32_e32 v4, v4, v5
-; VI-SDAG-NEXT:    v_cmp_lt_f32_e64 vcc, |v7|, s3
-; VI-SDAG-NEXT:    v_cndmask_b32_e32 v4, v7, v4, vcc
-; VI-SDAG-NEXT:    v_sub_f32_e32 v2, v4, v2
-; VI-SDAG-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
+; VI-SDAG-NEXT:    v_log_f32_e32 v5, v1
+; VI-SDAG-NEXT:    s_mov_b32 s3, 0x7f800000
+; VI-SDAG-NEXT:    v_cmp_lt_f32_e64 vcc, |v3|, s3
+; VI-SDAG-NEXT:    v_cndmask_b32_e32 v1, v3, v4, vcc
+; VI-SDAG-NEXT:    v_sub_f32_e32 v1, v1, v2
+; VI-SDAG-NEXT:    v_and_b32_e32 v2, 0xfffff000, v5
+; VI-SDAG-NEXT:    v_sub_f32_e32 v3, v5, v2
+; VI-SDAG-NEXT:    v_mul_f32_e32 v4, 0x3f317000, v3
+; VI-SDAG-NEXT:    v_mul_f32_e32 v3, 0x3805fdf4, v3
+; VI-SDAG-NEXT:    v_mul_f32_e32 v6, 0x3805fdf4, v2
+; VI-SDAG-NEXT:    v_add_f32_e32 v3, v6, v3
+; VI-SDAG-NEXT:    v_add_f32_e32 v3, v4, v3
+; VI-SDAG-NEXT:    v_mul_f32_e32 v2, 0x3f317000, v2
+; VI-SDAG-NEXT:    v_add_f32_e32 v2, v2, v3
+; VI-SDAG-NEXT:    v_cmp_lt_f32_e64 vcc, |v5|, s3
+; VI-SDAG-NEXT:    v_cndmask_b32_e32 v2, v5, v2, vcc
+; VI-SDAG-NEXT:    v_sub_f32_e32 v0, v2, v0
+; VI-SDAG-NEXT:    v_mov_b32_e32 v3, s1
+; VI-SDAG-NEXT:    v_mov_b32_e32 v2, s0
+; VI-SDAG-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
 ; VI-SDAG-NEXT:    s_endpgm
 ;
 ; VI-GISEL-LABEL: s_log_v2f32:
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.log10.ll b/llvm/test/CodeGen/AMDGPU/llvm.log10.ll
index c4fdac3ac5b0e..24e2fb4c8d9d7 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.log10.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.log10.ll
@@ -321,39 +321,38 @@ define amdgpu_kernel void @s_log10_f32(ptr addrspace(1) %out, float %in) {
 define amdgpu_kernel void @s_log10_v2f32(ptr addrspace(1) %out, <2 x float> %in) {
 ; SI-SDAG-LABEL: s_log10_v2f32:
 ; SI-SDAG:       ; %bb.0:
-; SI-SDAG-NEXT:    s_load_dwordx4 s[4:7], s[4:5], 0x9
+; SI-SDAG-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
 ; SI-SDAG-NEXT:    v_mov_b32_e32 v0, 0x800000
 ; SI-SDAG-NEXT:    v_mov_b32_e32 v1, 0x411a209b
-; SI-SDAG-NEXT:    s_mov_b32 s8, 0x3284fbcf
+; SI-SDAG-NEXT:    s_mov_b32 s8, 0x3e9a209a
 ; SI-SDAG-NEXT:    s_mov_b32 s9, 0x7f800000
 ; SI-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
-; SI-SDAG-NEXT:    v_cmp_lt_f32_e32 vcc, s7, v0
-; SI-SDAG-NEXT:    s_and_b64 s[0:1], vcc, exec
-; SI-SDAG-NEXT:    s_cselect_b32 s0, 32, 0
-; SI-SDAG-NEXT:    v_mov_b32_e32 v3, s0
-; SI-SDAG-NEXT:    v_ldexp_f32_e32 v3, s7, v3
+; SI-SDAG-NEXT:    s_mov_b64 s[4:5], s[2:3]
+; SI-SDAG-NEXT:    v_cmp_lt_f32_e32 vcc, s5, v0
+; SI-SDAG-NEXT:    s_and_b64 s[2:3], vcc, exec
+; SI-SDAG-NEXT:    s_cselect_b32 s2, 32, 0
+; SI-SDAG-NEXT:    v_mov_b32_e32 v3, s2
+; SI-SDAG-NEXT:    v_ldexp_f32_e32 v3, s5, v3
 ; SI-SDAG-NEXT:    v_log_f32_e32 v3, v3
 ; SI-SDAG-NEXT:    v_cndmask_b32_e32 v2, 0, v1, vcc
-; SI-SDAG-NEXT:    v_cmp_lt_f32_e32 vcc, s6, v0
-; SI-SDAG-NEXT:    s_mov_b32 s0, s4
-; SI-SDAG-NEXT:    s_mov_b32 s1, s5
-; SI-SDAG-NEXT:    s_and_b64 s[4:5], vcc, exec
-; SI-SDAG-NEXT:    s_mov_b32 s7, 0x3e9a209a
+; SI-SDAG-NEXT:    v_cmp_lt_f32_e32 vcc, s4, v0
+; SI-SDAG-NEXT:    s_and_b64 s[6:7], vcc, exec
 ; SI-SDAG-NEXT:    v_mul_f32_e32 v4, 0x3e9a209a, v3
-; SI-SDAG-NEXT:    s_cselect_b32 s4, 32, 0
-; SI-SDAG-NEXT:    v_fma_f32 v5, v3, s7, -v4
+; SI-SDAG-NEXT:    s_cselect_b32 s6, 32, 0
+; SI-SDAG-NEXT:    s_mov_b32 s5, 0x3284fbcf
+; SI-SDAG-NEXT:    v_fma_f32 v5, v3, s8, -v4
 ; SI-SDAG-NEXT:    v_cndmask_b32_e32 v0, 0, v1, vcc
-; SI-SDAG-NEXT:    v_mov_b32_e32 v1, s4
-; SI-SDAG-NEXT:    v_fma_f32 v5, v3, s8, v5
-; SI-SDAG-NEXT:    v_ldexp_f32_e32 v1, s6, v1
+; SI-SDAG-NEXT:    v_mov_b32_e32 v1, s6
+; SI-SDAG-NEXT:    v_fma_f32 v5, v3, s5, v5
+; SI-SDAG-NEXT:    v_ldexp_f32_e32 v1, s4, v1
 ; SI-SDAG-NEXT:    v_add_f32_e32 v4, v4, v5
 ; SI-SDAG-NEXT:    v_log_f32_e32 v5, v1
 ; SI-SDAG-NEXT:    v_cmp_lt_f32_e64 vcc, |v3|, s9
 ; SI-SDAG-NEXT:    v_cndmask_b32_e32 v1, v3, v4, vcc
 ; SI-SDAG-NEXT:    v_sub_f32_e32 v1, v1, v2
 ; SI-SDAG-NEXT:    v_mul_f32_e32 v2, 0x3e9a209a, v5
-; SI-SDAG-NEXT:    v_fma_f32 v3, v5, s7, -v2
-; SI-SDAG-NEXT:    v_fma_f32 v3, v5, s8, v3
+; SI-SDAG-NEXT:    v_fma_f32 v3, v5, s8, -v2
+; SI-SDAG-NEXT:    v_fma_f32 v3, v5, s5, v3
 ; SI-SDAG-NEXT:    v_add_f32_e32 v2, v2, v3
 ; SI-SDAG-NEXT:    v_cmp_lt_f32_e64 vcc, |v5|, s9
 ; SI-SDAG-NEXT:    v_cndmask_b32_e32 v2, v5, v2, vcc
@@ -406,51 +405,51 @@ define amdgpu_kernel void @s_log10_v2f32(ptr addrspace(1) %out, <2 x float> %in)
 ; VI-SDAG-LABEL: s_log10_v2f32:
 ; VI-SDAG:       ; %bb.0:
 ; VI-SDAG-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
-; VI-SDAG-NEXT:    v_mov_b32_e32 v2, 0x800000
-; VI-SDAG-NEXT:    v_mov_b32_e32 v3, 0x411a209b
+; VI-SDAG-NEXT:    v_mov_b32_e32 v0, 0x800000
+; VI-SDAG-NEXT:    v_mov_b32_e32 v1, 0x411a209b
 ; VI-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
-; VI-SDAG-NEXT:    v_cmp_lt_f32_e32 vcc, s3, v2
+; VI-SDAG-NEXT:    v_cmp_lt_f32_e32 vcc, s3, v0
 ; VI-SDAG-NEXT:    s_and_b64 s[4:5], vcc, exec
 ; VI-SDAG-NEXT:    s_cselect_b32 s4, 32, 0
-; VI-SDAG-NEXT:    v_mov_b32_e32 v0, s4
-; VI-SDAG-NEXT:    v_ldexp_f32 v0, s3, v0
-; VI-SDAG-NEXT:    v_log_f32_e32 v5, v0
-; VI-SDAG-NEXT:    v_cndmask_b32_e32 v4, 0, v3, vcc
-; VI-SDAG-NEXT:    v_cmp_lt_f32_e32 vcc, s2, v2
-; VI-SDAG-NEXT:    v_mov_b32_e32 v0, s0
-; VI-SDAG-NEXT:    v_and_b32_e32 v6, 0xfffff000, v5
-; VI-SDAG-NEXT:    v_mov_b32_e32 v1, s1
-; VI-SDAG-NEXT:    v_sub_f32_e32 v7, v5, v6
-; VI-SDAG-NEXT:    s_and_b64 s[0:1], vcc, exec
-; VI-SDAG-NEXT:    v_mul_f32_e32 v8, 0x369a84fb, v6
-; VI-SDAG-NEXT:    v_mul_f32_e32 v9, 0x3e9a2000, v7
-; VI-SDAG-NEXT:    v_mul_f32_e32 v7, 0x369a84fb, v7
-; VI-SDAG-NEXT:    s_cselect_b32 s0, 32, 0
-; VI-SDAG-NEXT:    v_add_f32_e32 v7, v8, v7
-; VI-SDAG-NEXT:    v_cndmask_b32_e32 v2, 0, v3, vcc
-; VI-SDAG-NEXT:    v_mov_b32_e32 v3, s0
-; VI-SDAG-NEXT:    v_add_f32_e32 v7, v9, v7
-; VI-SDAG-NEXT:    v_mul_f32_e32 v6, 0x3e9a2000, v6
-; VI-SDAG-NEXT:    v_ldexp_f32 v3, s2, v3
-; VI-SDAG-NEXT:    v_add_f32_e32 v6, v6, v7
-; VI-SDAG-NEXT:    v_log_f32_e32 v7, v3
-; VI-SDAG-NEXT:    s_mov_b32 s3, 0x7f800000
-; VI-SDAG-NEXT:    v_cmp_lt_f32_e64 vcc, |v5|, s3
-; VI-SDAG-NEXT:    v_cndmask_b32_e32 v3, v5, v6, vcc
-; VI-SDAG-NEXT:    v_sub_f32_e32 v3, v3, v4
-; VI-SDAG-NEXT:    v_and_b32_e32 v4, 0xfffff000, v7
-; VI-SDAG-NEXT:    v_sub_f32_e32 v5, v7, v4
-; VI-SDAG-NEXT:    v_mul_f32_e32 v6, 0x3e9a2000, v5
+; VI-SDAG-NEXT:    v_mov_b32_e32 v3, s4
+; VI-SDAG-NEXT:    v_ldexp_f32 v3, s3, v3
+; VI-SDAG-NEXT:    v_log_f32_e32 v3, v3
+; VI-SDAG-NEXT:    v_cndmask_b32_e32 v2, 0, v1, vcc
+; VI-SDAG-NEXT:    v_cmp_lt_f32_e32 vcc, s2, v0
+; VI-SDAG-NEXT:    s_and_b64 s[4:5], vcc, exec
+; VI-SDAG-NEXT:    v_and_b32_e32 v4, 0xfffff000, v3
+; VI-SDAG-NEXT:    v_sub_f32_e32 v5, v3, v4
+; VI-SDAG-NEXT:    v_mul_f32_e32 v6, 0x369a84fb, v4
+; VI-SDAG-NEXT:    v_mul_f32_e32 v7, 0x3e9a2000, v5
 ; VI-SDAG-NEXT:    v_mul_f32_e32 v5, 0x369a84fb, v5
-; VI-SDAG-NEXT:    v_mul_f32_e32 v8, 0x369a84fb, v4
-; VI-SDAG-NEXT:    v_add_f32_e32 v5, v8, v5
+; VI-SDAG-NEXT:    s_cselect_b32 s4, 32, 0
 ; VI-SDAG-NEXT:    v_add_f32_e32 v5, v6, v5
+; VI-SDAG-NEXT:    v_cndmask_b32_e32 v0, 0, v1, vcc
+; VI-SDAG-NEXT:    v_mov_b32_e32 v1, s4
 ; VI-SDAG-NEXT:    v_mul_f32_e32 v4, 0x3e9a2000, v4
+; VI-SDAG-NEXT:    v_add_f32_e32 v5, v7, v5
+; VI-SDAG-NEXT:    v_ldexp_f32 v1, s2, v1
 ; VI-SDAG-NEXT:    v_add_f32_e32 v4, v4, v5
-; VI-SDAG-NEXT:    v_cmp_lt_f32_e64 vcc, |v7|, s3
-; VI-SDAG-NEXT:    v_cndmask_b32_e32 v4, v7, v4, vcc
-; VI-SDAG-NEXT:    v_sub_f32_e32 v2, v4, v2
-; VI-SDAG-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
+; VI-SDAG-NEXT:    v_log_f32_e32 v5, v1
+; VI-SDAG-NEXT:    s_mov_b32 s3, 0x7f800000
+; VI-SDAG-NEXT:    v_cmp_lt_f32_e64 vcc, |v3|, s3
+; VI-SDAG-NEXT:    v_cndmask_b32_e32 v1, v3, v4, vcc
+; VI-SDAG-NEXT:    v_sub_f32_e32 v1, v1, v2
+; VI-SDAG-NEXT:    v_and_b32_e32 v2, 0xfffff000, v5
+; VI-SDAG-NEXT:    v_sub_f32_e32 v3, v5, v2
+; VI-SDAG-NEXT:    v_mul_f32_e32 v4, 0x3e9a2000, v3
+; VI-SDAG-NEXT:    v_mul_f32_e32 v3, 0x369a84fb, v3
+; VI-SDAG-NEXT:    v_mul_f32_e32 v6, 0x369a84fb, v2
+; VI-SDAG-NEXT:    v_add_f32_e32 v3, v6, v3
+; VI-SDAG-NEXT:    v_add_f32_e32 v3, v4, v3
+; VI-SDAG-NEXT:    v_mul_f32_e32 v2, 0x3e9a2000, v2
+; VI-SDAG-NEXT:    v_add_f32_e32 v2, v2, v3
+; VI-SDAG-NEXT:    v_cmp_lt_f32_e64 vcc, |v5|, s3
+; VI-SDAG-NEXT:    v_cndmask_b32_e32 v2, v5, v2, vcc
+; VI-SDAG-NEXT:    v_sub_f32_e32 v0, v2, v0
+; VI-SDAG-NEXT:    v_mov_b32_e32 v3, s1
+; VI-SDAG-NEXT:    v_mov_b32_e32 v2, s0
+; VI-SDAG-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
 ; VI-SDAG-NEXT:    s_endpgm
 ;
 ; VI-GISEL-LABEL: s_log10_v2f32:
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.log2.ll b/llvm/test/CodeGen/AMDGPU/llvm.log2.ll
index 444f37059406a..e24fd1f22bfa6 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.log2.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.log2.ll
@@ -221,8 +221,6 @@ define amdgpu_kernel void @s_log2_v2f32(ptr addrspace(1) %out, <2 x float> %in)
 ; SI-SDAG-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
 ; SI-SDAG-NEXT:    v_mov_b32_e32 v0, 0x800000
 ; SI-SDAG-NEXT:    v_mov_b32_e32 v1, 0x42000000
-; SI-SDAG-NEXT:    s_mov_b32 s7, 0xf000
-; SI-SDAG-NEXT:    s_mov_b32 s6, -1
 ; SI-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
 ; SI-SDAG-NEXT:    v_cmp_lt_f32_e32 vcc, s3, v0
 ; SI-SDAG-NEXT:    s_and_b64 s[4:5], vcc, exec
@@ -238,11 +236,11 @@ define amdgpu_kernel void @s_log2_v2f32(ptr addrspace(1) %out, <2 x float> %in)
 ; SI-SDAG-NEXT:    v_ldexp_f32_e32 v1, s2, v1
 ; SI-SDAG-NEXT:    v_log_f32_e32 v3, v3
 ; SI-SDAG-NEXT:    v_log_f32_e32 v4, v1
-; SI-SDAG-NEXT:    s_mov_b32 s4, s0
-; SI-SDAG-NEXT:    s_mov_b32 s5, s1
+; SI-SDAG-NEXT:    s_mov_b32 s3, 0xf000
+; SI-SDAG-NEXT:    s_mov_b32 s2, -1
 ; SI-SDAG-NEXT:    v_sub_f32_e32 v1, v3, v2
 ; SI-SDAG-NEXT:    v_sub_f32_e32 v0, v4, v0
-; SI-SDAG-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; SI-SDAG-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
 ; SI-SDAG-NEXT:    s_endpgm
 ;
 ; SI-GISEL-LABEL: s_log2_v2f32:
@@ -285,16 +283,16 @@ define amdgpu_kernel void @s_log2_v2f32(ptr addrspace(1) %out, <2 x float> %in)
 ; VI-SDAG-NEXT:    s_and_b64 s[4:5], vcc, exec
 ; VI-SDAG-NEXT:    v_ldexp_f32 v3, s3, v3
 ; VI-SDAG-NEXT:    s_cselect_b32 s3, 32, 0
-; VI-SDAG-NEXT:    v_mov_b32_e32 v0, s3
-; VI-SDAG-NEXT:    v_ldexp_f32 v0, s2, v0
+; VI-SDAG-NEXT:    v_cndmask_b32_e32 v0, 0, v1, vcc
+; VI-SDAG-NEXT:    v_mov_b32_e32 v1, s3
 ; VI-SDAG-NEXT:    v_log_f32_e32 v3, v3
-; VI-SDAG-NEXT:    v_log_f32_e32 v5, v0
-; VI-SDAG-NEXT:    v_cndmask_b32_e32 v4, 0, v1, vcc
-; VI-SDAG-NEXT:    v_mov_b32_e32 v0, s0
-; VI-SDAG-NEXT:    v_mov_b32_e32 v1, s1
-; VI-SDAG-NEXT:    v_sub_f32_e32 v3, v3, v2
-; VI-SDAG-NEXT:    v_sub_f32_e32 v2, v5, v4
-; VI-SDAG-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
+; VI-SDAG-NEXT:    v_ldexp_f32 v1, s2, v1
+; VI-SDAG-NEXT:    v_log_f32_e32 v4, v1
+; VI-SDAG-NEXT:    v_sub_f32_e32 v1, v3, v2
+; VI-SDAG-NEXT:    v_mov_b32_e32 v3, s1
+; VI-SDAG-NEXT:    v_sub_f32_e32 v0, v4, v0
+; VI-SDAG-NEXT:    v_mov_b32_e32 v2, s0
+; VI-SDAG-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
 ; VI-SDAG-NEXT:    s_endpgm
 ;
 ; VI-GISEL-LABEL: s_log2_v2f32:
diff --git a/llvm/test/CodeGen/AMDGPU/max.ll b/llvm/test/CodeGen/AMDGPU/max.ll
index 32a644ed334cc..ac6dd30283554 100644
--- a/llvm/test/CodeGen/AMDGPU/max.ll
+++ b/llvm/test/CodeGen/AMDGPU/max.ll
@@ -362,16 +362,15 @@ define amdgpu_kernel void @s_test_imax_sgt_imm_v2i32(ptr addrspace(1) %out, <2 x
 ; SI-LABEL: s_test_imax_sgt_imm_v2i32:
 ; SI:       ; %bb.0:
 ; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
-; SI-NEXT:    s_mov_b32 s7, 0xf000
-; SI-NEXT:    s_mov_b32 s6, -1
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
-; SI-NEXT:    s_mov_b32 s4, s0
-; SI-NEXT:    s_mov_b32 s5, s1
-; SI-NEXT:    s_max_i32 s0, s3, 9
-; SI-NEXT:    s_max_i32 s1, s2, 9
-; SI-NEXT:    v_mov_b32_e32 v0, s1
-; SI-NEXT:    v_mov_b32_e32 v1, s0
-; SI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; SI-NEXT:    s_mov_b64 s[4:5], s[2:3]
+; SI-NEXT:    s_max_i32 s5, s5, 9
+; SI-NEXT:    s_max_i32 s4, s4, 9
+; SI-NEXT:    s_mov_b32 s3, 0xf000
+; SI-NEXT:    s_mov_b32 s2, -1
+; SI-NEXT:    v_mov_b32_e32 v0, s4
+; SI-NEXT:    v_mov_b32_e32 v1, s5
+; SI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
 ; SI-NEXT:    s_endpgm
 ;
 ; GFX1250-LABEL: s_test_imax_sgt_imm_v2i32:
@@ -869,16 +868,15 @@ define amdgpu_kernel void @s_test_umax_ugt_imm_v2i32(ptr addrspace(1) %out, <2 x
 ; SI-LABEL: s_test_umax_ugt_imm_v2i32:
 ; SI:       ; %bb.0:
 ; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
-; SI-NEXT:    s_mov_b32 s7, 0xf000
-; SI-NEXT:    s_mov_b32 s6, -1
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
-; SI-NEXT:    s_mov_b32 s4, s0
-; SI-NEXT:    s_mov_b32 s5, s1
-; SI-NEXT:    s_max_u32 s0, s3, 23
-; SI-NEXT:    s_max_u32 s1, s2, 15
-; SI-NEXT:    v_mov_b32_e32 v0, s1
-; SI-NEXT:    v_mov_b32_e32 v1, s0
-; SI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; SI-NEXT:    s_mov_b64 s[4:5], s[2:3]
+; SI-NEXT:    s_max_u32 s5, s5, 23
+; SI-NEXT:    s_max_u32 s4, s4, 15
+; SI-NEXT:    s_mov_b32 s3, 0xf000
+; SI-NEXT:    s_mov_b32 s2, -1
+; SI-NEXT:    v_mov_b32_e32 v0, s4
+; SI-NEXT:    v_mov_b32_e32 v1, s5
+; SI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
 ; SI-NEXT:    s_endpgm
 ;
 ; GFX1250-LABEL: s_test_umax_ugt_imm_v2i32:
diff --git a/llvm/test/CodeGen/AMDGPU/min.ll b/llvm/test/CodeGen/AMDGPU/min.ll
index c571cfc3648e2..eff0680fe9a31 100644
--- a/llvm/test/CodeGen/AMDGPU/min.ll
+++ b/llvm/test/CodeGen/AMDGPU/min.ll
@@ -1074,118 +1074,118 @@ define amdgpu_kernel void @s_test_imin_sle_v4i16(ptr addrspace(1) %out, <4 x i16
 ;
 ; CI-LABEL: s_test_imin_sle_v4i16:
 ; CI:       ; %bb.0:
-; CI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
-; CI-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x4
+; CI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x2
+; CI-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; CI-NEXT:    s_add_i32 s12, s12, s17
 ; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
-; CI-NEXT:    v_mov_b32_e32 v0, s0
-; CI-NEXT:    v_mov_b32_e32 v1, s1
-; CI-NEXT:    s_ashr_i32 s0, s2, 16
-; CI-NEXT:    s_ashr_i32 s1, s3, 16
+; CI-NEXT:    s_ashr_i32 s6, s0, 16
+; CI-NEXT:    s_ashr_i32 s7, s1, 16
+; CI-NEXT:    s_sext_i32_i16 s0, s0
+; CI-NEXT:    s_sext_i32_i16 s1, s1
+; CI-NEXT:    s_ashr_i32 s8, s2, 16
+; CI-NEXT:    s_ashr_i32 s9, s3, 16
 ; CI-NEXT:    s_sext_i32_i16 s2, s2
 ; CI-NEXT:    s_sext_i32_i16 s3, s3
-; CI-NEXT:    s_ashr_i32 s6, s4, 16
-; CI-NEXT:    s_ashr_i32 s7, s5, 16
-; CI-NEXT:    s_sext_i32_i16 s4, s4
-; CI-NEXT:    s_sext_i32_i16 s5, s5
-; CI-NEXT:    s_min_i32 s1, s1, s7
-; CI-NEXT:    s_min_i32 s3, s3, s5
-; CI-NEXT:    s_min_i32 s0, s0, s6
-; CI-NEXT:    s_min_i32 s2, s2, s4
-; CI-NEXT:    s_lshl_b32 s1, s1, 16
-; CI-NEXT:    s_and_b32 s3, s3, 0xffff
-; CI-NEXT:    s_lshl_b32 s0, s0, 16
-; CI-NEXT:    s_and_b32 s2, s2, 0xffff
-; CI-NEXT:    s_or_b32 s1, s3, s1
-; CI-NEXT:    s_or_b32 s0, s2, s0
-; CI-NEXT:    v_mov_b32_e32 v2, s0
-; CI-NEXT:    v_mov_b32_e32 v3, s1
-; CI-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
+; CI-NEXT:    s_min_i32 s7, s7, s9
+; CI-NEXT:    s_min_i32 s1, s1, s3
+; CI-NEXT:    s_min_i32 s3, s6, s8
+; CI-NEXT:    s_min_i32 s0, s0, s2
+; CI-NEXT:    s_lshl_b32 s7, s7, 16
+; CI-NEXT:    s_and_b32 s1, s1, 0xffff
+; CI-NEXT:    s_lshl_b32 s3, s3, 16
+; CI-NEXT:    s_and_b32 s0, s0, 0xffff
+; CI-NEXT:    s_or_b32 s1, s1, s7
+; CI-NEXT:    s_or_b32 s0, s0, s3
+; CI-NEXT:    v_mov_b32_e32 v2, s4
+; CI-NEXT:    v_mov_b32_e32 v0, s0
+; CI-NEXT:    v_mov_b32_e32 v1, s1
+; CI-NEXT:    v_mov_b32_e32 v3, s5
+; CI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
 ; CI-NEXT:    s_endpgm
 ;
 ; VI-LABEL: s_test_imin_sle_v4i16:
 ; VI:       ; %bb.0:
-; VI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
-; VI-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x10
+; VI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x8
+; VI-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; VI-NEXT:    s_add_i32 s12, s12, s17
 ; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
-; VI-NEXT:    v_mov_b32_e32 v0, s0
-; VI-NEXT:    v_mov_b32_e32 v1, s1
-; VI-NEXT:    s_ashr_i32 s0, s5, 16
-; VI-NEXT:    s_ashr_i32 s1, s3, 16
-; VI-NEXT:    s_min_i32 s0, s1, s0
-; VI-NEXT:    s_sext_i32_i16 s1, s5
+; VI-NEXT:    s_ashr_i32 s6, s3, 16
+; VI-NEXT:    s_ashr_i32 s7, s1, 16
 ; VI-NEXT:    s_sext_i32_i16 s3, s3
-; VI-NEXT:    s_min_i32 s1, s3, s1
-; VI-NEXT:    s_lshl_b32 s0, s0, 16
+; VI-NEXT:    s_sext_i32_i16 s1, s1
+; VI-NEXT:    s_min_i32 s6, s7, s6
+; VI-NEXT:    s_min_i32 s1, s1, s3
+; VI-NEXT:    s_lshl_b32 s6, s6, 16
 ; VI-NEXT:    s_and_b32 s1, s1, 0xffff
-; VI-NEXT:    s_or_b32 s0, s1, s0
-; VI-NEXT:    s_ashr_i32 s1, s4, 16
+; VI-NEXT:    s_or_b32 s1, s1, s6
 ; VI-NEXT:    s_ashr_i32 s3, s2, 16
-; VI-NEXT:    s_min_i32 s1, s3, s1
-; VI-NEXT:    s_sext_i32_i16 s3, s4
+; VI-NEXT:    s_ashr_i32 s6, s0, 16
 ; VI-NEXT:    s_sext_i32_i16 s2, s2
-; VI-NEXT:    s_min_i32 s2, s2, s3
-; VI-NEXT:    s_lshl_b32 s1, s1, 16
-; VI-NEXT:    s_and_b32 s2, s2, 0xffff
-; VI-NEXT:    s_or_b32 s1, s2, s1
-; VI-NEXT:    v_mov_b32_e32 v2, s1
-; VI-NEXT:    v_mov_b32_e32 v3, s0
-; VI-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
+; VI-NEXT:    s_sext_i32_i16 s0, s0
+; VI-NEXT:    s_min_i32 s3, s6, s3
+; VI-NEXT:    s_min_i32 s0, s0, s2
+; VI-NEXT:    s_lshl_b32 s3, s3, 16
+; VI-NEXT:    s_and_b32 s0, s0, 0xffff
+; VI-NEXT:    s_or_b32 s0, s0, s3
+; VI-NEXT:    v_mov_b32_e32 v2, s4
+; VI-NEXT:    v_mov_b32_e32 v0, s0
+; VI-NEXT:    v_mov_b32_e32 v1, s1
+; VI-NEXT:    v_mov_b32_e32 v3, s5
+; VI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
 ; VI-NEXT:    s_endpgm
 ;
 ; GFX9-LABEL: s_test_imin_sle_v4i16:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x10
-; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x8
+; GFX9-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX9-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    v_mov_b32_e32 v0, s5
-; GFX9-NEXT:    v_mov_b32_e32 v3, s4
-; GFX9-NEXT:    v_pk_min_i16 v1, s3, v0
-; GFX9-NEXT:    v_pk_min_i16 v0, s2, v3
-; GFX9-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX9-NEXT:    v_mov_b32_e32 v0, s3
+; GFX9-NEXT:    v_mov_b32_e32 v3, s2
+; GFX9-NEXT:    v_pk_min_i16 v1, s1, v0
+; GFX9-NEXT:    v_pk_min_i16 v0, s0, v3
+; GFX9-NEXT:    global_store_dwordx2 v2, v[0:1], s[4:5]
 ; GFX9-NEXT:    s_endpgm
 ;
 ; GFX10-LABEL: s_test_imin_sle_v4i16:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_clause 0x1
-; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
-; GFX10-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x10
+; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x8
+; GFX10-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX10-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-NEXT:    v_pk_min_i16 v1, s3, s5
-; GFX10-NEXT:    v_pk_min_i16 v0, s2, s4
-; GFX10-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX10-NEXT:    v_pk_min_i16 v1, s1, s3
+; GFX10-NEXT:    v_pk_min_i16 v0, s0, s2
+; GFX10-NEXT:    global_store_dwordx2 v2, v[0:1], s[4:5]
 ; GFX10-NEXT:    s_endpgm
 ;
 ; GFX11-LABEL: s_test_imin_sle_v4i16:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_clause 0x1
-; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x0
-; GFX11-NEXT:    s_load_b64 s[4:5], s[4:5], 0x10
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x8
+; GFX11-NEXT:    s_load_b64 s[4:5], s[4:5], 0x0
 ; GFX11-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX11-NEXT:    v_pk_min_i16 v1, s3, s5
-; GFX11-NEXT:    v_pk_min_i16 v0, s2, s4
-; GFX11-NEXT:    global_store_b64 v2, v[0:1], s[0:1]
+; GFX11-NEXT:    v_pk_min_i16 v1, s1, s3
+; GFX11-NEXT:    v_pk_min_i16 v0, s0, s2
+; GFX11-NEXT:    global_store_b64 v2, v[0:1], s[4:5]
 ; GFX11-NEXT:    s_endpgm
 ;
 ; GFX1250-LABEL: s_test_imin_sle_v4i16:
 ; GFX1250:       ; %bb.0:
 ; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
 ; GFX1250-NEXT:    s_clause 0x1
-; GFX1250-NEXT:    s_load_b128 s[0:3], s[4:5], 0x0
-; GFX1250-NEXT:    s_load_b64 s[6:7], s[4:5], 0x10
+; GFX1250-NEXT:    s_load_b128 s[0:3], s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b64 s[6:7], s[4:5], 0x0
 ; GFX1250-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX1250-NEXT:    s_wait_kmcnt 0x0
-; GFX1250-NEXT:    v_pk_min_i16 v1, s3, s7
-; GFX1250-NEXT:    v_pk_min_i16 v0, s2, s6
-; GFX1250-NEXT:    global_store_b64 v2, v[0:1], s[0:1]
+; GFX1250-NEXT:    v_pk_min_i16 v1, s1, s3
+; GFX1250-NEXT:    v_pk_min_i16 v0, s0, s2
+; GFX1250-NEXT:    global_store_b64 v2, v[0:1], s[6:7]
 ; GFX1250-NEXT:    s_endpgm
   %cmp = icmp sle <4 x i16> %a, %b
   %val = select <4 x i1> %cmp, <4 x i16> %a, <4 x i16> %b
@@ -1636,92 +1636,92 @@ define amdgpu_kernel void @s_test_imin_slt_v2i32(ptr addrspace(1) %out, <2 x i32
 ;
 ; CI-LABEL: s_test_imin_slt_v2i32:
 ; CI:       ; %bb.0:
-; CI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
-; CI-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x4
+; CI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x2
+; CI-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; CI-NEXT:    s_add_i32 s12, s12, s17
 ; CI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; CI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
+; CI-NEXT:    s_min_i32 s1, s1, s3
+; CI-NEXT:    s_min_i32 s0, s0, s2
+; CI-NEXT:    v_mov_b32_e32 v2, s4
 ; CI-NEXT:    v_mov_b32_e32 v0, s0
 ; CI-NEXT:    v_mov_b32_e32 v1, s1
-; CI-NEXT:    s_min_i32 s0, s3, s5
-; CI-NEXT:    s_min_i32 s1, s2, s4
-; CI-NEXT:    v_mov_b32_e32 v2, s1
-; CI-NEXT:    v_mov_b32_e32 v3, s0
-; CI-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
+; CI-NEXT:    v_mov_b32_e32 v3, s5
+; CI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
 ; CI-NEXT:    s_endpgm
 ;
 ; VI-LABEL: s_test_imin_slt_v2i32:
 ; VI:       ; %bb.0:
-; VI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
-; VI-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x10
+; VI-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x8
+; VI-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; VI-NEXT:    s_add_i32 s12, s12, s17
 ; VI-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; VI-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    s_min_i32 s1, s1, s3
+; VI-NEXT:    s_min_i32 s0, s0, s2
+; VI-NEXT:    v_mov_b32_e32 v2, s4
 ; VI-NEXT:    v_mov_b32_e32 v0, s0
 ; VI-NEXT:    v_mov_b32_e32 v1, s1
-; VI-NEXT:    s_min_i32 s0, s3, s5
-; VI-NEXT:    s_min_i32 s1, s2, s4
-; VI-NEXT:    v_mov_b32_e32 v2, s1
-; VI-NEXT:    v_mov_b32_e32 v3, s0
-; VI-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
+; VI-NEXT:    v_mov_b32_e32 v3, s5
+; VI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
 ; VI-NEXT:    s_endpgm
 ;
 ; GFX9-LABEL: s_test_imin_slt_v2i32:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
-; GFX9-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x10
+; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x8
+; GFX9-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX9-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    s_min_i32 s3, s3, s5
-; GFX9-NEXT:    s_min_i32 s2, s2, s4
-; GFX9-NEXT:    v_mov_b32_e32 v0, s2
-; GFX9-NEXT:    v_mov_b32_e32 v1, s3
-; GFX9-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX9-NEXT:    s_min_i32 s1, s1, s3
+; GFX9-NEXT:    s_min_i32 s0, s0, s2
+; GFX9-NEXT:    v_mov_b32_e32 v0, s0
+; GFX9-NEXT:    v_mov_b32_e32 v1, s1
+; GFX9-NEXT:    global_store_dwordx2 v2, v[0:1], s[4:5]
 ; GFX9-NEXT:    s_endpgm
 ;
 ; GFX10-LABEL: s_test_imin_slt_v2i32:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_clause 0x1
-; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
-; GFX10-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x10
+; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x8
+; GFX10-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
 ; GFX10-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-NEXT:    s_min_i32 s2, s2, s4
-; GFX10-NEXT:    s_min_i32 s3, s3, s5
-; GFX10-NEXT:    v_mov_b32_e32 v0, s2
-; GFX10-NEXT:    v_mov_b32_e32 v1, s3
-; GFX10-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX10-NEXT:    s_min_i32 s0, s0, s2
+; GFX10-NEXT:    s_min_i32 s1, s1, s3
+; GFX10-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-NEXT:    global_store_dwordx2 v2, v[0:1], s[4:5]
 ; GFX10-NEXT:    s_endpgm
 ;
 ; GFX11-LABEL: s_test_imin_slt_v2i32:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_clause 0x1
-; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x0
-; GFX11-NEXT:    s_load_b64 s[4:5], s[4:5], 0x10
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x8
+; GFX11-NEXT:    s_load_b64 s[4:5], s[4:5], 0x0
 ; GFX11-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX11-NEXT:    s_min_i32 s2, s2, s4
-; GFX11-NEXT:    s_min_i32 s3, s3, s5
-; GFX11-NEXT:    v_mov_b32_e32 v0, s2
-; GFX11-NEXT:    v_mov_b32_e32 v1, s3
-; GFX11-NEXT:    global_store_b64 v2, v[0:1], s[0:1]
+; GFX11-NEXT:    s_min_i32 s0, s0, s2
+; GFX11-NEXT:    s_min_i32 s1, s1, s3
+; GFX11-NEXT:    v_mov_b32_e32 v0, s0
+; GFX11-NEXT:    v_mov_b32_e32 v1, s1
+; GFX11-NEXT:    global_store_b64 v2, v[0:1], s[4:5]
 ; GFX11-NEXT:    s_endpgm
 ;
 ; GFX1250-LABEL: s_test_imin_slt_v2i32:
 ; GFX1250:       ; %bb.0:
 ; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
 ; GFX1250-NEXT:    s_clause 0x1
-; GFX1250-NEXT:    s_load_b128 s[0:3], s[4:5], 0x0
-; GFX1250-NEXT:    s_load_b64 s[6:7], s[4:5], 0x10
+; GFX1250-NEXT:    s_load_b128 s[0:3], s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b64 s[6:7], s[4:5], 0x0
 ; GFX1250-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX1250-NEXT:    s_wait_kmcnt 0x0
-; GFX1250-NEXT:    s_min_i32 s2, s2, s6
-; GFX1250-NEXT:    s_min_i32 s3, s3, s7
-; GFX1250-NEXT:    v_mov_b32_e32 v0, s2
-; GFX1250-NEXT:    v_mov_b32_e32 v1, s3
-; GFX1250-NEXT:    global_store_b64 v2, v[0:1], s[0:1]
+; GFX1250-NEXT:    s_min_i32 s0, s0, s2
+; GFX1250-NEXT:    s_min_i32 s1, s1, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    global_store_b64 v2, v[0:1], s[6:7]
 ; GFX1250-NEXT:    s_endpgm
   %cmp = icmp slt <2 x i32> %a, %b
   %val = select <2 x i1> %cmp, <2 x i32> %a, <2 x i32> %b
diff --git a/llvm/test/CodeGen/AMDGPU/occupancy-levels.ll b/llvm/test/CodeGen/AMDGPU/occupancy-levels.ll
index d1ab92e1d48ff..9278c024f8905 100644
--- a/llvm/test/CodeGen/AMDGPU/occupancy-levels.ll
+++ b/llvm/test/CodeGen/AMDGPU/occupancy-levels.ll
@@ -1,4 +1,5 @@
 ; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -mattr=-xnack < %s | FileCheck --check-prefixes=GCN,GFX9 %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx950 -mattr=-xnack < %s | FileCheck --check-prefixes=GCN,GFX950 %s
 ; RUN: llc -mtriple=amdgcn -mcpu=gfx1010 -mattr=-xnack < %s | FileCheck --check-prefixes=GCN,GFX10,GFX10W32,GFX1010,GFX1010W32 %s
 ; RUN: llc -mtriple=amdgcn -mcpu=gfx1010 -mattr=-xnack -mattr=+wavefrontsize64 < %s | FileCheck --check-prefixes=GCN,GFX10,GFX10W64,GFX1010,GFX1010W64 %s
 ; RUN: llc -mtriple=amdgcn -mcpu=gfx1030 < %s | FileCheck --check-prefixes=GCN,GFX10,GFX10W32,GFX1030,GFX1030W32 %s
@@ -19,49 +20,60 @@
 ; RUN: llc -mtriple=amdgcn -mcpu=gfx1153 -mattr=+wavefrontsize64 < %s | FileCheck --check-prefixes=GCN,GFX1030,GFX1030W64 %s
 ; RUN: llc -mtriple=amdgcn -mcpu=gfx1200 < %s | FileCheck --check-prefixes=GCN,GFX1100,GFX1100W32 %s
 ; RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64 < %s | FileCheck --check-prefixes=GCN,GFX1100,GFX1100W64 %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1250 < %s | FileCheck --check-prefixes=GCN,GFX1250 %s
 
 ; GCN-LABEL: {{^}}max_occupancy:
 ; GFX9:       ; Occupancy: 10
+; GFX950:     ; Occupancy: 8
 ; GFX1010:    ; Occupancy: 20
 ; GFX1030:    ; Occupancy: 16
 ; GFX1100:    ; Occupancy: 16
+; GFX1250:    ; Occupancy: 16
 define amdgpu_kernel void @max_occupancy() #10 {
   ret void
 }
 
 ; GCN-LABEL: {{^}}limited_occupancy_3:
 ; GFX9:       ; Occupancy: 3
+; GFX950:     ; Occupancy: 3
 ; GFX10W64:   ; Occupancy: 3
 ; GFX10W32:   ; Occupancy: 4
 ; GFX1100W64: ; Occupancy: 3
 ; GFX1100W32: ; Occupancy: 5
+; GFX1250:    ; Occupancy: 3
 define amdgpu_kernel void @limited_occupancy_3() #0 {
   ret void
 }
 
 ; GCN-LABEL: {{^}}limited_occupancy_18:
 ; GFX9:       ; Occupancy: 10
+; GFX950:     ; Occupancy: 8
 ; GFX1010:    ; Occupancy: 18
 ; GFX1030:    ; Occupancy: 16
 ; GFX1100:    ; Occupancy: 16
+; GFX1250:    ; Occupancy: 16
 define amdgpu_kernel void @limited_occupancy_18() #1 {
   ret void
 }
 
 ; GCN-LABEL: {{^}}limited_occupancy_19:
 ; GFX9:       ; Occupancy: 10
+; GFX950:     ; Occupancy: 8
 ; GFX1010:    ; Occupancy: 20
 ; GFX1030:    ; Occupancy: 16
 ; GFX1100:    ; Occupancy: 16
+; GFX1250:    ; Occupancy: 16
 define amdgpu_kernel void @limited_occupancy_19() #2 {
   ret void
 }
 
 ; GCN-LABEL: {{^}}used_24_vgprs:
 ; GFX9:       ; Occupancy: 10
+; GFX950:     ; Occupancy: 8
 ; GFX1010:    ; Occupancy: 20
 ; GFX1030:    ; Occupancy: 16
 ; GFX1100:    ; Occupancy: 16
+; GFX1250:    ; Occupancy: 16
 define amdgpu_kernel void @used_24_vgprs() #10 {
   call void asm sideeffect "", "~{v23}" ()
   ret void
@@ -69,10 +81,12 @@ define amdgpu_kernel void @used_24_vgprs() #10 {
 
 ; GCN-LABEL: {{^}}used_28_vgprs:
 ; GFX9:       ; Occupancy: 9
+; GFX950:     ; Occupancy: 8
 ; GFX1010W64: ; Occupancy: 18
 ; GFX1010W32: ; Occupancy: 20
 ; GFX1030:    ; Occupancy: 16
 ; GFX1100:    ; Occupancy: 16
+; GFX1250:    ; Occupancy: 16
 define amdgpu_kernel void @used_28_vgprs() #10 {
   call void asm sideeffect "", "~{v27}" ()
   ret void
@@ -80,10 +94,12 @@ define amdgpu_kernel void @used_28_vgprs() #10 {
 
 ; GCN-LABEL: {{^}}used_32_vgprs:
 ; GFX9:       ; Occupancy: 8
+; GFX950:     ; Occupancy: 8
 ; GFX10W64:   ; Occupancy: 16
 ; GFX1010W32: ; Occupancy: 20
 ; GFX1030W32: ; Occupancy: 16
 ; GFX1100:    ; Occupancy: 16
+; GFX1250:    ; Occupancy: 16
 define amdgpu_kernel void @used_32_vgprs() #10 {
   call void asm sideeffect "", "~{v31}" ()
   ret void
@@ -91,11 +107,13 @@ define amdgpu_kernel void @used_32_vgprs() #10 {
 
 ; GCN-LABEL: {{^}}used_36_vgprs:
 ; GFX9:       ; Occupancy: 7
+; GFX950:     ; Occupancy: 8
 ; GFX1010W64: ; Occupancy: 14
 ; GFX1010W32: ; Occupancy: 20
 ; GFX1030W64: ; Occupancy: 12
 ; GFX1030W32: ; Occupancy: 16
 ; GFX1100:    ; Occupancy: 16
+; GFX1250:    ; Occupancy: 16
 define amdgpu_kernel void @used_36_vgprs() #10 {
   call void asm sideeffect "", "~{v35}" ()
   ret void
@@ -103,10 +121,12 @@ define amdgpu_kernel void @used_36_vgprs() #10 {
 
 ; GCN-LABEL: {{^}}used_40_vgprs:
 ; GFX9:       ; Occupancy: 6
+; GFX950:     ; Occupancy: 8
 ; GFX10W64:   ; Occupancy: 12
 ; GFX1010W32: ; Occupancy: 20
 ; GFX1030W32: ; Occupancy: 16
 ; GFX1100:    ; Occupancy: 16
+; GFX1250:    ; Occupancy: 16
 define amdgpu_kernel void @used_40_vgprs() #10 {
   call void asm sideeffect "", "~{v39}" ()
   ret void
@@ -114,11 +134,13 @@ define amdgpu_kernel void @used_40_vgprs() #10 {
 
 ; GCN-LABEL: {{^}}used_44_vgprs:
 ; GFX9:       ; Occupancy: 5
+; GFX950:     ; Occupancy: 8
 ; GFX1010W64: ; Occupancy: 11
 ; GFX1010W32: ; Occupancy: 20
 ; GFX1030W64: ; Occupancy: 10
 ; GFX1030W32: ; Occupancy: 16
 ; GFX1100:    ; Occupancy: 16
+; GFX1250:    ; Occupancy: 16
 define amdgpu_kernel void @used_44_vgprs() #10 {
   call void asm sideeffect "", "~{v43}" ()
   ret void
@@ -126,10 +148,12 @@ define amdgpu_kernel void @used_44_vgprs() #10 {
 
 ; GCN-LABEL: {{^}}used_48_vgprs:
 ; GFX9:       ; Occupancy: 5
+; GFX950:     ; Occupancy: 8
 ; GFX10W64:   ; Occupancy: 10
 ; GFX1010W32: ; Occupancy: 20
 ; GFX1030W32: ; Occupancy: 16
 ; GFX1100:    ; Occupancy: 16
+; GFX1250:    ; Occupancy: 16
 define amdgpu_kernel void @used_48_vgprs() #10 {
   call void asm sideeffect "", "~{v47}" ()
   ret void
@@ -137,11 +161,13 @@ define amdgpu_kernel void @used_48_vgprs() #10 {
 
 ; GCN-LABEL: {{^}}used_56_vgprs:
 ; GFX9:       ; Occupancy: 4
+; GFX950:     ; Occupancy: 8
 ; GFX10W64:   ; Occupancy: 9
 ; GFX1010W32: ; Occupancy: 18
 ; GFX1030W32: ; Occupancy: 16
 ; GFX1100W64: ; Occupancy: 12
 ; GFX1100W32: ; Occupancy: 16
+; GFX1250:    ; Occupancy: 16
 define amdgpu_kernel void @used_56_vgprs() #10 {
   call void asm sideeffect "", "~{v55}" ()
   ret void
@@ -149,10 +175,12 @@ define amdgpu_kernel void @used_56_vgprs() #10 {
 
 ; GCN-LABEL: {{^}}used_64_vgprs:
 ; GFX9:       ; Occupancy: 4
+; GFX950:     ; Occupancy: 8
 ; GFX10W64:   ; Occupancy: 8
 ; GFX10W32:   ; Occupancy: 16
 ; GFX1100W64: ; Occupancy: 10
 ; GFX1100W32: ; Occupancy: 16
+; GFX1250:    ; Occupancy: 16
 define amdgpu_kernel void @used_64_vgprs() #10 {
   call void asm sideeffect "", "~{v63}" ()
   ret void
@@ -160,11 +188,13 @@ define amdgpu_kernel void @used_64_vgprs() #10 {
 
 ; GCN-LABEL: {{^}}used_72_vgprs:
 ; GFX9:       ; Occupancy: 3
+; GFX950:     ; Occupancy: 7
 ; GFX10W64:   ; Occupancy: 7
 ; GFX1010W32: ; Occupancy: 14
 ; GFX1030W32: ; Occupancy: 12
 ; GFX1100W64: ; Occupancy: 10
 ; GFX1100W32: ; Occupancy: 16
+; GFX1250:    ; Occupancy: 12
 define amdgpu_kernel void @used_72_vgprs() #10 {
   call void asm sideeffect "", "~{v71}" ()
   ret void
@@ -172,10 +202,12 @@ define amdgpu_kernel void @used_72_vgprs() #10 {
 
 ; GCN-LABEL: {{^}}used_80_vgprs:
 ; GFX9:       ; Occupancy: 3
+; GFX950:     ; Occupancy: 6
 ; GFX10W64:   ; Occupancy: 6
 ; GFX10W32:   ; Occupancy: 12
 ; GFX1100W64: ; Occupancy: 9
 ; GFX1100W32: ; Occupancy: 16
+; GFX1250:    ; Occupancy: 12
 define amdgpu_kernel void @used_80_vgprs() #10 {
   call void asm sideeffect "", "~{v79}" ()
   ret void
@@ -183,12 +215,14 @@ define amdgpu_kernel void @used_80_vgprs() #10 {
 
 ; GCN-LABEL: {{^}}used_84_vgprs:
 ; GFX9:       ; Occupancy: 3
+; GFX950:     ; Occupancy: 5
 ; GFX1010W64: ; Occupancy: 6
 ; GFX1010W32: ; Occupancy: 11
 ; GFX1030W64: ; Occupancy: 5
 ; GFX1030W32: ; Occupancy: 10
 ; GFX1100W64: ; Occupancy: 9
 ; GFX1100W32: ; Occupancy: 16
+; GFX1250:    ; Occupancy: 10
 define amdgpu_kernel void @used_84_vgprs() #10 {
   call void asm sideeffect "", "~{v83}" ()
   ret void
@@ -196,11 +230,13 @@ define amdgpu_kernel void @used_84_vgprs() #10 {
 
 ; GCN-LABEL: {{^}}used_88_vgprs:
 ; GFX9:       ; Occupancy: 2
+; GFX950:     ; Occupancy: 5
 ; GFX10W64:   ; Occupancy: 5
 ; GFX1010W32: ; Occupancy: 11
 ; GFX1030W32: ; Occupancy: 10
 ; GFX1100W64: ; Occupancy: 8
 ; GFX1100W32: ; Occupancy: 16
+; GFX1250:    ; Occupancy: 10
 define amdgpu_kernel void @used_88_vgprs() #10 {
   call void asm sideeffect "", "~{v87}" ()
   ret void
@@ -208,10 +244,12 @@ define amdgpu_kernel void @used_88_vgprs() #10 {
 
 ; GCN-LABEL: {{^}}used_96_vgprs:
 ; GFX9:       ; Occupancy: 2
+; GFX950:     ; Occupancy: 5
 ; GFX10W64:   ; Occupancy: 5
 ; GFX10W32:   ; Occupancy: 10
 ; GFX1100W64: ; Occupancy: 8
 ; GFX1100W32: ; Occupancy: 16
+; GFX1250:    ; Occupancy: 10
 define amdgpu_kernel void @used_96_vgprs() #10 {
   call void asm sideeffect "", "~{v95}" ()
   ret void
@@ -219,11 +257,13 @@ define amdgpu_kernel void @used_96_vgprs() #10 {
 
 ; GCN-LABEL: {{^}}used_100_vgprs:
 ; GFX9:       ; Occupancy: 2
+; GFX950:     ; Occupancy: 4
 ; GFX1010W64: ; Occupancy: 5
 ; GFX1030W64: ; Occupancy: 4
 ; GFX10W32:   ; Occupancy: 9
 ; GFX1100W64: ; Occupancy: 7
 ; GFX1100W32: ; Occupancy: 12
+; GFX1250:    ; Occupancy: 9
 define amdgpu_kernel void @used_100_vgprs() #10 {
   call void asm sideeffect "", "~{v99}" ()
   ret void
@@ -231,10 +271,12 @@ define amdgpu_kernel void @used_100_vgprs() #10 {
 
 ; GCN-LABEL: {{^}}used_112_vgprs:
 ; GFX9:       ; Occupancy: 2
+; GFX950:     ; Occupancy: 4
 ; GFX10W64:   ; Occupancy: 4
 ; GFX10W32:   ; Occupancy: 9
 ; GFX1100W64: ; Occupancy: 6
 ; GFX1100W32: ; Occupancy: 12
+; GFX1250:    ; Occupancy: 9
 define amdgpu_kernel void @used_112_vgprs() #10 {
   call void asm sideeffect "", "~{v111}" ()
   ret void
@@ -242,10 +284,12 @@ define amdgpu_kernel void @used_112_vgprs() #10 {
 
 ; GCN-LABEL: {{^}}used_128_vgprs:
 ; GFX9:       ; Occupancy: 2
+; GFX950:     ; Occupancy: 4
 ; GFX10W64:   ; Occupancy: 4
 ; GFX10W32:   ; Occupancy: 8
 ; GFX1100W64: ; Occupancy: 5
 ; GFX1100W32: ; Occupancy: 10
+; GFX1250:    ; Occupancy: 8
 define amdgpu_kernel void @used_128_vgprs() #10 {
   call void asm sideeffect "", "~{v127}" ()
   ret void
@@ -253,10 +297,12 @@ define amdgpu_kernel void @used_128_vgprs() #10 {
 
 ; GCN-LABEL: {{^}}used_144_vgprs:
 ; GFX9:       ; Occupancy: 1
+; GFX950:     ; Occupancy: 3
 ; GFX10W64:   ; Occupancy: 3
 ; GFX10W32:   ; Occupancy: 7
 ; GFX1100W64: ; Occupancy: 5
 ; GFX1100W32: ; Occupancy: 10
+; GFX1250:    ; Occupancy: 7
 define amdgpu_kernel void @used_144_vgprs() #10 {
   call void asm sideeffect "", "~{v143}" ()
   ret void
@@ -264,11 +310,13 @@ define amdgpu_kernel void @used_144_vgprs() #10 {
 
 ; GCN-LABEL: {{^}}used_168_vgprs:
 ; GFX9:       ; Occupancy: 1
+; GFX950:     ; Occupancy: 3
 ; GFX10W64:   ; Occupancy: 3
 ; GFX1010W32: ; Occupancy: 6
 ; GFX1030W32: ; Occupancy: 5
 ; GFX1100W64: ; Occupancy: 4
 ; GFX1100W32: ; Occupancy: 9
+; GFX1250:    ; Occupancy: 5
 define amdgpu_kernel void @used_168_vgprs() #10 {
   call void asm sideeffect "", "~{v167}" ()
   ret void
@@ -276,11 +324,13 @@ define amdgpu_kernel void @used_168_vgprs() #10 {
 
 ; GCN-LABEL: {{^}}used_200_vgprs:
 ; GFX9:       ; Occupancy: 1
+; GFX950:     ; Occupancy: 2
 ; GFX10W64:   ; Occupancy: 2
 ; GFX1010W32: ; Occupancy: 5
 ; GFX1030W32: ; Occupancy: 4
 ; GFX1100W64: ; Occupancy: 3
 ; GFX1100W32: ; Occupancy: 7
+; GFX1250:    ; Occupancy: 4
 define amdgpu_kernel void @used_200_vgprs() #10 {
   call void asm sideeffect "", "~{v199}" ()
   ret void
@@ -288,10 +338,12 @@ define amdgpu_kernel void @used_200_vgprs() #10 {
 
 ; GCN-LABEL: {{^}}used_256_vgprs:
 ; GFX9:       ; Occupancy: 1
+; GFX950:     ; Occupancy: 2
 ; GFX10W64:   ; Occupancy: 2
 ; GFX10W32:   ; Occupancy: 4
 ; GFX1100W64: ; Occupancy: 2
 ; GFX1100W32: ; Occupancy: 5
+; GFX1250:    ; Occupancy: 4
 define amdgpu_kernel void @used_256_vgprs() #10 {
   call void asm sideeffect "", "~{v255}" ()
   ret void
@@ -299,9 +351,11 @@ define amdgpu_kernel void @used_256_vgprs() #10 {
 
 ; GCN-LABEL: {{^}}used_80_sgprs:
 ; GFX9:       ; Occupancy: 10
+; GFX950:     ; Occupancy: 8
 ; GFX1010:    ; Occupancy: 20
 ; GFX1030:    ; Occupancy: 16
 ; GFX1100:    ; Occupancy: 16
+; GFX1250:    ; Occupancy: 16
 define amdgpu_kernel void @used_80_sgprs() #10 {
   call void asm sideeffect "", "~{s79}" ()
   ret void
@@ -309,9 +363,11 @@ define amdgpu_kernel void @used_80_sgprs() #10 {
 
 ; GCN-LABEL: {{^}}used_88_sgprs:
 ; GFX9:       ; Occupancy: 9
+; GFX950:     ; Occupancy: 8
 ; GFX1010:    ; Occupancy: 20
 ; GFX1030:    ; Occupancy: 16
 ; GFX1100:    ; Occupancy: 16
+; GFX1250:    ; Occupancy: 16
 define amdgpu_kernel void @used_88_sgprs() #10 {
   call void asm sideeffect "", "~{s87}" ()
   ret void
@@ -319,9 +375,11 @@ define amdgpu_kernel void @used_88_sgprs() #10 {
 
 ; GCN-LABEL: {{^}}used_100_sgprs:
 ; GFX9:       ; Occupancy: 8
+; GFX950:     ; Occupancy: 7
 ; GFX1010:    ; Occupancy: 20
 ; GFX1030:    ; Occupancy: 16
 ; GFX1100:    ; Occupancy: 16
+; GFX1250:    ; Occupancy: 16
 define amdgpu_kernel void @used_100_sgprs() #10 {
   call void asm sideeffect "", "~{s99}" ()
   ret void
@@ -329,9 +387,11 @@ define amdgpu_kernel void @used_100_sgprs() #10 {
 
 ; GCN-LABEL: {{^}}used_101_sgprs:
 ; GFX9:       ; Occupancy: 7
+; GFX950:     ; Occupancy: 7
 ; GFX1010:    ; Occupancy: 20
 ; GFX1030:    ; Occupancy: 16
 ; GFX1100:    ; Occupancy: 16
+; GFX1250:    ; Occupancy: 16
 define amdgpu_kernel void @used_101_sgprs() #10 {
   call void asm sideeffect "", "~{s100}" ()
   ret void
@@ -339,10 +399,12 @@ define amdgpu_kernel void @used_101_sgprs() #10 {
 
 ; GCN-LABEL: {{^}}used_lds_6552:
 ; GFX9:       ; Occupancy: 8
+; GFX950:     ; Occupancy: 8
 ; GFX1010W64: ; Occupancy: 20
 ; GFX1030W64: ; Occupancy: 16
 ; GFX10W32:   ; Occupancy: 16
 ; GFX1100:    ; Occupancy: 16
+; GFX1250:    ; Occupancy: 16
 @lds6552 = internal addrspace(3) global [6552 x i8] poison, align 4
 define amdgpu_kernel void @used_lds_6552() {
   store volatile i8 1, ptr addrspace(3) @lds6552
@@ -351,10 +413,12 @@ define amdgpu_kernel void @used_lds_6552() {
 
 ; GCN-LABEL: {{^}}used_lds_6556:
 ; GFX9:       ; Occupancy: 8
+; GFX950:     ; Occupancy: 8
 ; GFX1010W64: ; Occupancy: 20
 ; GFX1030W64: ; Occupancy: 16
 ; GFX10W32:   ; Occupancy: 16
 ; GFX1100:    ; Occupancy: 16
+; GFX1250:    ; Occupancy: 16
 @lds6556 = internal addrspace(3) global [6556 x i8] poison, align 4
 define amdgpu_kernel void @used_lds_6556() {
   store volatile i8 1, ptr addrspace(3) @lds6556
@@ -363,10 +427,12 @@ define amdgpu_kernel void @used_lds_6556() {
 
 ; GCN-LABEL: {{^}}used_lds_13112:
 ; GFX9:       ; Occupancy: 8
+; GFX950:     ; Occupancy: 8
 ; GFX1010W64: ; Occupancy: 20
 ; GFX1030W64: ; Occupancy: 16
 ; GFX10W32:   ; Occupancy: 16
 ; GFX1100:    ; Occupancy: 16
+; GFX1250:    ; Occupancy: 16
 @lds13112 = internal addrspace(3) global [13112 x i8] poison, align 4
 define amdgpu_kernel void @used_lds_13112() {
   store volatile i8 1, ptr addrspace(3) @lds13112
@@ -375,10 +441,12 @@ define amdgpu_kernel void @used_lds_13112() {
 
 ; GCN-LABEL: {{^}}used_lds_8252_max_group_size_64:
 ; GFX9:       ; Occupancy: 2{{$}}
+; GFX950:     ; Occupancy: 5{{$}}
 ; GFX10W64:   ; Occupancy: 4{{$}}
 ; GFX10W32:   ; Occupancy: 8{{$}}
 ; GFX1100W64: ; Occupancy: 4{{$}}
 ; GFX1100W32: ; Occupancy: 8{{$}}
+; GFX1250:    ; Occupancy: 10{{$}}
 @lds8252 = internal addrspace(3) global [8252 x i8] poison, align 4
 define amdgpu_kernel void @used_lds_8252_max_group_size_64() #3 {
   store volatile i8 1, ptr addrspace(3) @lds8252
@@ -387,10 +455,12 @@ define amdgpu_kernel void @used_lds_8252_max_group_size_64() #3 {
 
 ; GCN-LABEL: {{^}}used_lds_8252_max_group_size_96:
 ; GFX9:       ; Occupancy: 4{{$}}
+; GFX950:     ; Occupancy: 8{{$}}
 ; GFX10W64:   ; Occupancy: 8{{$}}
 ; GFX10W32:   ; Occupancy: 12{{$}}
 ; GFX1100W64: ; Occupancy: 8{{$}}
 ; GFX1100W32: ; Occupancy: 12{{$}}
+; GFX1250:    ; Occupancy: 12{{$}}
 define amdgpu_kernel void @used_lds_8252_max_group_size_96() #4 {
   store volatile i8 1, ptr addrspace(3) @lds8252
   ret void
@@ -398,10 +468,12 @@ define amdgpu_kernel void @used_lds_8252_max_group_size_96() #4 {
 
 ; GCN-LABEL: {{^}}used_lds_8252_max_group_size_128:
 ; GFX9:       ; Occupancy: 4{{$}}
+; GFX950:     ; Occupancy: 8{{$}}
 ; GFX10W64:   ; Occupancy: 8{{$}}
 ; GFX10W32:   ; Occupancy: 15{{$}}
 ; GFX1100W64: ; Occupancy: 8{{$}}
 ; GFX1100W32: ; Occupancy: 15{{$}}
+; GFX1250:    ; Occupancy: 16{{$}}
 define amdgpu_kernel void @used_lds_8252_max_group_size_128() #5 {
   store volatile i8 1, ptr addrspace(3) @lds8252
   ret void
@@ -409,11 +481,13 @@ define amdgpu_kernel void @used_lds_8252_max_group_size_128() #5 {
 
 ; GCN-LABEL: {{^}}used_lds_8252_max_group_size_192:
 ; GFX9:       ; Occupancy: 6{{$}}
+; GFX950:     ; Occupancy: 8{{$}}
 ; GFX10W64:   ; Occupancy: 12{{$}}
 ; GFX1010W32: ; Occupancy: 20{{$}}
 ; GFX1030W32: ; Occupancy: 15{{$}}
 ; GFX1100W64: ; Occupancy: 12{{$}}
 ; GFX1100W32: ; Occupancy: 15{{$}}
+; GFX1250:    ; Occupancy: 15{{$}}
 define amdgpu_kernel void @used_lds_8252_max_group_size_192() #6 {
   store volatile i8 1, ptr addrspace(3) @lds8252
   ret void
@@ -421,11 +495,13 @@ define amdgpu_kernel void @used_lds_8252_max_group_size_192() #6 {
 
 ; GCN-LABEL: {{^}}used_lds_8252_max_group_size_256:
 ; GFX9:       ; Occupancy: 7{{$}}
+; GFX950:     ; Occupancy: 8{{$}}
 ; GFX10W64:   ; Occupancy: 15{{$}}
 ; GFX1010W32: ; Occupancy: 20{{$}}
 ; GFX1030W32: ; Occupancy: 16{{$}}
 ; GFX1100W64: ; Occupancy: 15{{$}}
 ; GFX1100W32: ; Occupancy: 16{{$}}
+; GFX1250:    ; Occupancy: 16{{$}}
 define amdgpu_kernel void @used_lds_8252_max_group_size_256() #7 {
   store volatile i8 1, ptr addrspace(3) @lds8252
   ret void
@@ -433,9 +509,11 @@ define amdgpu_kernel void @used_lds_8252_max_group_size_256() #7 {
 
 ; GCN-LABEL: {{^}}used_lds_8252_max_group_size_512:
 ; GFX9:       ; Occupancy: 10{{$}}
+; GFX950:     ; Occupancy: 8{{$}}
 ; GFX1010:    ; Occupancy: 20{{$}}
 ; GFX1030:    ; Occupancy: 16{{$}}
 ; GFX1100:    ; Occupancy: 16{{$}}
+; GFX1250:    ; Occupancy: 16{{$}}
 define amdgpu_kernel void @used_lds_8252_max_group_size_512() #8 {
   store volatile i8 1, ptr addrspace(3) @lds8252
   ret void
@@ -443,10 +521,12 @@ define amdgpu_kernel void @used_lds_8252_max_group_size_512() #8 {
 
 ; GCN-LABEL: {{^}}used_lds_8252_max_group_size_1024:
 ; GFX9:       ; Occupancy: 8{{$}}
+; GFX950:     ; Occupancy: 8{{$}}
 ; GFX1010W32: ; Occupancy: 16{{$}}
 ; GFX1010W64: ; Occupancy: 20{{$}}
 ; GFX1030:    ; Occupancy: 16{{$}}
 ; GFX1100:    ; Occupancy: 16{{$}}
+; GFX1250:    ; Occupancy: 16{{$}}
 define amdgpu_kernel void @used_lds_8252_max_group_size_1024() #9 {
   store volatile i8 1, ptr addrspace(3) @lds8252
   ret void
@@ -454,8 +534,10 @@ define amdgpu_kernel void @used_lds_8252_max_group_size_1024() #9 {
 
 ; GCN-LABEL: {{^}}used_lds_8252_max_group_size_32:
 ; GFX9:       ; Occupancy: 2{{$}}
+; GFX950:     ; Occupancy: 5{{$}}
 ; GFX10:      ; Occupancy: 4{{$}}
 ; GFX1100:    ; Occupancy: 4{{$}}
+; GFX1250:    ; Occupancy: 10{{$}}
 define amdgpu_kernel void @used_lds_8252_max_group_size_32() #10 {
   store volatile i8 1, ptr addrspace(3) @lds8252
   ret void
diff --git a/llvm/test/CodeGen/AMDGPU/packed-fp32.ll b/llvm/test/CodeGen/AMDGPU/packed-fp32.ll
index 6eefafa37648f..1ed024f7aed36 100644
--- a/llvm/test/CodeGen/AMDGPU/packed-fp32.ll
+++ b/llvm/test/CodeGen/AMDGPU/packed-fp32.ll
@@ -76,33 +76,19 @@ define amdgpu_kernel void @fadd_v2_vs(ptr addrspace(1) %a, <2 x float> %x) {
 ; PACKED-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1]
 ; PACKED-NEXT:    s_endpgm
 ;
-; GFX1250-SDAG-LABEL: fadd_v2_vs:
-; GFX1250-SDAG:       ; %bb.0:
-; GFX1250-SDAG-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
-; GFX1250-SDAG-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
-; GFX1250-SDAG-NEXT:    v_and_b32_e32 v4, 0x3ff, v0
-; GFX1250-SDAG-NEXT:    s_wait_kmcnt 0x0
-; GFX1250-SDAG-NEXT:    global_load_b64 v[0:1], v4, s[0:1] scale_offset
-; GFX1250-SDAG-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
-; GFX1250-SDAG-NEXT:    s_wait_loadcnt 0x0
-; GFX1250-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-SDAG-NEXT:    v_pk_add_f32 v[0:1], v[0:1], v[2:3]
-; GFX1250-SDAG-NEXT:    global_store_b64 v4, v[0:1], s[0:1] scale_offset
-; GFX1250-SDAG-NEXT:    s_endpgm
-;
-; GFX1250-GISEL-LABEL: fadd_v2_vs:
-; GFX1250-GISEL:       ; %bb.0:
-; GFX1250-GISEL-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
-; GFX1250-GISEL-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
-; GFX1250-GISEL-NEXT:    v_and_b32_e32 v4, 0x3ff, v0
-; GFX1250-GISEL-NEXT:    s_wait_kmcnt 0x0
-; GFX1250-GISEL-NEXT:    global_load_b64 v[0:1], v4, s[0:1] scale_offset
-; GFX1250-GISEL-NEXT:    v_mov_b64_e32 v[2:3], s[2:3]
-; GFX1250-GISEL-NEXT:    s_wait_loadcnt 0x0
-; GFX1250-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-GISEL-NEXT:    v_pk_add_f32 v[0:1], v[0:1], v[2:3]
-; GFX1250-GISEL-NEXT:    global_store_b64 v4, v[0:1], s[0:1] scale_offset
-; GFX1250-GISEL-NEXT:    s_endpgm
+; GFX1250-LABEL: fadd_v2_vs:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1250-NEXT:    v_and_b32_e32 v4, 0x3ff, v0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    global_load_b64 v[0:1], v4, s[0:1] scale_offset
+; GFX1250-NEXT:    v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT:    v_pk_add_f32 v[0:1], v[0:1], v[2:3]
+; GFX1250-NEXT:    global_store_b64 v4, v[0:1], s[0:1] scale_offset
+; GFX1250-NEXT:    s_endpgm
   %id = tail call i32 @llvm.amdgcn.workitem.id.x()
   %gep = getelementptr inbounds <2 x float>, ptr addrspace(1) %a, i32 %id
   %load = load <2 x float>, ptr addrspace(1) %gep, align 8
@@ -1377,33 +1363,19 @@ define amdgpu_kernel void @fmul_v2_vs(ptr addrspace(1) %a, <2 x float> %x) {
 ; PACKED-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1]
 ; PACKED-NEXT:    s_endpgm
 ;
-; GFX1250-SDAG-LABEL: fmul_v2_vs:
-; GFX1250-SDAG:       ; %bb.0:
-; GFX1250-SDAG-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
-; GFX1250-SDAG-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
-; GFX1250-SDAG-NEXT:    v_and_b32_e32 v4, 0x3ff, v0
-; GFX1250-SDAG-NEXT:    s_wait_kmcnt 0x0
-; GFX1250-SDAG-NEXT:    global_load_b64 v[0:1], v4, s[0:1] scale_offset
-; GFX1250-SDAG-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
-; GFX1250-SDAG-NEXT:    s_wait_loadcnt 0x0
-; GFX1250-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-SDAG-NEXT:    v_pk_mul_f32 v[0:1], v[0:1], v[2:3]
-; GFX1250-SDAG-NEXT:    global_store_b64 v4, v[0:1], s[0:1] scale_offset
-; GFX1250-SDAG-NEXT:    s_endpgm
-;
-; GFX1250-GISEL-LABEL: fmul_v2_vs:
-; GFX1250-GISEL:       ; %bb.0:
-; GFX1250-GISEL-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
-; GFX1250-GISEL-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
-; GFX1250-GISEL-NEXT:    v_and_b32_e32 v4, 0x3ff, v0
-; GFX1250-GISEL-NEXT:    s_wait_kmcnt 0x0
-; GFX1250-GISEL-NEXT:    global_load_b64 v[0:1], v4, s[0:1] scale_offset
-; GFX1250-GISEL-NEXT:    v_mov_b64_e32 v[2:3], s[2:3]
-; GFX1250-GISEL-NEXT:    s_wait_loadcnt 0x0
-; GFX1250-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-GISEL-NEXT:    v_pk_mul_f32 v[0:1], v[0:1], v[2:3]
-; GFX1250-GISEL-NEXT:    global_store_b64 v4, v[0:1], s[0:1] scale_offset
-; GFX1250-GISEL-NEXT:    s_endpgm
+; GFX1250-LABEL: fmul_v2_vs:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1250-NEXT:    v_and_b32_e32 v4, 0x3ff, v0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    global_load_b64 v[0:1], v4, s[0:1] scale_offset
+; GFX1250-NEXT:    v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT:    v_pk_mul_f32 v[0:1], v[0:1], v[2:3]
+; GFX1250-NEXT:    global_store_b64 v4, v[0:1], s[0:1] scale_offset
+; GFX1250-NEXT:    s_endpgm
   %id = tail call i32 @llvm.amdgcn.workitem.id.x()
   %gep = getelementptr inbounds <2 x float>, ptr addrspace(1) %a, i32 %id
   %load = load <2 x float>, ptr addrspace(1) %gep, align 8
@@ -3568,8 +3540,8 @@ define amdgpu_kernel void @fadd_fadd_fsub(<2 x float> %arg, <2 x float> %arg1, p
 ; GFX900-LABEL: fadd_fadd_fsub:
 ; GFX900:       ; %bb.0: ; %bb
 ; GFX900-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX900-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
 ; GFX900-NEXT:    v_mov_b32_e32 v2, 0
-; GFX900-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x34
 ; GFX900-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX900-NEXT:    v_mov_b32_e32 v0, s3
 ; GFX900-NEXT:    v_add_f32_e32 v0, s1, v0
@@ -3577,14 +3549,14 @@ define amdgpu_kernel void @fadd_fadd_fsub(<2 x float> %arg, <2 x float> %arg1, p
 ; GFX900-NEXT:    v_add_f32_e32 v3, s2, v0
 ; GFX900-NEXT:    v_sub_f32_e32 v0, s0, v1
 ; GFX900-NEXT:    v_subrev_f32_e32 v1, s3, v3
-; GFX900-NEXT:    global_store_dwordx2 v2, v[0:1], s[4:5]
+; GFX900-NEXT:    global_store_dwordx2 v2, v[0:1], s[6:7]
 ; GFX900-NEXT:    s_endpgm
 ;
 ; PACKED-SDAG-LABEL: fadd_fadd_fsub:
 ; PACKED-SDAG:       ; %bb.0: ; %bb
 ; PACKED-SDAG-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; PACKED-SDAG-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
 ; PACKED-SDAG-NEXT:    v_mov_b32_e32 v4, 0
-; PACKED-SDAG-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x34
 ; PACKED-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
 ; PACKED-SDAG-NEXT:    v_mov_b32_e32 v0, s3
 ; PACKED-SDAG-NEXT:    v_add_f32_e32 v0, s1, v0
@@ -3592,7 +3564,7 @@ define amdgpu_kernel void @fadd_fadd_fsub(<2 x float> %arg, <2 x float> %arg1, p
 ; PACKED-SDAG-NEXT:    v_mov_b32_e32 v2, s0
 ; PACKED-SDAG-NEXT:    v_mov_b32_e32 v3, v0
 ; PACKED-SDAG-NEXT:    v_pk_add_f32 v[0:1], v[2:3], s[2:3] neg_lo:[0,1] neg_hi:[0,1]
-; PACKED-SDAG-NEXT:    global_store_dwordx2 v4, v[0:1], s[4:5]
+; PACKED-SDAG-NEXT:    global_store_dwordx2 v4, v[0:1], s[6:7]
 ; PACKED-SDAG-NEXT:    s_endpgm
 ;
 ; GFX90A-GISEL-LABEL: fadd_fadd_fsub:
diff --git a/llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-scale-to-agpr.mir b/llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-scale-to-agpr.mir
index 999ea42910d92..e35927e8bf00d 100644
--- a/llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-scale-to-agpr.mir
+++ b/llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-scale-to-agpr.mir
@@ -1,7 +1,9 @@
-# RUN: not --crash llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx950 -run-pass=greedy,amdgpu-rewrite-agpr-copy-mfma -verify-machineinstrs -o - %s 2>&1 | FileCheck %s
-# CHECK: Illegal virtual register for instruction
-# CHECK: Expected a VGPR_32 register, but got a AGPR_32 register
- 
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx950 -run-pass=greedy,amdgpu-rewrite-agpr-copy-mfma -verify-machineinstrs -o - %s | FileCheck %s
+# CHECK: bb.1:
+# CHECK: dead %{{[0-9]+}}:vreg_128_align2 = V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64 %{{[0-9]+}}, %{{[0-9]+}}, %{{[0-9]+}}, 4, 4, %{{[0-9]+}}, %[[REG:[0-9]+]], 4, 0, implicit $mode, implicit $exec
+# CHECK: %{{[0-9]+}}:agpr_32 = IMPLICIT_DEF
+# CHECK: %[[REG]]:vgpr_32 = COPY %{{[0-9]+}}
+
 # Test for issue in amdgpu-rewrite-agpr-copy-mfma, which reassigns scale operand
 # in vgpr_32 register to agpr_32, not permitted by instruction format.
 ---
diff --git a/llvm/test/CodeGen/AMDGPU/rotl.ll b/llvm/test/CodeGen/AMDGPU/rotl.ll
index bfdfce12cecf7..0a1d15bf945f9 100644
--- a/llvm/test/CodeGen/AMDGPU/rotl.ll
+++ b/llvm/test/CodeGen/AMDGPU/rotl.ll
@@ -94,64 +94,62 @@ define amdgpu_kernel void @rotl_v2i32(ptr addrspace(1) %in, <2 x i32> %x, <2 x i
 ;
 ; SI-LABEL: rotl_v2i32:
 ; SI:       ; %bb.0: ; %entry
-; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
-; SI-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xd
+; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0xb
+; SI-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x9
+; SI-NEXT:    s_mov_b32 s7, 0xf000
+; SI-NEXT:    s_mov_b32 s6, -1
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
-; SI-NEXT:    s_mov_b32 s6, s2
-; SI-NEXT:    s_mov_b32 s7, s3
-; SI-NEXT:    s_mov_b32 s3, 0xf000
-; SI-NEXT:    s_mov_b32 s2, -1
-; SI-NEXT:    s_sub_i32 s5, 32, s5
-; SI-NEXT:    s_sub_i32 s4, 32, s4
-; SI-NEXT:    v_mov_b32_e32 v0, s5
-; SI-NEXT:    v_alignbit_b32 v1, s7, s7, v0
-; SI-NEXT:    v_mov_b32_e32 v0, s4
-; SI-NEXT:    v_alignbit_b32 v0, s6, s6, v0
-; SI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; SI-NEXT:    s_sub_i32 s3, 32, s3
+; SI-NEXT:    s_sub_i32 s2, 32, s2
+; SI-NEXT:    v_mov_b32_e32 v0, s3
+; SI-NEXT:    v_alignbit_b32 v1, s1, s1, v0
+; SI-NEXT:    v_mov_b32_e32 v0, s2
+; SI-NEXT:    v_alignbit_b32 v0, s0, s0, v0
+; SI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
 ; SI-NEXT:    s_endpgm
 ;
 ; GFX8-LABEL: rotl_v2i32:
 ; GFX8:       ; %bb.0: ; %entry
-; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX8-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x34
+; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x2c
+; GFX8-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x24
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    v_mov_b32_e32 v1, s1
-; GFX8-NEXT:    s_sub_i32 s1, 32, s5
-; GFX8-NEXT:    v_mov_b32_e32 v0, s0
-; GFX8-NEXT:    s_sub_i32 s0, 32, s4
-; GFX8-NEXT:    v_mov_b32_e32 v2, s1
-; GFX8-NEXT:    v_alignbit_b32 v3, s3, s3, v2
-; GFX8-NEXT:    v_mov_b32_e32 v2, s0
-; GFX8-NEXT:    v_alignbit_b32 v2, s2, s2, v2
-; GFX8-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
+; GFX8-NEXT:    s_sub_i32 s2, 32, s2
+; GFX8-NEXT:    s_sub_i32 s3, 32, s3
+; GFX8-NEXT:    v_mov_b32_e32 v0, s3
+; GFX8-NEXT:    v_mov_b32_e32 v2, s2
+; GFX8-NEXT:    v_alignbit_b32 v1, s1, s1, v0
+; GFX8-NEXT:    v_alignbit_b32 v0, s0, s0, v2
+; GFX8-NEXT:    v_mov_b32_e32 v2, s4
+; GFX8-NEXT:    v_mov_b32_e32 v3, s5
+; GFX8-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
 ; GFX8-NEXT:    s_endpgm
 ;
 ; GFX10-LABEL: rotl_v2i32:
 ; GFX10:       ; %bb.0: ; %entry
 ; GFX10-NEXT:    s_clause 0x1
-; GFX10-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
-; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x2c
+; GFX10-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x24
 ; GFX10-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-NEXT:    s_sub_i32 s4, 32, s7
-; GFX10-NEXT:    s_sub_i32 s5, 32, s6
-; GFX10-NEXT:    v_alignbit_b32 v1, s3, s3, s4
-; GFX10-NEXT:    v_alignbit_b32 v0, s2, s2, s5
-; GFX10-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX10-NEXT:    s_sub_i32 s3, 32, s3
+; GFX10-NEXT:    s_sub_i32 s2, 32, s2
+; GFX10-NEXT:    v_alignbit_b32 v1, s1, s1, s3
+; GFX10-NEXT:    v_alignbit_b32 v0, s0, s0, s2
+; GFX10-NEXT:    global_store_dwordx2 v2, v[0:1], s[6:7]
 ; GFX10-NEXT:    s_endpgm
 ;
 ; GFX11-LABEL: rotl_v2i32:
 ; GFX11:       ; %bb.0: ; %entry
 ; GFX11-NEXT:    s_clause 0x1
-; GFX11-NEXT:    s_load_b64 s[6:7], s[4:5], 0x34
-; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x2c
+; GFX11-NEXT:    s_load_b64 s[4:5], s[4:5], 0x24
 ; GFX11-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX11-NEXT:    s_sub_i32 s4, 32, s7
-; GFX11-NEXT:    s_sub_i32 s5, 32, s6
-; GFX11-NEXT:    v_alignbit_b32 v1, s3, s3, s4
-; GFX11-NEXT:    v_alignbit_b32 v0, s2, s2, s5
-; GFX11-NEXT:    global_store_b64 v2, v[0:1], s[0:1]
+; GFX11-NEXT:    s_sub_i32 s3, 32, s3
+; GFX11-NEXT:    s_sub_i32 s2, 32, s2
+; GFX11-NEXT:    v_alignbit_b32 v1, s1, s1, s3
+; GFX11-NEXT:    v_alignbit_b32 v0, s0, s0, s2
+; GFX11-NEXT:    global_store_b64 v2, v[0:1], s[4:5]
 ; GFX11-NEXT:    s_endpgm
 entry:
   %0 = shl <2 x i32> %x, %y
diff --git a/llvm/test/CodeGen/AMDGPU/rotr.ll b/llvm/test/CodeGen/AMDGPU/rotr.ll
index 938d24481aaf7..403a556688091 100644
--- a/llvm/test/CodeGen/AMDGPU/rotr.ll
+++ b/llvm/test/CodeGen/AMDGPU/rotr.ll
@@ -83,56 +83,54 @@ define amdgpu_kernel void @rotr_v2i32(ptr addrspace(1) %in, <2 x i32> %x, <2 x i
 ;
 ; SI-LABEL: rotr_v2i32:
 ; SI:       ; %bb.0: ; %entry
-; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
-; SI-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xd
+; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0xb
+; SI-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x9
+; SI-NEXT:    s_mov_b32 s7, 0xf000
+; SI-NEXT:    s_mov_b32 s6, -1
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
-; SI-NEXT:    s_mov_b32 s6, s2
-; SI-NEXT:    s_mov_b32 s7, s3
-; SI-NEXT:    s_mov_b32 s3, 0xf000
-; SI-NEXT:    s_mov_b32 s2, -1
-; SI-NEXT:    v_mov_b32_e32 v0, s5
-; SI-NEXT:    v_alignbit_b32 v1, s7, s7, v0
-; SI-NEXT:    v_mov_b32_e32 v0, s4
-; SI-NEXT:    v_alignbit_b32 v0, s6, s6, v0
-; SI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; SI-NEXT:    v_mov_b32_e32 v0, s3
+; SI-NEXT:    v_alignbit_b32 v1, s1, s1, v0
+; SI-NEXT:    v_mov_b32_e32 v0, s2
+; SI-NEXT:    v_alignbit_b32 v0, s0, s0, v0
+; SI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
 ; SI-NEXT:    s_endpgm
 ;
 ; GFX8-LABEL: rotr_v2i32:
 ; GFX8:       ; %bb.0: ; %entry
-; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX8-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x34
+; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x2c
+; GFX8-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x24
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    v_mov_b32_e32 v0, s0
-; GFX8-NEXT:    v_mov_b32_e32 v2, s5
-; GFX8-NEXT:    v_mov_b32_e32 v4, s4
-; GFX8-NEXT:    v_mov_b32_e32 v1, s1
-; GFX8-NEXT:    v_alignbit_b32 v3, s3, s3, v2
-; GFX8-NEXT:    v_alignbit_b32 v2, s2, s2, v4
-; GFX8-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
+; GFX8-NEXT:    v_mov_b32_e32 v0, s3
+; GFX8-NEXT:    v_mov_b32_e32 v2, s2
+; GFX8-NEXT:    v_alignbit_b32 v1, s1, s1, v0
+; GFX8-NEXT:    v_alignbit_b32 v0, s0, s0, v2
+; GFX8-NEXT:    v_mov_b32_e32 v2, s4
+; GFX8-NEXT:    v_mov_b32_e32 v3, s5
+; GFX8-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
 ; GFX8-NEXT:    s_endpgm
 ;
 ; GFX10-LABEL: rotr_v2i32:
 ; GFX10:       ; %bb.0: ; %entry
 ; GFX10-NEXT:    s_clause 0x1
-; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX10-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x2c
+; GFX10-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x24
 ; GFX10-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-NEXT:    v_alignbit_b32 v1, s3, s3, s7
-; GFX10-NEXT:    v_alignbit_b32 v0, s2, s2, s6
-; GFX10-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX10-NEXT:    v_alignbit_b32 v1, s1, s1, s3
+; GFX10-NEXT:    v_alignbit_b32 v0, s0, s0, s2
+; GFX10-NEXT:    global_store_dwordx2 v2, v[0:1], s[6:7]
 ; GFX10-NEXT:    s_endpgm
 ;
 ; GFX11-LABEL: rotr_v2i32:
 ; GFX11:       ; %bb.0: ; %entry
 ; GFX11-NEXT:    s_clause 0x1
-; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x2c
+; GFX11-NEXT:    s_load_b64 s[4:5], s[4:5], 0x24
 ; GFX11-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX11-NEXT:    v_alignbit_b32 v1, s3, s3, s5
-; GFX11-NEXT:    v_alignbit_b32 v0, s2, s2, s4
-; GFX11-NEXT:    global_store_b64 v2, v[0:1], s[0:1]
+; GFX11-NEXT:    v_alignbit_b32 v1, s1, s1, s3
+; GFX11-NEXT:    v_alignbit_b32 v0, s0, s0, s2
+; GFX11-NEXT:    global_store_b64 v2, v[0:1], s[4:5]
 ; GFX11-NEXT:    s_endpgm
 entry:
   %tmp0 = sub <2 x i32> <i32 32, i32 32>, %y
diff --git a/llvm/test/CodeGen/AMDGPU/s_addk_i32.ll b/llvm/test/CodeGen/AMDGPU/s_addk_i32.ll
index 401b6f20d3405..f14a5cc19774d 100644
--- a/llvm/test/CodeGen/AMDGPU/s_addk_i32.ll
+++ b/llvm/test/CodeGen/AMDGPU/s_addk_i32.ll
@@ -56,8 +56,8 @@ define amdgpu_kernel void @s_addk_i32_k3(ptr addrspace(1) %out, i32 %b) #0 {
 }
 
 ; SI-LABEL: {{^}}s_addk_v2i32_k0:
-; SI-DAG: s_addk_i32 {{s[0-9]+}}, 0x42
 ; SI-DAG: s_addk_i32 {{s[0-9]+}}, 0x41
+; SI-DAG: s_addk_i32 {{s[0-9]+}}, 0x42
 ; SI: s_endpgm
 ; Note: dummy argument here to prevent combining of descriptor loads for %out and %b
 define amdgpu_kernel void @s_addk_v2i32_k0(ptr addrspace(1) %out, i32 %dummy, <2 x i32> %b) #0 {
diff --git a/llvm/test/CodeGen/AMDGPU/sminmax.v2i16.ll b/llvm/test/CodeGen/AMDGPU/sminmax.v2i16.ll
index be10302c42854..76f8f484fc763 100644
--- a/llvm/test/CodeGen/AMDGPU/sminmax.v2i16.ll
+++ b/llvm/test/CodeGen/AMDGPU/sminmax.v2i16.ll
@@ -331,80 +331,79 @@ define amdgpu_kernel void @s_abs_v4i16(ptr addrspace(1) %out, <4 x i16> %val) #0
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
-; VI-NEXT:    v_mov_b32_e32 v1, s1
-; VI-NEXT:    s_sub_i32 s1, 0, s2
-; VI-NEXT:    s_lshr_b32 s5, s2, 16
-; VI-NEXT:    v_mov_b32_e32 v0, s0
-; VI-NEXT:    s_sub_i32 s0, 0, s3
-; VI-NEXT:    s_lshr_b32 s4, s3, 16
-; VI-NEXT:    s_sub_i32 s5, 0, s5
-; VI-NEXT:    s_ashr_i32 s6, s2, 16
-; VI-NEXT:    s_sext_i32_i16 s1, s1
-; VI-NEXT:    s_sext_i32_i16 s2, s2
-; VI-NEXT:    s_sub_i32 s4, 0, s4
-; VI-NEXT:    s_sext_i32_i16 s5, s5
-; VI-NEXT:    s_max_i32 s1, s2, s1
-; VI-NEXT:    s_sext_i32_i16 s0, s0
-; VI-NEXT:    s_sext_i32_i16 s2, s3
-; VI-NEXT:    s_max_i32 s5, s6, s5
-; VI-NEXT:    s_ashr_i32 s6, s3, 16
+; VI-NEXT:    s_lshr_b32 s7, s2, 16
+; VI-NEXT:    s_sub_i32 s7, 0, s7
+; VI-NEXT:    s_sub_i32 s4, 0, s3
+; VI-NEXT:    s_lshr_b32 s6, s3, 16
+; VI-NEXT:    s_ashr_i32 s8, s2, 16
+; VI-NEXT:    s_sext_i32_i16 s7, s7
+; VI-NEXT:    s_sub_i32 s5, 0, s2
+; VI-NEXT:    s_sub_i32 s6, 0, s6
+; VI-NEXT:    s_max_i32 s7, s8, s7
+; VI-NEXT:    s_ashr_i32 s8, s3, 16
 ; VI-NEXT:    s_sext_i32_i16 s4, s4
-; VI-NEXT:    s_max_i32 s0, s2, s0
-; VI-NEXT:    s_max_i32 s4, s6, s4
-; VI-NEXT:    s_add_i32 s0, s0, 2
-; VI-NEXT:    s_lshl_b32 s2, s4, 16
-; VI-NEXT:    s_and_b32 s0, s0, 0xffff
-; VI-NEXT:    s_add_i32 s1, s1, 2
-; VI-NEXT:    s_or_b32 s0, s2, s0
-; VI-NEXT:    s_lshl_b32 s2, s5, 16
-; VI-NEXT:    s_and_b32 s1, s1, 0xffff
-; VI-NEXT:    s_or_b32 s1, s2, s1
-; VI-NEXT:    s_add_i32 s0, s0, 0x20000
-; VI-NEXT:    s_add_i32 s1, s1, 0x20000
-; VI-NEXT:    v_mov_b32_e32 v2, s1
-; VI-NEXT:    v_mov_b32_e32 v3, s0
-; VI-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
+; VI-NEXT:    s_sext_i32_i16 s3, s3
+; VI-NEXT:    s_sext_i32_i16 s6, s6
+; VI-NEXT:    s_sext_i32_i16 s5, s5
+; VI-NEXT:    s_sext_i32_i16 s2, s2
+; VI-NEXT:    s_max_i32 s3, s3, s4
+; VI-NEXT:    s_max_i32 s6, s8, s6
+; VI-NEXT:    s_max_i32 s2, s2, s5
+; VI-NEXT:    s_add_i32 s3, s3, 2
+; VI-NEXT:    s_lshl_b32 s4, s6, 16
+; VI-NEXT:    s_and_b32 s3, s3, 0xffff
+; VI-NEXT:    s_add_i32 s2, s2, 2
+; VI-NEXT:    s_or_b32 s3, s4, s3
+; VI-NEXT:    s_lshl_b32 s4, s7, 16
+; VI-NEXT:    s_and_b32 s2, s2, 0xffff
+; VI-NEXT:    s_or_b32 s2, s4, s2
+; VI-NEXT:    s_add_i32 s3, s3, 0x20000
+; VI-NEXT:    s_add_i32 s2, s2, 0x20000
+; VI-NEXT:    v_mov_b32_e32 v3, s1
+; VI-NEXT:    v_mov_b32_e32 v0, s2
+; VI-NEXT:    v_mov_b32_e32 v1, s3
+; VI-NEXT:    v_mov_b32_e32 v2, s0
+; VI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
 ; VI-NEXT:    s_endpgm
 ;
 ; CI-LABEL: s_abs_v4i16:
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
-; CI-NEXT:    s_mov_b32 s7, 0xf000
-; CI-NEXT:    s_mov_b32 s6, -1
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
-; CI-NEXT:    s_mov_b32 s4, s0
-; CI-NEXT:    s_mov_b32 s5, s1
-; CI-NEXT:    s_ashr_i32 s0, s3, 16
-; CI-NEXT:    s_ashr_i32 s1, s2, 16
-; CI-NEXT:    s_lshr_b32 s8, s2, 16
-; CI-NEXT:    s_lshr_b32 s9, s3, 16
-; CI-NEXT:    s_sext_i32_i16 s10, s3
-; CI-NEXT:    s_sext_i32_i16 s11, s2
-; CI-NEXT:    s_sub_i32 s3, 0, s3
-; CI-NEXT:    s_sub_i32 s2, 0, s2
-; CI-NEXT:    s_sext_i32_i16 s3, s3
-; CI-NEXT:    s_sext_i32_i16 s2, s2
+; CI-NEXT:    s_mov_b64 s[4:5], s[2:3]
+; CI-NEXT:    s_ashr_i32 s6, s5, 16
+; CI-NEXT:    s_lshr_b32 s9, s5, 16
+; CI-NEXT:    s_sext_i32_i16 s10, s5
+; CI-NEXT:    s_sub_i32 s5, 0, s5
+; CI-NEXT:    s_ashr_i32 s7, s4, 16
+; CI-NEXT:    s_lshr_b32 s8, s4, 16
+; CI-NEXT:    s_sext_i32_i16 s11, s4
+; CI-NEXT:    s_sext_i32_i16 s5, s5
+; CI-NEXT:    s_sub_i32 s4, 0, s4
 ; CI-NEXT:    s_sub_i32 s9, 0, s9
-; CI-NEXT:    s_sub_i32 s8, 0, s8
+; CI-NEXT:    s_sext_i32_i16 s4, s4
 ; CI-NEXT:    s_sext_i32_i16 s9, s9
+; CI-NEXT:    s_sub_i32 s8, 0, s8
+; CI-NEXT:    s_max_i32 s5, s10, s5
 ; CI-NEXT:    s_sext_i32_i16 s8, s8
-; CI-NEXT:    s_max_i32 s2, s11, s2
-; CI-NEXT:    s_max_i32 s3, s10, s3
-; CI-NEXT:    s_max_i32 s1, s1, s8
-; CI-NEXT:    s_max_i32 s0, s0, s9
-; CI-NEXT:    s_add_i32 s3, s3, 2
-; CI-NEXT:    s_add_i32 s2, s2, 2
-; CI-NEXT:    s_lshl_b32 s0, s0, 16
-; CI-NEXT:    s_and_b32 s3, s3, 0xffff
-; CI-NEXT:    s_lshl_b32 s1, s1, 16
-; CI-NEXT:    s_and_b32 s2, s2, 0xffff
-; CI-NEXT:    s_or_b32 s0, s0, s3
-; CI-NEXT:    s_or_b32 s1, s1, s2
-; CI-NEXT:    s_add_i32 s0, s0, 0x20000
-; CI-NEXT:    s_add_i32 s1, s1, 0x20000
-; CI-NEXT:    v_mov_b32_e32 v0, s1
-; CI-NEXT:    v_mov_b32_e32 v1, s0
-; CI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; CI-NEXT:    s_max_i32 s6, s6, s9
+; CI-NEXT:    s_max_i32 s4, s11, s4
+; CI-NEXT:    s_add_i32 s5, s5, 2
+; CI-NEXT:    s_max_i32 s7, s7, s8
+; CI-NEXT:    s_lshl_b32 s6, s6, 16
+; CI-NEXT:    s_and_b32 s5, s5, 0xffff
+; CI-NEXT:    s_add_i32 s4, s4, 2
+; CI-NEXT:    s_or_b32 s5, s6, s5
+; CI-NEXT:    s_lshl_b32 s6, s7, 16
+; CI-NEXT:    s_and_b32 s4, s4, 0xffff
+; CI-NEXT:    s_or_b32 s4, s6, s4
+; CI-NEXT:    s_add_i32 s5, s5, 0x20000
+; CI-NEXT:    s_add_i32 s4, s4, 0x20000
+; CI-NEXT:    s_mov_b32 s3, 0xf000
+; CI-NEXT:    s_mov_b32 s2, -1
+; CI-NEXT:    v_mov_b32_e32 v0, s4
+; CI-NEXT:    v_mov_b32_e32 v1, s5
+; CI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
 ; CI-NEXT:    s_endpgm
   %z0 = insertelement <4 x i16> poison, i16 0, i16 0
   %z1 = insertelement <4 x i16> %z0, i16 0, i16 1
diff --git a/llvm/test/CodeGen/AMDGPU/store-to-constant.ll b/llvm/test/CodeGen/AMDGPU/store-to-constant.ll
index d8f7f8d7fefcc..9b3b52012f327 100644
--- a/llvm/test/CodeGen/AMDGPU/store-to-constant.ll
+++ b/llvm/test/CodeGen/AMDGPU/store-to-constant.ll
@@ -136,8 +136,7 @@ define amdgpu_kernel void @store_as4_2xi32(ptr addrspace(4) %p, <2 x i32> %v) {
 ; CHECK-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; CHECK-NEXT:    v_mov_b32_e32 v0, 0
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
-; CHECK-NEXT:    v_mov_b32_e32 v2, s2
-; CHECK-NEXT:    v_mov_b32_e32 v3, s3
+; CHECK-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
 ; CHECK-NEXT:    global_store_dwordx2 v0, v[2:3], s[0:1]
 ; CHECK-NEXT:    s_endpgm
   store <2 x i32> %v, ptr addrspace(4) %p
@@ -164,8 +163,7 @@ define amdgpu_kernel void @store_as4_2xfloat(ptr addrspace(4) %p, <2 x float> %v
 ; CHECK-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
 ; CHECK-NEXT:    v_mov_b32_e32 v0, 0
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
-; CHECK-NEXT:    v_mov_b32_e32 v2, s2
-; CHECK-NEXT:    v_mov_b32_e32 v3, s3
+; CHECK-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
 ; CHECK-NEXT:    global_store_dwordx2 v0, v[2:3], s[0:1]
 ; CHECK-NEXT:    s_endpgm
   store <2 x float> %v, ptr addrspace(4) %p
diff --git a/llvm/test/CodeGen/AMDGPU/udivrem.ll b/llvm/test/CodeGen/AMDGPU/udivrem.ll
index fc42f476fe7d0..eaab3531824c4 100644
--- a/llvm/test/CodeGen/AMDGPU/udivrem.ll
+++ b/llvm/test/CodeGen/AMDGPU/udivrem.ll
@@ -164,102 +164,98 @@ define amdgpu_kernel void @test_udivrem_v2(ptr addrspace(1) %out, <2 x i32> %x,
 ;
 ; GFX6-LABEL: test_udivrem_v2:
 ; GFX6:       ; %bb.0:
-; GFX6-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0xd
+; GFX6-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0xb
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x9
 ; GFX6-NEXT:    s_mov_b32 s7, 0xf000
-; GFX6-NEXT:    s_mov_b32 s6, -1
 ; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX6-NEXT:    v_cvt_f32_u32_e32 v0, s8
-; GFX6-NEXT:    s_sub_i32 s0, 0, s8
-; GFX6-NEXT:    v_cvt_f32_u32_e32 v2, s9
+; GFX6-NEXT:    v_cvt_f32_u32_e32 v0, s2
+; GFX6-NEXT:    s_sub_i32 s6, 0, s2
+; GFX6-NEXT:    v_cvt_f32_u32_e32 v2, s3
 ; GFX6-NEXT:    v_rcp_iflag_f32_e32 v0, v0
 ; GFX6-NEXT:    v_rcp_iflag_f32_e32 v2, v2
 ; GFX6-NEXT:    v_mul_f32_e32 v0, 0x4f7ffffe, v0
 ; GFX6-NEXT:    v_cvt_u32_f32_e32 v0, v0
-; GFX6-NEXT:    v_mul_lo_u32 v1, s0, v0
-; GFX6-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GFX6-NEXT:    v_mul_lo_u32 v1, s6, v0
 ; GFX6-NEXT:    v_mul_hi_u32 v1, v0, v1
-; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX6-NEXT:    s_mov_b32 s5, s1
 ; GFX6-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
-; GFX6-NEXT:    v_mul_hi_u32 v0, s2, v0
+; GFX6-NEXT:    v_mul_hi_u32 v0, s0, v0
 ; GFX6-NEXT:    v_mul_f32_e32 v1, 0x4f7ffffe, v2
 ; GFX6-NEXT:    v_cvt_u32_f32_e32 v1, v1
-; GFX6-NEXT:    v_readfirstlane_b32 s4, v0
-; GFX6-NEXT:    s_mul_i32 s4, s4, s8
-; GFX6-NEXT:    s_sub_i32 s2, s2, s4
-; GFX6-NEXT:    s_sub_i32 s4, s2, s8
-; GFX6-NEXT:    s_cmp_ge_u32 s2, s8
-; GFX6-NEXT:    s_cselect_b32 s2, s4, s2
-; GFX6-NEXT:    s_sub_i32 s4, s2, s8
-; GFX6-NEXT:    s_cmp_ge_u32 s2, s8
-; GFX6-NEXT:    s_cselect_b32 s2, s4, s2
-; GFX6-NEXT:    s_sub_i32 s4, 0, s9
-; GFX6-NEXT:    v_mul_lo_u32 v0, s4, v1
-; GFX6-NEXT:    s_mov_b32 s4, s0
+; GFX6-NEXT:    v_readfirstlane_b32 s6, v0
+; GFX6-NEXT:    s_mul_i32 s6, s6, s2
+; GFX6-NEXT:    s_sub_i32 s0, s0, s6
+; GFX6-NEXT:    s_sub_i32 s6, s0, s2
+; GFX6-NEXT:    s_cmp_ge_u32 s0, s2
+; GFX6-NEXT:    s_cselect_b32 s0, s6, s0
+; GFX6-NEXT:    s_sub_i32 s6, s0, s2
+; GFX6-NEXT:    s_cmp_ge_u32 s0, s2
+; GFX6-NEXT:    s_cselect_b32 s0, s6, s0
+; GFX6-NEXT:    s_sub_i32 s2, 0, s3
+; GFX6-NEXT:    v_mul_lo_u32 v0, s2, v1
+; GFX6-NEXT:    s_mov_b32 s6, -1
 ; GFX6-NEXT:    v_mul_hi_u32 v0, v1, v0
 ; GFX6-NEXT:    v_add_i32_e32 v0, vcc, v1, v0
-; GFX6-NEXT:    v_mul_hi_u32 v0, s3, v0
-; GFX6-NEXT:    v_readfirstlane_b32 s0, v0
-; GFX6-NEXT:    s_mul_i32 s0, s0, s9
-; GFX6-NEXT:    s_sub_i32 s0, s3, s0
-; GFX6-NEXT:    s_sub_i32 s1, s0, s9
-; GFX6-NEXT:    s_cmp_ge_u32 s0, s9
-; GFX6-NEXT:    s_cselect_b32 s0, s1, s0
-; GFX6-NEXT:    s_sub_i32 s1, s0, s9
-; GFX6-NEXT:    s_cmp_ge_u32 s0, s9
-; GFX6-NEXT:    s_cselect_b32 s0, s1, s0
-; GFX6-NEXT:    v_mov_b32_e32 v0, s2
-; GFX6-NEXT:    v_mov_b32_e32 v1, s0
+; GFX6-NEXT:    v_mul_hi_u32 v0, s1, v0
+; GFX6-NEXT:    v_readfirstlane_b32 s2, v0
+; GFX6-NEXT:    s_mul_i32 s2, s2, s3
+; GFX6-NEXT:    s_sub_i32 s1, s1, s2
+; GFX6-NEXT:    s_sub_i32 s2, s1, s3
+; GFX6-NEXT:    s_cmp_ge_u32 s1, s3
+; GFX6-NEXT:    s_cselect_b32 s1, s2, s1
+; GFX6-NEXT:    s_sub_i32 s2, s1, s3
+; GFX6-NEXT:    s_cmp_ge_u32 s1, s3
+; GFX6-NEXT:    s_cselect_b32 s1, s2, s1
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX6-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
 ; GFX6-NEXT:    s_endpgm
 ;
 ; GFX8-LABEL: test_udivrem_v2:
 ; GFX8:       ; %bb.0:
-; GFX8-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x2c
+; GFX8-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x24
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    v_cvt_f32_u32_e32 v0, s6
-; GFX8-NEXT:    s_sub_i32 s0, 0, s6
-; GFX8-NEXT:    v_cvt_f32_u32_e32 v2, s7
+; GFX8-NEXT:    v_cvt_f32_u32_e32 v0, s2
+; GFX8-NEXT:    s_sub_i32 s6, 0, s2
+; GFX8-NEXT:    v_cvt_f32_u32_e32 v2, s3
 ; GFX8-NEXT:    v_rcp_iflag_f32_e32 v0, v0
 ; GFX8-NEXT:    v_rcp_iflag_f32_e32 v2, v2
 ; GFX8-NEXT:    v_mul_f32_e32 v0, 0x4f7ffffe, v0
 ; GFX8-NEXT:    v_cvt_u32_f32_e32 v0, v0
-; GFX8-NEXT:    v_mul_lo_u32 v1, s0, v0
-; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX8-NEXT:    v_mul_lo_u32 v1, s6, v0
 ; GFX8-NEXT:    v_mul_hi_u32 v1, v0, v1
 ; GFX8-NEXT:    v_add_u32_e32 v0, vcc, v0, v1
-; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    v_mul_hi_u32 v0, s2, v0
+; GFX8-NEXT:    v_mul_hi_u32 v0, s0, v0
 ; GFX8-NEXT:    v_mul_f32_e32 v1, 0x4f7ffffe, v2
 ; GFX8-NEXT:    v_cvt_u32_f32_e32 v1, v1
-; GFX8-NEXT:    v_readfirstlane_b32 s4, v0
-; GFX8-NEXT:    s_mul_i32 s4, s4, s6
-; GFX8-NEXT:    s_sub_i32 s2, s2, s4
-; GFX8-NEXT:    s_sub_i32 s4, s2, s6
-; GFX8-NEXT:    s_cmp_ge_u32 s2, s6
-; GFX8-NEXT:    s_cselect_b32 s2, s4, s2
-; GFX8-NEXT:    s_sub_i32 s4, s2, s6
-; GFX8-NEXT:    s_cmp_ge_u32 s2, s6
-; GFX8-NEXT:    s_cselect_b32 s2, s4, s2
-; GFX8-NEXT:    s_sub_i32 s4, 0, s7
-; GFX8-NEXT:    v_mul_lo_u32 v0, s4, v1
+; GFX8-NEXT:    v_mov_b32_e32 v2, s4
+; GFX8-NEXT:    v_readfirstlane_b32 s6, v0
+; GFX8-NEXT:    s_mul_i32 s6, s6, s2
+; GFX8-NEXT:    s_sub_i32 s0, s0, s6
+; GFX8-NEXT:    s_sub_i32 s6, s0, s2
+; GFX8-NEXT:    s_cmp_ge_u32 s0, s2
+; GFX8-NEXT:    s_cselect_b32 s0, s6, s0
+; GFX8-NEXT:    s_sub_i32 s6, s0, s2
+; GFX8-NEXT:    s_cmp_ge_u32 s0, s2
+; GFX8-NEXT:    s_cselect_b32 s0, s6, s0
+; GFX8-NEXT:    s_sub_i32 s2, 0, s3
+; GFX8-NEXT:    v_mul_lo_u32 v0, s2, v1
+; GFX8-NEXT:    v_mov_b32_e32 v3, s5
 ; GFX8-NEXT:    v_mul_hi_u32 v0, v1, v0
 ; GFX8-NEXT:    v_add_u32_e32 v0, vcc, v1, v0
-; GFX8-NEXT:    v_mul_hi_u32 v2, s3, v0
+; GFX8-NEXT:    v_mul_hi_u32 v1, s1, v0
 ; GFX8-NEXT:    v_mov_b32_e32 v0, s0
-; GFX8-NEXT:    v_mov_b32_e32 v1, s1
-; GFX8-NEXT:    v_readfirstlane_b32 s0, v2
-; GFX8-NEXT:    s_mul_i32 s0, s0, s7
-; GFX8-NEXT:    s_sub_i32 s0, s3, s0
-; GFX8-NEXT:    s_sub_i32 s1, s0, s7
-; GFX8-NEXT:    s_cmp_ge_u32 s0, s7
+; GFX8-NEXT:    v_readfirstlane_b32 s0, v1
+; GFX8-NEXT:    s_mul_i32 s0, s0, s3
+; GFX8-NEXT:    s_sub_i32 s0, s1, s0
+; GFX8-NEXT:    s_sub_i32 s1, s0, s3
+; GFX8-NEXT:    s_cmp_ge_u32 s0, s3
 ; GFX8-NEXT:    s_cselect_b32 s0, s1, s0
-; GFX8-NEXT:    s_sub_i32 s1, s0, s7
-; GFX8-NEXT:    s_cmp_ge_u32 s0, s7
+; GFX8-NEXT:    s_sub_i32 s1, s0, s3
+; GFX8-NEXT:    s_cmp_ge_u32 s0, s3
 ; GFX8-NEXT:    s_cselect_b32 s0, s1, s0
-; GFX8-NEXT:    v_mov_b32_e32 v2, s2
-; GFX8-NEXT:    v_mov_b32_e32 v3, s0
-; GFX8-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
+; GFX8-NEXT:    v_mov_b32_e32 v1, s0
+; GFX8-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
 ; GFX8-NEXT:    s_endpgm
   %result0 = udiv <2 x i32> %x, %y
   store <2 x i32> %result0, ptr addrspace(1) %out
diff --git a/llvm/test/CodeGen/AMDGPU/uint_to_fp.f64.ll b/llvm/test/CodeGen/AMDGPU/uint_to_fp.f64.ll
index bd311a1054a41..983acfc2c0699 100644
--- a/llvm/test/CodeGen/AMDGPU/uint_to_fp.f64.ll
+++ b/llvm/test/CodeGen/AMDGPU/uint_to_fp.f64.ll
@@ -340,8 +340,8 @@ define amdgpu_kernel void @s_uint_to_fp_v2i32_to_v2f64(ptr addrspace(1) %out, <2
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-NEXT:    v_cvt_f64_u32_e32 v[2:3], s3
 ; GCN-NEXT:    v_cvt_f64_u32_e32 v[0:1], s2
-; GCN-NEXT:    v_mov_b32_e32 v4, s0
 ; GCN-NEXT:    v_mov_b32_e32 v5, s1
+; GCN-NEXT:    v_mov_b32_e32 v4, s0
 ; GCN-NEXT:    flat_store_dwordx4 v[4:5], v[0:3]
 ; GCN-NEXT:    s_endpgm
 ;
diff --git a/llvm/test/CodeGen/SPIRV/zero-length-array.ll b/llvm/test/CodeGen/SPIRV/zero-length-array.ll
index 5fd94d25dfd87..cb34529ebfecd 100644
--- a/llvm/test/CodeGen/SPIRV/zero-length-array.ll
+++ b/llvm/test/CodeGen/SPIRV/zero-length-array.ll
@@ -1,7 +1,9 @@
-; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv-unknown-vulkan-compute %s -o - | FileCheck %s
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv-unknown-vulkan-compute < %s | FileCheck %s
 ; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-vulkan-compute %s -o - -filetype=obj | spirv-val %}
 
-; Nothing is generated, but compilation doesn't crash.
+; RUN: not llc -verify-machineinstrs -O0 -mtriple=spirv64-unknown-unknown < %s 2>&1 | FileCheck -check-prefix=CHECK-ERR %s
+
+; For compute, nothing is generated, but compilation doesn't crash.
 ; CHECK: OpName %[[#FOO:]] "foo"
 ; CHECK: OpName %[[#RTM:]] "reg2mem alloca point"
 ; CHECK: %[[#INT:]] = OpTypeInt 32 0
@@ -11,6 +13,10 @@
 ; CHECK-NEXT: OpReturn
 ; CHECK-NEXT: OpFunctionEnd
 
+
+; For non-compute, error.
+; CHECK-ERR: LLVM ERROR: Runtime arrays are not allowed in non-shader SPIR-V modules
+
 define spir_func void @foo() {
 entry:
   %i = alloca [0 x i32], align 4
diff --git a/llvm/test/CodeGen/Thumb2/mve-blockplacement.ll b/llvm/test/CodeGen/Thumb2/mve-blockplacement.ll
index d076cb00ad7e0..706a7c34c3df5 100644
--- a/llvm/test/CodeGen/Thumb2/mve-blockplacement.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-blockplacement.ll
@@ -66,9 +66,8 @@ define i32 @test(i8 zeroext %var_2, i16 signext %var_15, ptr %arr_60) {
 ; CHECK-NEXT:    cset r6, ne
 ; CHECK-NEXT:    strb r6, [r5]
 ; CHECK-NEXT:    add.w r2, r2, #792
-; CHECK-NEXT:    ldrb r6, [r3]
+; CHECK-NEXT:    ldrb r6, [r3], #2
 ; CHECK-NEXT:    adds r4, #8
-; CHECK-NEXT:    adds r3, #2
 ; CHECK-NEXT:    cmp r6, #0
 ; CHECK-NEXT:    ite ne
 ; CHECK-NEXT:    sxthne r6, r1
@@ -101,8 +100,7 @@ define i32 @test(i8 zeroext %var_2, i16 signext %var_15, ptr %arr_60) {
 ; CHECK-NEXT:    cset r6, ne
 ; CHECK-NEXT:    adds r4, #8
 ; CHECK-NEXT:    strb r6, [r5]
-; CHECK-NEXT:    ldrb r6, [r3]
-; CHECK-NEXT:    adds r3, #2
+; CHECK-NEXT:    ldrb r6, [r3], #2
 ; CHECK-NEXT:    cmp r6, #0
 ; CHECK-NEXT:    ite ne
 ; CHECK-NEXT:    sxthne r6, r1
@@ -134,8 +132,7 @@ define i32 @test(i8 zeroext %var_2, i16 signext %var_15, ptr %arr_60) {
 ; CHECK-NEXT:    cset r4, ne
 ; CHECK-NEXT:    add.w r11, r11, #8
 ; CHECK-NEXT:    strb r4, [r5]
-; CHECK-NEXT:    ldrb r4, [r3]
-; CHECK-NEXT:    adds r3, #2
+; CHECK-NEXT:    ldrb r4, [r3], #2
 ; CHECK-NEXT:    cmp r4, #0
 ; CHECK-NEXT:    ite ne
 ; CHECK-NEXT:    sxthne r4, r1
diff --git a/llvm/test/CodeGen/Thumb2/mve-intrinsics/strict-intrinsics.ll b/llvm/test/CodeGen/Thumb2/mve-intrinsics/strict-intrinsics.ll
index d19844c683a8f..9e42f3984c24d 100644
--- a/llvm/test/CodeGen/Thumb2/mve-intrinsics/strict-intrinsics.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-intrinsics/strict-intrinsics.ll
@@ -240,4 +240,93 @@ entry:
   ret <8 x half> %0
 }
 
-attributes #0 = { strictfp }
\ No newline at end of file
+
+define arm_aapcs_vfpcc <8 x half> @test_vminnmq_f16(<8 x half> %a, <8 x half> %b) #0 {
+; CHECK-LABEL: test_vminnmq_f16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmaxnm.f16 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %2 = tail call <8 x half> @llvm.arm.mve.vmaxnm.v8f16(<8 x half> %a, <8 x half> %b)
+  ret <8 x half> %2
+}
+
+define arm_aapcs_vfpcc <4 x float> @test_vminnmq_f32(<4 x float> %a, <4 x float> %b) #0 {
+; CHECK-LABEL: test_vminnmq_f32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmaxnm.f32 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %2 = tail call <4 x float> @llvm.arm.mve.vmaxnm.v4f32(<4 x float> %a, <4 x float> %b)
+  ret <4 x float> %2
+}
+
+define arm_aapcs_vfpcc <8 x half> @test_vmaxnmq_f16(<8 x half> %a, <8 x half> %b) #0 {
+; CHECK-LABEL: test_vmaxnmq_f16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmaxnm.f16 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %2 = tail call <8 x half> @llvm.arm.mve.vmaxnm.v8f16(<8 x half> %a, <8 x half> %b)
+  ret <8 x half> %2
+}
+
+define arm_aapcs_vfpcc <4 x float> @test_vmaxnmq_f32(<4 x float> %a, <4 x float> %b) #0 {
+; CHECK-LABEL: test_vmaxnmq_f32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmaxnm.f32 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %2 = tail call <4 x float> @llvm.arm.mve.vmaxnm.v4f32(<4 x float> %a, <4 x float> %b)
+  ret <4 x float> %2
+}
+
+define arm_aapcs_vfpcc <8 x half> @test_vminnmaq_f16(<8 x half> %a, <8 x half> %b) #0 {
+; CHECK-LABEL: test_vminnmaq_f16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmaxnma.f16 q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = tail call <8 x half> @llvm.fabs.v8f16(<8 x half> %a)
+  %1 = tail call <8 x half> @llvm.fabs.v8f16(<8 x half> %b)
+  %2 = tail call <8 x half> @llvm.arm.mve.vmaxnm.v8f16(<8 x half> %0, <8 x half> %1)
+  ret <8 x half> %2
+}
+
+define arm_aapcs_vfpcc <4 x float> @test_vminnmaq_f32(<4 x float> %a, <4 x float> %b) #0 {
+; CHECK-LABEL: test_vminnmaq_f32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmaxnma.f32 q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = tail call <4 x float> @llvm.fabs.v4f32(<4 x float> %a)
+  %1 = tail call <4 x float> @llvm.fabs.v4f32(<4 x float> %b)
+  %2 = tail call <4 x float> @llvm.arm.mve.vmaxnm.v4f32(<4 x float> %0, <4 x float> %1)
+  ret <4 x float> %2
+}
+
+define arm_aapcs_vfpcc <8 x half> @test_vmaxnmaq_f16(<8 x half> %a, <8 x half> %b) #0 {
+; CHECK-LABEL: test_vmaxnmaq_f16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmaxnma.f16 q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = tail call <8 x half> @llvm.fabs.v8f16(<8 x half> %a)
+  %1 = tail call <8 x half> @llvm.fabs.v8f16(<8 x half> %b)
+  %2 = tail call <8 x half> @llvm.arm.mve.vmaxnm.v8f16(<8 x half> %0, <8 x half> %1)
+  ret <8 x half> %2
+}
+
+define arm_aapcs_vfpcc <4 x float> @test_vmaxnmaq_f32(<4 x float> %a, <4 x float> %b) #0 {
+; CHECK-LABEL: test_vmaxnmaq_f32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmaxnma.f32 q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = tail call <4 x float> @llvm.fabs.v4f32(<4 x float> %a)
+  %1 = tail call <4 x float> @llvm.fabs.v4f32(<4 x float> %b)
+  %2 = tail call <4 x float> @llvm.arm.mve.vmaxnm.v4f32(<4 x float> %0, <4 x float> %1)
+  ret <4 x float> %2
+}
+
+attributes #0 = { strictfp }
diff --git a/llvm/test/CodeGen/WebAssembly/masked-shifts.ll b/llvm/test/CodeGen/WebAssembly/masked-shifts.ll
index 368f30fd5d7ed..8f90fa68e8fbd 100644
--- a/llvm/test/CodeGen/WebAssembly/masked-shifts.ll
+++ b/llvm/test/CodeGen/WebAssembly/masked-shifts.ll
@@ -46,6 +46,21 @@ define i32 @sra_i32(i32 %v, i32 %x) {
   ret i32 %a
 }
 
+define i64 @sra_i64_zext(i64 %v, i32 %x) {
+; CHECK-LABEL: sra_i64_zext:
+; CHECK:         .functype sra_i64_zext (i64, i32) -> (i64)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    local.get 1
+; CHECK-NEXT:    i64.extend_i32_u
+; CHECK-NEXT:    i64.shr_s
+; CHECK-NEXT:    # fallthrough-return
+  %m = and i32 %x, 63
+  %z = zext i32 %m to i64
+  %a = ashr i64 %v, %z
+  ret i64 %a
+}
+
 define i32 @srl_i32(i32 %v, i32 %x) {
 ; CHECK-LABEL: srl_i32:
 ; CHECK:         .functype srl_i32 (i32, i32) -> (i32)
@@ -59,6 +74,21 @@ define i32 @srl_i32(i32 %v, i32 %x) {
   ret i32 %a
 }
 
+define i64 @srl_i64_zext(i64 %v, i32 %x) {
+; CHECK-LABEL: srl_i64_zext:
+; CHECK:         .functype srl_i64_zext (i64, i32) -> (i64)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    local.get 1
+; CHECK-NEXT:    i64.extend_i32_u
+; CHECK-NEXT:    i64.shr_u
+; CHECK-NEXT:    # fallthrough-return
+  %m = and i32 %x, 63
+  %z = zext i32 %m to i64
+  %a = lshr i64 %v, %z
+  ret i64 %a
+}
+
 define i64 @shl_i64(i64 %v, i64 %x) {
 ; CHECK-LABEL: shl_i64:
 ; CHECK:         .functype shl_i64 (i64, i64) -> (i64)
diff --git a/llvm/test/CodeGen/X86/combine-fcmp.ll b/llvm/test/CodeGen/X86/combine-fcmp.ll
new file mode 100644
index 0000000000000..f2666f69949b7
--- /dev/null
+++ b/llvm/test/CodeGen/X86/combine-fcmp.ll
@@ -0,0 +1,330 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64    | FileCheck %s --check-prefixes=SSE
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64-v2 | FileCheck %s --check-prefixes=SSE
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=sandybridge | FileCheck %s --check-prefixes=AVX1OR2,AVX1
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64-v3 | FileCheck %s --check-prefixes=AVX1OR2,AVX2
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64-v4 | FileCheck %s --check-prefixes=AVX512
+
+define i4 @concat_fcmp_v4f64_v2f64(<2 x double> %a0, <2 x double> %a1) {
+; SSE-LABEL: concat_fcmp_v4f64_v2f64:
+; SSE:       # %bb.0:
+; SSE-NEXT:    xorpd %xmm2, %xmm2
+; SSE-NEXT:    xorpd %xmm3, %xmm3
+; SSE-NEXT:    cmpltpd %xmm0, %xmm3
+; SSE-NEXT:    cmpltpd %xmm1, %xmm2
+; SSE-NEXT:    shufps {{.*#+}} xmm3 = xmm3[0,2],xmm2[0,2]
+; SSE-NEXT:    movmskps %xmm3, %eax
+; SSE-NEXT:    # kill: def $al killed $al killed $eax
+; SSE-NEXT:    retq
+;
+; AVX1OR2-LABEL: concat_fcmp_v4f64_v2f64:
+; AVX1OR2:       # %bb.0:
+; AVX1OR2-NEXT:    vxorpd %xmm2, %xmm2, %xmm2
+; AVX1OR2-NEXT:    vcmpltpd %xmm0, %xmm2, %xmm0
+; AVX1OR2-NEXT:    vcmpltpd %xmm1, %xmm2, %xmm1
+; AVX1OR2-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,2],xmm1[0,2]
+; AVX1OR2-NEXT:    vmovmskps %xmm0, %eax
+; AVX1OR2-NEXT:    # kill: def $al killed $al killed $eax
+; AVX1OR2-NEXT:    retq
+;
+; AVX512-LABEL: concat_fcmp_v4f64_v2f64:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vxorpd %xmm2, %xmm2, %xmm2
+; AVX512-NEXT:    vcmpltpd %xmm0, %xmm2, %k0
+; AVX512-NEXT:    vcmpltpd %xmm1, %xmm2, %k1
+; AVX512-NEXT:    kshiftlb $2, %k1, %k1
+; AVX512-NEXT:    korw %k1, %k0, %k0
+; AVX512-NEXT:    kmovd %k0, %eax
+; AVX512-NEXT:    # kill: def $al killed $al killed $eax
+; AVX512-NEXT:    retq
+  %v0 = fcmp ogt <2 x double> %a0, zeroinitializer
+  %v1 = fcmp ogt <2 x double> %a1, zeroinitializer
+  %v = shufflevector <2 x i1> %v0, <2 x i1> %v1, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %r = bitcast <4 x i1> %v to i4
+  ret i4 %r
+}
+
+define i8 @concat_fcmp_v8f32_v4f32(<4 x float> %a0, <4 x float> %a1) {
+; SSE-LABEL: concat_fcmp_v8f32_v4f32:
+; SSE:       # %bb.0:
+; SSE-NEXT:    xorps %xmm2, %xmm2
+; SSE-NEXT:    cmpeqps %xmm2, %xmm0
+; SSE-NEXT:    cmpeqps %xmm2, %xmm1
+; SSE-NEXT:    packssdw %xmm1, %xmm0
+; SSE-NEXT:    packsswb %xmm0, %xmm0
+; SSE-NEXT:    pmovmskb %xmm0, %eax
+; SSE-NEXT:    # kill: def $al killed $al killed $eax
+; SSE-NEXT:    retq
+;
+; AVX1OR2-LABEL: concat_fcmp_v8f32_v4f32:
+; AVX1OR2:       # %bb.0:
+; AVX1OR2-NEXT:    vxorps %xmm2, %xmm2, %xmm2
+; AVX1OR2-NEXT:    vcmpeqps %xmm2, %xmm0, %xmm0
+; AVX1OR2-NEXT:    vcmpeqps %xmm2, %xmm1, %xmm1
+; AVX1OR2-NEXT:    vpackssdw %xmm1, %xmm0, %xmm0
+; AVX1OR2-NEXT:    vpacksswb %xmm0, %xmm0, %xmm0
+; AVX1OR2-NEXT:    vpmovmskb %xmm0, %eax
+; AVX1OR2-NEXT:    # kill: def $al killed $al killed $eax
+; AVX1OR2-NEXT:    retq
+;
+; AVX512-LABEL: concat_fcmp_v8f32_v4f32:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
+; AVX512-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX512-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; AVX512-NEXT:    vcmpeqps %ymm1, %ymm0, %k0
+; AVX512-NEXT:    kmovd %k0, %eax
+; AVX512-NEXT:    # kill: def $al killed $al killed $eax
+; AVX512-NEXT:    vzeroupper
+; AVX512-NEXT:    retq
+  %v0 = fcmp oeq <4 x float> %a0, zeroinitializer
+  %v1 = fcmp oeq <4 x float> %a1, zeroinitializer
+  %v = shufflevector <4 x i1> %v0, <4 x i1> %v1, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %r = bitcast <8 x i1> %v to i8
+  ret i8 %r
+}
+
+define i8 @concat_fcmp_v8f64_v2f64(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2, <2 x double> %a3) {
+; SSE-LABEL: concat_fcmp_v8f64_v2f64:
+; SSE:       # %bb.0:
+; SSE-NEXT:    xorpd %xmm4, %xmm4
+; SSE-NEXT:    cmpltpd %xmm4, %xmm0
+; SSE-NEXT:    cmpltpd %xmm4, %xmm1
+; SSE-NEXT:    packssdw %xmm1, %xmm0
+; SSE-NEXT:    cmpltpd %xmm4, %xmm2
+; SSE-NEXT:    cmpltpd %xmm4, %xmm3
+; SSE-NEXT:    packssdw %xmm3, %xmm2
+; SSE-NEXT:    packssdw %xmm0, %xmm0
+; SSE-NEXT:    packssdw %xmm2, %xmm2
+; SSE-NEXT:    packsswb %xmm2, %xmm0
+; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,3,2,3]
+; SSE-NEXT:    pmovmskb %xmm0, %eax
+; SSE-NEXT:    # kill: def $al killed $al killed $eax
+; SSE-NEXT:    retq
+;
+; AVX1OR2-LABEL: concat_fcmp_v8f64_v2f64:
+; AVX1OR2:       # %bb.0:
+; AVX1OR2-NEXT:    vxorpd %xmm4, %xmm4, %xmm4
+; AVX1OR2-NEXT:    vcmpltpd %xmm4, %xmm0, %xmm0
+; AVX1OR2-NEXT:    vcmpltpd %xmm4, %xmm1, %xmm1
+; AVX1OR2-NEXT:    vpackssdw %xmm1, %xmm0, %xmm0
+; AVX1OR2-NEXT:    vcmpltpd %xmm4, %xmm2, %xmm1
+; AVX1OR2-NEXT:    vcmpltpd %xmm4, %xmm3, %xmm2
+; AVX1OR2-NEXT:    vpackssdw %xmm2, %xmm1, %xmm1
+; AVX1OR2-NEXT:    vpackssdw %xmm1, %xmm1, %xmm1
+; AVX1OR2-NEXT:    vpackssdw %xmm0, %xmm0, %xmm0
+; AVX1OR2-NEXT:    vpacksswb %xmm1, %xmm0, %xmm0
+; AVX1OR2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,3,0,3]
+; AVX1OR2-NEXT:    vpmovmskb %xmm0, %eax
+; AVX1OR2-NEXT:    # kill: def $al killed $al killed $eax
+; AVX1OR2-NEXT:    retq
+;
+; AVX512-LABEL: concat_fcmp_v8f64_v2f64:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    # kill: def $xmm2 killed $xmm2 def $ymm2
+; AVX512-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
+; AVX512-NEXT:    vinsertf128 $1, %xmm3, %ymm2, %ymm2
+; AVX512-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX512-NEXT:    vinsertf64x4 $1, %ymm2, %zmm0, %zmm0
+; AVX512-NEXT:    vxorpd %xmm1, %xmm1, %xmm1
+; AVX512-NEXT:    vcmpltpd %zmm1, %zmm0, %k0
+; AVX512-NEXT:    kmovd %k0, %eax
+; AVX512-NEXT:    # kill: def $al killed $al killed $eax
+; AVX512-NEXT:    vzeroupper
+; AVX512-NEXT:    retq
+  %v0 = fcmp olt <2 x double> %a0, zeroinitializer
+  %v1 = fcmp olt <2 x double> %a1, zeroinitializer
+  %v2 = fcmp olt <2 x double> %a2, zeroinitializer
+  %v3 = fcmp olt <2 x double> %a3, zeroinitializer
+  %v01 = shufflevector <2 x i1> %v0, <2 x i1> %v1, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %v23 = shufflevector <2 x i1> %v2, <2 x i1> %v3, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %v = shufflevector <4 x i1> %v01, <4 x i1> %v23, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %r = bitcast <8 x i1> %v to i8
+  ret i8 %r
+}
+
+define i16 @concat_fcmp_v16f32_v4f32(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2, <4 x float> %a3) {
+; SSE-LABEL: concat_fcmp_v16f32_v4f32:
+; SSE:       # %bb.0:
+; SSE-NEXT:    xorps %xmm4, %xmm4
+; SSE-NEXT:    xorps %xmm5, %xmm5
+; SSE-NEXT:    cmpleps %xmm0, %xmm5
+; SSE-NEXT:    xorps %xmm0, %xmm0
+; SSE-NEXT:    cmpleps %xmm1, %xmm0
+; SSE-NEXT:    packssdw %xmm0, %xmm5
+; SSE-NEXT:    xorps %xmm0, %xmm0
+; SSE-NEXT:    cmpleps %xmm2, %xmm0
+; SSE-NEXT:    cmpleps %xmm3, %xmm4
+; SSE-NEXT:    packssdw %xmm4, %xmm0
+; SSE-NEXT:    packsswb %xmm0, %xmm5
+; SSE-NEXT:    pmovmskb %xmm5, %eax
+; SSE-NEXT:    # kill: def $ax killed $ax killed $eax
+; SSE-NEXT:    retq
+;
+; AVX1OR2-LABEL: concat_fcmp_v16f32_v4f32:
+; AVX1OR2:       # %bb.0:
+; AVX1OR2-NEXT:    vxorps %xmm4, %xmm4, %xmm4
+; AVX1OR2-NEXT:    vcmpleps %xmm0, %xmm4, %xmm0
+; AVX1OR2-NEXT:    vcmpleps %xmm1, %xmm4, %xmm1
+; AVX1OR2-NEXT:    vpackssdw %xmm1, %xmm0, %xmm0
+; AVX1OR2-NEXT:    vcmpleps %xmm2, %xmm4, %xmm1
+; AVX1OR2-NEXT:    vcmpleps %xmm3, %xmm4, %xmm2
+; AVX1OR2-NEXT:    vpackssdw %xmm2, %xmm1, %xmm1
+; AVX1OR2-NEXT:    vpacksswb %xmm1, %xmm0, %xmm0
+; AVX1OR2-NEXT:    vpmovmskb %xmm0, %eax
+; AVX1OR2-NEXT:    # kill: def $ax killed $ax killed $eax
+; AVX1OR2-NEXT:    retq
+;
+; AVX512-LABEL: concat_fcmp_v16f32_v4f32:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    # kill: def $xmm2 killed $xmm2 def $ymm2
+; AVX512-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
+; AVX512-NEXT:    vinsertf128 $1, %xmm3, %ymm2, %ymm2
+; AVX512-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX512-NEXT:    vinsertf64x4 $1, %ymm2, %zmm0, %zmm0
+; AVX512-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; AVX512-NEXT:    vcmpleps %zmm0, %zmm1, %k0
+; AVX512-NEXT:    kmovd %k0, %eax
+; AVX512-NEXT:    # kill: def $ax killed $ax killed $eax
+; AVX512-NEXT:    vzeroupper
+; AVX512-NEXT:    retq
+  %v0 = fcmp oge <4 x float> %a0, zeroinitializer
+  %v1 = fcmp oge <4 x float> %a1, zeroinitializer
+  %v2 = fcmp oge <4 x float> %a2, zeroinitializer
+  %v3 = fcmp oge <4 x float> %a3, zeroinitializer
+  %v01 = shufflevector <4 x i1> %v0, <4 x i1> %v1, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %v23 = shufflevector <4 x i1> %v2, <4 x i1> %v3, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %v = shufflevector <8 x i1> %v01, <8 x i1> %v23, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  %r = bitcast <16 x i1> %v to i16
+  ret i16 %r
+}
+
+define i8 @concat_fcmp_v8f64_v4f64(<4 x double> %a0, <4 x double> %a1) {
+; SSE-LABEL: concat_fcmp_v8f64_v4f64:
+; SSE:       # %bb.0:
+; SSE-NEXT:    xorpd %xmm4, %xmm4
+; SSE-NEXT:    movapd %xmm1, %xmm5
+; SSE-NEXT:    cmpneqpd %xmm4, %xmm5
+; SSE-NEXT:    cmpordpd %xmm4, %xmm1
+; SSE-NEXT:    andpd %xmm5, %xmm1
+; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
+; SSE-NEXT:    movapd %xmm0, %xmm5
+; SSE-NEXT:    cmpneqpd %xmm4, %xmm5
+; SSE-NEXT:    cmpordpd %xmm4, %xmm0
+; SSE-NEXT:    andpd %xmm5, %xmm0
+; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; SSE-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; SSE-NEXT:    movapd %xmm3, %xmm1
+; SSE-NEXT:    cmpneqpd %xmm4, %xmm1
+; SSE-NEXT:    cmpordpd %xmm4, %xmm3
+; SSE-NEXT:    andpd %xmm1, %xmm3
+; SSE-NEXT:    movapd %xmm2, %xmm1
+; SSE-NEXT:    cmpneqpd %xmm4, %xmm1
+; SSE-NEXT:    cmpordpd %xmm4, %xmm2
+; SSE-NEXT:    andpd %xmm1, %xmm2
+; SSE-NEXT:    packssdw %xmm3, %xmm2
+; SSE-NEXT:    packssdw %xmm2, %xmm2
+; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; SSE-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,2,1,3,4,5,6,7]
+; SSE-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
+; SSE-NEXT:    packsswb %xmm0, %xmm0
+; SSE-NEXT:    pmovmskb %xmm0, %eax
+; SSE-NEXT:    # kill: def $al killed $al killed $eax
+; SSE-NEXT:    retq
+;
+; AVX1-LABEL: concat_fcmp_v8f64_v4f64:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vxorpd %xmm2, %xmm2, %xmm2
+; AVX1-NEXT:    vcmpneq_oqpd %ymm2, %ymm0, %ymm0
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm3
+; AVX1-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,2],xmm3[0,2]
+; AVX1-NEXT:    vcmpneq_oqpd %ymm2, %ymm1, %ymm1
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
+; AVX1-NEXT:    vshufps {{.*#+}} xmm1 = xmm1[0,2],xmm2[0,2]
+; AVX1-NEXT:    vpackssdw %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm0
+; AVX1-NEXT:    vpackuswb %xmm0, %xmm0, %xmm0
+; AVX1-NEXT:    vpmovmskb %xmm0, %eax
+; AVX1-NEXT:    # kill: def $al killed $al killed $eax
+; AVX1-NEXT:    vzeroupper
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: concat_fcmp_v8f64_v4f64:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vxorpd %xmm2, %xmm2, %xmm2
+; AVX2-NEXT:    vcmpneq_oqpd %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    vextractf128 $1, %ymm0, %xmm3
+; AVX2-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,2],xmm3[0,2]
+; AVX2-NEXT:    vcmpneq_oqpd %ymm2, %ymm1, %ymm1
+; AVX2-NEXT:    vextractf128 $1, %ymm1, %xmm2
+; AVX2-NEXT:    vshufps {{.*#+}} xmm1 = xmm1[0,2],xmm2[0,2]
+; AVX2-NEXT:    vpackssdw %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[1,3,5,7,9,11,13,15,u,u,u,u,u,u,u,u]
+; AVX2-NEXT:    vpmovmskb %xmm0, %eax
+; AVX2-NEXT:    # kill: def $al killed $al killed $eax
+; AVX2-NEXT:    vzeroupper
+; AVX2-NEXT:    retq
+;
+; AVX512-LABEL: concat_fcmp_v8f64_v4f64:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512-NEXT:    vinsertf64x4 $1, %ymm1, %zmm0, %zmm0
+; AVX512-NEXT:    vxorpd %xmm1, %xmm1, %xmm1
+; AVX512-NEXT:    vcmpneq_oqpd %zmm1, %zmm0, %k0
+; AVX512-NEXT:    kmovd %k0, %eax
+; AVX512-NEXT:    # kill: def $al killed $al killed $eax
+; AVX512-NEXT:    vzeroupper
+; AVX512-NEXT:    retq
+  %v0 = fcmp one <4 x double> %a0, zeroinitializer
+  %v1 = fcmp one <4 x double> %a1, zeroinitializer
+  %v = shufflevector <4 x i1> %v0, <4 x i1> %v1, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %r = bitcast <8 x i1> %v to i8
+  ret i8 %r
+}
+
+define i16 @concat_fcmp_v16f32_v8f32(<8 x float> %a0, <8 x float> %a1) {
+; SSE-LABEL: concat_fcmp_v16f32_v8f32:
+; SSE:       # %bb.0:
+; SSE-NEXT:    xorps %xmm4, %xmm4
+; SSE-NEXT:    cmpleps %xmm4, %xmm1
+; SSE-NEXT:    cmpleps %xmm4, %xmm0
+; SSE-NEXT:    packssdw %xmm1, %xmm0
+; SSE-NEXT:    cmpleps %xmm4, %xmm3
+; SSE-NEXT:    cmpleps %xmm4, %xmm2
+; SSE-NEXT:    packssdw %xmm3, %xmm2
+; SSE-NEXT:    packsswb %xmm2, %xmm0
+; SSE-NEXT:    pmovmskb %xmm0, %eax
+; SSE-NEXT:    # kill: def $ax killed $ax killed $eax
+; SSE-NEXT:    retq
+;
+; AVX1OR2-LABEL: concat_fcmp_v16f32_v8f32:
+; AVX1OR2:       # %bb.0:
+; AVX1OR2-NEXT:    vxorps %xmm2, %xmm2, %xmm2
+; AVX1OR2-NEXT:    vcmpleps %ymm2, %ymm0, %ymm0
+; AVX1OR2-NEXT:    vextractf128 $1, %ymm0, %xmm3
+; AVX1OR2-NEXT:    vpackssdw %xmm3, %xmm0, %xmm0
+; AVX1OR2-NEXT:    vcmpleps %ymm2, %ymm1, %ymm1
+; AVX1OR2-NEXT:    vextractf128 $1, %ymm1, %xmm2
+; AVX1OR2-NEXT:    vpackssdw %xmm2, %xmm1, %xmm1
+; AVX1OR2-NEXT:    vpacksswb %xmm1, %xmm0, %xmm0
+; AVX1OR2-NEXT:    vpmovmskb %xmm0, %eax
+; AVX1OR2-NEXT:    # kill: def $ax killed $ax killed $eax
+; AVX1OR2-NEXT:    vzeroupper
+; AVX1OR2-NEXT:    retq
+;
+; AVX512-LABEL: concat_fcmp_v16f32_v8f32:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512-NEXT:    vinsertf64x4 $1, %ymm1, %zmm0, %zmm0
+; AVX512-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; AVX512-NEXT:    vcmpleps %zmm1, %zmm0, %k0
+; AVX512-NEXT:    kmovd %k0, %eax
+; AVX512-NEXT:    # kill: def $ax killed $ax killed $eax
+; AVX512-NEXT:    vzeroupper
+; AVX512-NEXT:    retq
+  %v0 = fcmp ole <8 x float> %a0, zeroinitializer
+  %v1 = fcmp ole <8 x float> %a1, zeroinitializer
+  %v = shufflevector <8 x i1> %v0, <8 x i1> %v1, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  %r = bitcast <16 x i1> %v to i16
+  ret i16 %r
+}
diff --git a/llvm/test/Transforms/Attributor/nofpclass.ll b/llvm/test/Transforms/Attributor/nofpclass.ll
index a9ebdaa397015..d82dc412f5e36 100644
--- a/llvm/test/Transforms/Attributor/nofpclass.ll
+++ b/llvm/test/Transforms/Attributor/nofpclass.ll
@@ -2667,15 +2667,10 @@ define [4 x float] @constant_aggregate_zero() {
 }
 
 define <vscale x 4 x float> @scalable_splat_pnorm() {
-; CHECK-CV: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(none)
-; CHECK-CV-LABEL: define noundef <vscale x 4 x float> @scalable_splat_pnorm
-; CHECK-CV-SAME: () #[[ATTR3]] {
-; CHECK-CV-NEXT:    ret <vscale x 4 x float> splat (float 1.000000e+00)
-;
-; CHECK-CI: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(none)
-; CHECK-CI-LABEL: define noundef nofpclass(nan inf zero sub nnorm) <vscale x 4 x float> @scalable_splat_pnorm
-; CHECK-CI-SAME: () #[[ATTR3]] {
-; CHECK-CI-NEXT:    ret <vscale x 4 x float> splat (float 1.000000e+00)
+; CHECK: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(none)
+; CHECK-LABEL: define noundef nofpclass(nan inf zero sub nnorm) <vscale x 4 x float> @scalable_splat_pnorm
+; CHECK-SAME: () #[[ATTR3]] {
+; CHECK-NEXT:    ret <vscale x 4 x float> splat (float 1.000000e+00)
 ;
   ret <vscale x 4 x float> splat (float 1.0)
 }
@@ -2689,6 +2684,19 @@ define <vscale x 4 x float> @scalable_splat_zero() {
   ret <vscale x 4 x float> zeroinitializer
 }
 
+define <vscale x 4 x float> @scalable_splat_nnan(float nofpclass(nan) %x) {
+; CHECK: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(none)
+; CHECK-LABEL: define nofpclass(nan) <vscale x 4 x float> @scalable_splat_nnan
+; CHECK-SAME: (float nofpclass(nan) [[X:%.*]]) #[[ATTR3]] {
+; CHECK-NEXT:    [[HEAD:%.*]] = insertelement <vscale x 4 x float> poison, float [[X]], i32 0
+; CHECK-NEXT:    [[SPLAT:%.*]] = shufflevector <vscale x 4 x float> [[HEAD]], <vscale x 4 x float> poison, <vscale x 4 x i32> zeroinitializer
+; CHECK-NEXT:    ret <vscale x 4 x float> [[SPLAT]]
+;
+  %head = insertelement <vscale x 4 x float> poison, float %x, i32 0
+  %splat = shufflevector <vscale x 4 x float> %head, <vscale x 4 x float> poison, <vscale x 4 x i32> zeroinitializer
+  ret <vscale x 4 x float> %splat
+}
+
 ; Verify we do not derive 'nofpclass(inf zero sub norm)' for the argument __x.
 ; See https://github.com/llvm/llvm-project/issues/78507
 
@@ -2989,5 +2997,7 @@ attributes #5 = { "denormal-fp-math"="ieee,positive-zero" }
 ;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
 ; CGSCC-CI: {{.*}}
 ; CGSCC-CV: {{.*}}
+; CHECK-CI: {{.*}}
+; CHECK-CV: {{.*}}
 ; TUNIT-CI: {{.*}}
 ; TUNIT-CV: {{.*}}
diff --git a/llvm/test/Transforms/IndVarSimplify/AArch64/widen-loop-comp.ll b/llvm/test/Transforms/IndVarSimplify/AArch64/widen-loop-comp.ll
index 257816650017a..1a6400997f080 100644
--- a/llvm/test/Transforms/IndVarSimplify/AArch64/widen-loop-comp.ll
+++ b/llvm/test/Transforms/IndVarSimplify/AArch64/widen-loop-comp.ll
@@ -97,7 +97,7 @@ define void @test2(ptr %a, ptr %b, i8 %limit, i1 %arg) {
 ; CHECK-LABEL: @test2(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[CONV:%.*]] = zext i8 [[LIMIT:%.*]] to i32
-; CHECK-NEXT:    br i1 %arg, label [[FOR_COND1_PREHEADER_PREHEADER:%.*]], label [[FOR_COND1_PREHEADER_US_PREHEADER:%.*]]
+; CHECK-NEXT:    br i1 [[ARG:%.*]], label [[FOR_COND1_PREHEADER_PREHEADER:%.*]], label [[FOR_COND1_PREHEADER_US_PREHEADER:%.*]]
 ; CHECK:       for.cond1.preheader.us.preheader:
 ; CHECK-NEXT:    [[SMAX:%.*]] = call i32 @llvm.smax.i32(i32 [[CONV]], i32 1)
 ; CHECK-NEXT:    br label [[FOR_COND1_PREHEADER_US:%.*]]
diff --git a/llvm/test/Transforms/IndVarSimplify/ARM/code-size.ll b/llvm/test/Transforms/IndVarSimplify/ARM/code-size.ll
index 2003b1a72206d..acba88ef5a54d 100644
--- a/llvm/test/Transforms/IndVarSimplify/ARM/code-size.ll
+++ b/llvm/test/Transforms/IndVarSimplify/ARM/code-size.ll
@@ -4,13 +4,12 @@
 
 define i32 @remove_loop(i32 %size) #0 {
 ; CHECK-V8M-LABEL: @remove_loop(
-; CHECK-V8M-SAME: i32 [[SIZE:%.*]]) #[[ATTR0:[0-9]+]] {
 ; CHECK-V8M-NEXT:  entry:
-; CHECK-V8M-NEXT:    br label %[[WHILE_COND:.*]]
+; CHECK-V8M-NEXT:    br label [[WHILE_COND:%.*]]
 ; CHECK-V8M:       while.cond:
-; CHECK-V8M-NEXT:    br i1 false, label %[[WHILE_COND]], label %[[WHILE_END:.*]]
+; CHECK-V8M-NEXT:    br i1 false, label [[WHILE_COND]], label [[WHILE_END:%.*]]
 ; CHECK-V8M:       while.end:
-; CHECK-V8M-NEXT:    [[TMP0:%.*]] = add i32 [[SIZE]], 31
+; CHECK-V8M-NEXT:    [[TMP0:%.*]] = add i32 [[SIZE:%.*]], 31
 ; CHECK-V8M-NEXT:    [[UMIN:%.*]] = call i32 @llvm.umin.i32(i32 [[SIZE]], i32 31)
 ; CHECK-V8M-NEXT:    [[TMP1:%.*]] = sub i32 [[TMP0]], [[UMIN]]
 ; CHECK-V8M-NEXT:    [[TMP2:%.*]] = lshr i32 [[TMP1]], 5
@@ -19,13 +18,12 @@ define i32 @remove_loop(i32 %size) #0 {
 ; CHECK-V8M-NEXT:    ret i32 [[TMP4]]
 ;
 ; CHECK-V8A-LABEL: @remove_loop(
-; CHECK-V8A-SAME: i32 [[SIZE:%.*]]) #[[ATTR0:[0-9]+]] {
 ; CHECK-V8A-NEXT:  entry:
-; CHECK-V8A-NEXT:    br label %[[WHILE_COND:.*]]
+; CHECK-V8A-NEXT:    br label [[WHILE_COND:%.*]]
 ; CHECK-V8A:       while.cond:
-; CHECK-V8A-NEXT:    br i1 false, label %[[WHILE_COND]], label %[[WHILE_END:.*]]
+; CHECK-V8A-NEXT:    br i1 false, label [[WHILE_COND]], label [[WHILE_END:%.*]]
 ; CHECK-V8A:       while.end:
-; CHECK-V8A-NEXT:    [[TMP0:%.*]] = add i32 [[SIZE]], 31
+; CHECK-V8A-NEXT:    [[TMP0:%.*]] = add i32 [[SIZE:%.*]], 31
 ; CHECK-V8A-NEXT:    [[UMIN:%.*]] = call i32 @llvm.umin.i32(i32 [[SIZE]], i32 31)
 ; CHECK-V8A-NEXT:    [[TMP1:%.*]] = sub i32 [[TMP0]], [[UMIN]]
 ; CHECK-V8A-NEXT:    [[TMP2:%.*]] = lshr i32 [[TMP1]], 5
diff --git a/llvm/test/Transforms/IndVarSimplify/X86/iv-widen.ll b/llvm/test/Transforms/IndVarSimplify/X86/iv-widen.ll
index cc0f2587266a2..45bb66d1d7d80 100644
--- a/llvm/test/Transforms/IndVarSimplify/X86/iv-widen.ll
+++ b/llvm/test/Transforms/IndVarSimplify/X86/iv-widen.ll
@@ -16,7 +16,7 @@ declare void @use(i64 %x)
 define void @loop_0(ptr %a, i1 %arg) {
 ; CHECK-LABEL: @loop_0(
 ; CHECK-NEXT:  Prologue:
-; CHECK-NEXT:    br i1 %arg, label [[B18_PREHEADER:%.*]], label [[B6:%.*]]
+; CHECK-NEXT:    br i1 [[ARG:%.*]], label [[B18_PREHEADER:%.*]], label [[B6:%.*]]
 ; CHECK:       B18.preheader:
 ; CHECK-NEXT:    br label [[B18:%.*]]
 ; CHECK:       B18:
@@ -70,7 +70,7 @@ exit24:                      ; preds = %B18
 define void @loop_0_dead(ptr %a, i1 %arg) {
 ; CHECK-LABEL: @loop_0_dead(
 ; CHECK-NEXT:  Prologue:
-; CHECK-NEXT:    br i1 %arg, label [[B18_PREHEADER:%.*]], label [[B6:%.*]]
+; CHECK-NEXT:    br i1 [[ARG:%.*]], label [[B18_PREHEADER:%.*]], label [[B6:%.*]]
 ; CHECK:       B18.preheader:
 ; CHECK-NEXT:    br label [[B18:%.*]]
 ; CHECK:       B18:
diff --git a/llvm/test/Transforms/IndVarSimplify/X86/pr59615.ll b/llvm/test/Transforms/IndVarSimplify/X86/pr59615.ll
index 17b7b9d40b07a..5ecb684b8b2f5 100644
--- a/llvm/test/Transforms/IndVarSimplify/X86/pr59615.ll
+++ b/llvm/test/Transforms/IndVarSimplify/X86/pr59615.ll
@@ -7,7 +7,7 @@ target triple = "x86_64-unknown-linux-gnu"
 define void @test() {
 ; CHECK-LABEL: @test(
 ; CHECK-NEXT:  bb:
-; CHECK-NEXT:    [[VAR:%.*]] = load atomic i32, ptr addrspace(1) poison unordered, align 8, !range [[RNG0:![0-9]+]], !invariant.load !1, !noundef !1
+; CHECK-NEXT:    [[VAR:%.*]] = load atomic i32, ptr addrspace(1) poison unordered, align 8, !range [[RNG0:![0-9]+]], !invariant.load [[META1:![0-9]+]], !noundef [[META1]]
 ; CHECK-NEXT:    [[VAR2:%.*]] = icmp eq i32 [[VAR]], 0
 ; CHECK-NEXT:    br i1 [[VAR2]], label [[BB18:%.*]], label [[BB19:%.*]]
 ; CHECK:       bb3:
@@ -16,7 +16,7 @@ define void @test() {
 ; CHECK:       bb7:
 ; CHECK-NEXT:    ret void
 ; CHECK:       bb8:
-; CHECK-NEXT:    [[VAR9:%.*]] = load atomic i32, ptr addrspace(1) poison unordered, align 8, !range [[RNG0]], !invariant.load !1, !noundef !1
+; CHECK-NEXT:    [[VAR9:%.*]] = load atomic i32, ptr addrspace(1) poison unordered, align 8, !range [[RNG0]], !invariant.load [[META1]], !noundef [[META1]]
 ; CHECK-NEXT:    [[TMP0:%.*]] = zext i32 [[VAR9]] to i64
 ; CHECK-NEXT:    [[VAR10:%.*]] = icmp ult i64 [[INDVARS_IV]], [[TMP0]]
 ; CHECK-NEXT:    br i1 [[VAR10]], label [[BB12]], label [[BB11:%.*]]
diff --git a/llvm/test/Transforms/IndVarSimplify/debugloc-rem-subst.ll b/llvm/test/Transforms/IndVarSimplify/debugloc-rem-subst.ll
index 121eec75c1b3c..4502416a19477 100644
--- a/llvm/test/Transforms/IndVarSimplify/debugloc-rem-subst.ll
+++ b/llvm/test/Transforms/IndVarSimplify/debugloc-rem-subst.ll
@@ -51,7 +51,7 @@ bb2:                                              ; preds = %bb2, %bb1
 !8 = !DILocation(line: 1, column: 1, scope: !5)
 ;.
 ; CHECK: [[META0:![0-9]+]] = distinct !DICompileUnit(language: DW_LANG_C, file: [[META1:![0-9]+]], producer: "debugify", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug)
-; CHECK: [[META1]] = !DIFile(filename: "llvm/test/Transforms/IndVarSimplify/debugloc-rem-subst.ll", directory: {{.*}})
+; CHECK: [[META1]] = !DIFile(filename: "{{.*}}debugloc-rem-subst.ll", directory: {{.*}})
 ; CHECK: [[DBG5]] = distinct !DISubprogram(name: "widget", linkageName: "widget", scope: null, file: [[META1]], line: 1, type: [[META6:![0-9]+]], scopeLine: 1, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: [[META0]])
 ; CHECK: [[META6]] = !DISubroutineType(types: [[META7:![0-9]+]])
 ; CHECK: [[META7]] = !{}
diff --git a/llvm/test/Transforms/IndVarSimplify/dont-recompute.ll b/llvm/test/Transforms/IndVarSimplify/dont-recompute.ll
index b4cd98cd234f0..6a809fe45d660 100644
--- a/llvm/test/Transforms/IndVarSimplify/dont-recompute.ll
+++ b/llvm/test/Transforms/IndVarSimplify/dont-recompute.ll
@@ -211,7 +211,7 @@ define void @test6(i32 %m, ptr %p) nounwind uwtable {
 ; CHECK-NEXT:    [[ADD]] = add i32 [[A_05]], [[M:%.*]]
 ; CHECK-NEXT:    [[SOFT_USE:%.*]] = add i32 [[ADD]], 123
 ; CHECK-NEXT:    [[PIDX:%.*]] = getelementptr i32, ptr [[P:%.*]], i32 [[ADD]]
-; CHECK-NEXT:    store i32 [[SOFT_USE]], ptr [[PIDX]]
+; CHECK-NEXT:    store i32 [[SOFT_USE]], ptr [[PIDX]], align 4
 ; CHECK-NEXT:    [[INC]] = add nuw nsw i32 [[I_06]], 1
 ; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[INC]], 186
 ; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_END:%.*]], label [[FOR_BODY]]
diff --git a/llvm/test/Transforms/IndVarSimplify/eliminate-exit.ll b/llvm/test/Transforms/IndVarSimplify/eliminate-exit.ll
index b24650830778f..488aed2ba0211 100644
--- a/llvm/test/Transforms/IndVarSimplify/eliminate-exit.ll
+++ b/llvm/test/Transforms/IndVarSimplify/eliminate-exit.ll
@@ -221,6 +221,220 @@ exit:
 }
 
 define void @many_exits([100 x i64] %len) {
+; CHECK-LABEL: @many_exits(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[LEN1:%.*]] = extractvalue [100 x i64] [[LEN:%.*]], 1
+; CHECK-NEXT:    [[LEN2:%.*]] = extractvalue [100 x i64] [[LEN]], 2
+; CHECK-NEXT:    [[LEN3:%.*]] = extractvalue [100 x i64] [[LEN]], 3
+; CHECK-NEXT:    [[LEN4:%.*]] = extractvalue [100 x i64] [[LEN]], 4
+; CHECK-NEXT:    [[LEN5:%.*]] = extractvalue [100 x i64] [[LEN]], 5
+; CHECK-NEXT:    [[LEN6:%.*]] = extractvalue [100 x i64] [[LEN]], 6
+; CHECK-NEXT:    [[LEN7:%.*]] = extractvalue [100 x i64] [[LEN]], 7
+; CHECK-NEXT:    [[LEN8:%.*]] = extractvalue [100 x i64] [[LEN]], 8
+; CHECK-NEXT:    [[LEN9:%.*]] = extractvalue [100 x i64] [[LEN]], 9
+; CHECK-NEXT:    [[LEN10:%.*]] = extractvalue [100 x i64] [[LEN]], 10
+; CHECK-NEXT:    [[LEN11:%.*]] = extractvalue [100 x i64] [[LEN]], 11
+; CHECK-NEXT:    [[LEN12:%.*]] = extractvalue [100 x i64] [[LEN]], 12
+; CHECK-NEXT:    [[LEN13:%.*]] = extractvalue [100 x i64] [[LEN]], 13
+; CHECK-NEXT:    [[LEN14:%.*]] = extractvalue [100 x i64] [[LEN]], 14
+; CHECK-NEXT:    [[LEN15:%.*]] = extractvalue [100 x i64] [[LEN]], 15
+; CHECK-NEXT:    [[LEN16:%.*]] = extractvalue [100 x i64] [[LEN]], 16
+; CHECK-NEXT:    [[LEN17:%.*]] = extractvalue [100 x i64] [[LEN]], 17
+; CHECK-NEXT:    [[LEN18:%.*]] = extractvalue [100 x i64] [[LEN]], 18
+; CHECK-NEXT:    [[LEN19:%.*]] = extractvalue [100 x i64] [[LEN]], 19
+; CHECK-NEXT:    [[LEN20:%.*]] = extractvalue [100 x i64] [[LEN]], 20
+; CHECK-NEXT:    [[LEN21:%.*]] = extractvalue [100 x i64] [[LEN]], 21
+; CHECK-NEXT:    [[LEN22:%.*]] = extractvalue [100 x i64] [[LEN]], 22
+; CHECK-NEXT:    [[LEN23:%.*]] = extractvalue [100 x i64] [[LEN]], 23
+; CHECK-NEXT:    [[LEN24:%.*]] = extractvalue [100 x i64] [[LEN]], 24
+; CHECK-NEXT:    [[LEN25:%.*]] = extractvalue [100 x i64] [[LEN]], 25
+; CHECK-NEXT:    [[LEN26:%.*]] = extractvalue [100 x i64] [[LEN]], 26
+; CHECK-NEXT:    [[LEN27:%.*]] = extractvalue [100 x i64] [[LEN]], 27
+; CHECK-NEXT:    [[LEN28:%.*]] = extractvalue [100 x i64] [[LEN]], 28
+; CHECK-NEXT:    [[LEN29:%.*]] = extractvalue [100 x i64] [[LEN]], 29
+; CHECK-NEXT:    [[LEN30:%.*]] = extractvalue [100 x i64] [[LEN]], 30
+; CHECK-NEXT:    [[LEN31:%.*]] = extractvalue [100 x i64] [[LEN]], 31
+; CHECK-NEXT:    [[LEN32:%.*]] = extractvalue [100 x i64] [[LEN]], 32
+; CHECK-NEXT:    [[LEN33:%.*]] = extractvalue [100 x i64] [[LEN]], 33
+; CHECK-NEXT:    [[LEN34:%.*]] = extractvalue [100 x i64] [[LEN]], 34
+; CHECK-NEXT:    [[LEN35:%.*]] = extractvalue [100 x i64] [[LEN]], 35
+; CHECK-NEXT:    [[LEN36:%.*]] = extractvalue [100 x i64] [[LEN]], 36
+; CHECK-NEXT:    [[LEN37:%.*]] = extractvalue [100 x i64] [[LEN]], 37
+; CHECK-NEXT:    [[LEN38:%.*]] = extractvalue [100 x i64] [[LEN]], 38
+; CHECK-NEXT:    [[LEN39:%.*]] = extractvalue [100 x i64] [[LEN]], 39
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[BACKEDGE:%.*]] ]
+; CHECK-NEXT:    [[LEN0:%.*]] = extractvalue [100 x i64] [[LEN]], 0
+; CHECK-NEXT:    [[EARLY0:%.*]] = icmp eq i64 [[IV]], [[LEN0]]
+; CHECK-NEXT:    call void @side_effect()
+; CHECK-NEXT:    br i1 [[EARLY0]], label [[EXIT:%.*]], label [[CONT0:%.*]]
+; CHECK:       cont0:
+; CHECK-NEXT:    [[EARLY1:%.*]] = icmp eq i64 [[IV]], [[LEN1]]
+; CHECK-NEXT:    call void @side_effect()
+; CHECK-NEXT:    br i1 [[EARLY1]], label [[EXIT]], label [[CONT1:%.*]]
+; CHECK:       cont1:
+; CHECK-NEXT:    [[EARLY2:%.*]] = icmp eq i64 [[IV]], [[LEN2]]
+; CHECK-NEXT:    call void @side_effect()
+; CHECK-NEXT:    br i1 [[EARLY2]], label [[EXIT]], label [[CONT2:%.*]]
+; CHECK:       cont2:
+; CHECK-NEXT:    [[EARLY3:%.*]] = icmp eq i64 [[IV]], [[LEN3]]
+; CHECK-NEXT:    call void @side_effect()
+; CHECK-NEXT:    br i1 [[EARLY3]], label [[EXIT]], label [[CONT3:%.*]]
+; CHECK:       cont3:
+; CHECK-NEXT:    [[EARLY4:%.*]] = icmp eq i64 [[IV]], [[LEN4]]
+; CHECK-NEXT:    call void @side_effect()
+; CHECK-NEXT:    br i1 [[EARLY4]], label [[EXIT]], label [[CONT4:%.*]]
+; CHECK:       cont4:
+; CHECK-NEXT:    [[EARLY5:%.*]] = icmp eq i64 [[IV]], [[LEN5]]
+; CHECK-NEXT:    call void @side_effect()
+; CHECK-NEXT:    br i1 [[EARLY5]], label [[EXIT]], label [[CONT5:%.*]]
+; CHECK:       cont5:
+; CHECK-NEXT:    [[EARLY6:%.*]] = icmp eq i64 [[IV]], [[LEN6]]
+; CHECK-NEXT:    call void @side_effect()
+; CHECK-NEXT:    br i1 [[EARLY6]], label [[EXIT]], label [[CONT6:%.*]]
+; CHECK:       cont6:
+; CHECK-NEXT:    [[EARLY7:%.*]] = icmp eq i64 [[IV]], [[LEN7]]
+; CHECK-NEXT:    call void @side_effect()
+; CHECK-NEXT:    br i1 [[EARLY7]], label [[EXIT]], label [[CONT7:%.*]]
+; CHECK:       cont7:
+; CHECK-NEXT:    [[EARLY8:%.*]] = icmp eq i64 [[IV]], [[LEN8]]
+; CHECK-NEXT:    call void @side_effect()
+; CHECK-NEXT:    br i1 [[EARLY8]], label [[EXIT]], label [[CONT8:%.*]]
+; CHECK:       cont8:
+; CHECK-NEXT:    [[EARLY9:%.*]] = icmp eq i64 [[IV]], [[LEN9]]
+; CHECK-NEXT:    call void @side_effect()
+; CHECK-NEXT:    br i1 [[EARLY9]], label [[EXIT]], label [[CONT9:%.*]]
+; CHECK:       cont9:
+; CHECK-NEXT:    [[EARLY10:%.*]] = icmp eq i64 [[IV]], [[LEN10]]
+; CHECK-NEXT:    call void @side_effect()
+; CHECK-NEXT:    br i1 [[EARLY10]], label [[EXIT]], label [[CONT10:%.*]]
+; CHECK:       cont10:
+; CHECK-NEXT:    [[EARLY11:%.*]] = icmp eq i64 [[IV]], [[LEN11]]
+; CHECK-NEXT:    call void @side_effect()
+; CHECK-NEXT:    br i1 [[EARLY11]], label [[EXIT]], label [[CONT11:%.*]]
+; CHECK:       cont11:
+; CHECK-NEXT:    [[EARLY12:%.*]] = icmp eq i64 [[IV]], [[LEN12]]
+; CHECK-NEXT:    call void @side_effect()
+; CHECK-NEXT:    br i1 [[EARLY12]], label [[EXIT]], label [[CONT12:%.*]]
+; CHECK:       cont12:
+; CHECK-NEXT:    [[EARLY13:%.*]] = icmp eq i64 [[IV]], [[LEN13]]
+; CHECK-NEXT:    call void @side_effect()
+; CHECK-NEXT:    br i1 [[EARLY13]], label [[EXIT]], label [[CONT13:%.*]]
+; CHECK:       cont13:
+; CHECK-NEXT:    [[EARLY14:%.*]] = icmp eq i64 [[IV]], [[LEN14]]
+; CHECK-NEXT:    call void @side_effect()
+; CHECK-NEXT:    br i1 [[EARLY14]], label [[EXIT]], label [[CONT14:%.*]]
+; CHECK:       cont14:
+; CHECK-NEXT:    [[EARLY15:%.*]] = icmp eq i64 [[IV]], [[LEN15]]
+; CHECK-NEXT:    call void @side_effect()
+; CHECK-NEXT:    br i1 [[EARLY15]], label [[EXIT]], label [[CONT15:%.*]]
+; CHECK:       cont15:
+; CHECK-NEXT:    [[EARLY16:%.*]] = icmp eq i64 [[IV]], [[LEN16]]
+; CHECK-NEXT:    call void @side_effect()
+; CHECK-NEXT:    br i1 [[EARLY16]], label [[EXIT]], label [[CONT16:%.*]]
+; CHECK:       cont16:
+; CHECK-NEXT:    [[EARLY17:%.*]] = icmp eq i64 [[IV]], [[LEN17]]
+; CHECK-NEXT:    call void @side_effect()
+; CHECK-NEXT:    br i1 [[EARLY17]], label [[EXIT]], label [[CONT17:%.*]]
+; CHECK:       cont17:
+; CHECK-NEXT:    [[EARLY18:%.*]] = icmp eq i64 [[IV]], [[LEN18]]
+; CHECK-NEXT:    call void @side_effect()
+; CHECK-NEXT:    br i1 [[EARLY18]], label [[EXIT]], label [[CONT18:%.*]]
+; CHECK:       cont18:
+; CHECK-NEXT:    [[EARLY19:%.*]] = icmp eq i64 [[IV]], [[LEN19]]
+; CHECK-NEXT:    call void @side_effect()
+; CHECK-NEXT:    br i1 [[EARLY19]], label [[EXIT]], label [[CONT19:%.*]]
+; CHECK:       cont19:
+; CHECK-NEXT:    [[EARLY20:%.*]] = icmp eq i64 [[IV]], [[LEN20]]
+; CHECK-NEXT:    call void @side_effect()
+; CHECK-NEXT:    br i1 [[EARLY20]], label [[EXIT]], label [[CONT20:%.*]]
+; CHECK:       cont20:
+; CHECK-NEXT:    [[EARLY21:%.*]] = icmp eq i64 [[IV]], [[LEN21]]
+; CHECK-NEXT:    call void @side_effect()
+; CHECK-NEXT:    br i1 [[EARLY21]], label [[EXIT]], label [[CONT21:%.*]]
+; CHECK:       cont21:
+; CHECK-NEXT:    [[EARLY22:%.*]] = icmp eq i64 [[IV]], [[LEN22]]
+; CHECK-NEXT:    call void @side_effect()
+; CHECK-NEXT:    br i1 [[EARLY22]], label [[EXIT]], label [[CONT22:%.*]]
+; CHECK:       cont22:
+; CHECK-NEXT:    [[EARLY23:%.*]] = icmp eq i64 [[IV]], [[LEN23]]
+; CHECK-NEXT:    call void @side_effect()
+; CHECK-NEXT:    br i1 [[EARLY23]], label [[EXIT]], label [[CONT23:%.*]]
+; CHECK:       cont23:
+; CHECK-NEXT:    [[EARLY24:%.*]] = icmp eq i64 [[IV]], [[LEN24]]
+; CHECK-NEXT:    call void @side_effect()
+; CHECK-NEXT:    br i1 [[EARLY24]], label [[EXIT]], label [[CONT24:%.*]]
+; CHECK:       cont24:
+; CHECK-NEXT:    [[EARLY25:%.*]] = icmp eq i64 [[IV]], [[LEN25]]
+; CHECK-NEXT:    call void @side_effect()
+; CHECK-NEXT:    br i1 [[EARLY25]], label [[EXIT]], label [[CONT25:%.*]]
+; CHECK:       cont25:
+; CHECK-NEXT:    [[EARLY26:%.*]] = icmp eq i64 [[IV]], [[LEN26]]
+; CHECK-NEXT:    call void @side_effect()
+; CHECK-NEXT:    br i1 [[EARLY26]], label [[EXIT]], label [[CONT26:%.*]]
+; CHECK:       cont26:
+; CHECK-NEXT:    [[EARLY27:%.*]] = icmp eq i64 [[IV]], [[LEN27]]
+; CHECK-NEXT:    call void @side_effect()
+; CHECK-NEXT:    br i1 [[EARLY27]], label [[EXIT]], label [[CONT27:%.*]]
+; CHECK:       cont27:
+; CHECK-NEXT:    [[EARLY28:%.*]] = icmp eq i64 [[IV]], [[LEN28]]
+; CHECK-NEXT:    call void @side_effect()
+; CHECK-NEXT:    br i1 [[EARLY28]], label [[EXIT]], label [[CONT28:%.*]]
+; CHECK:       cont28:
+; CHECK-NEXT:    [[EARLY29:%.*]] = icmp eq i64 [[IV]], [[LEN29]]
+; CHECK-NEXT:    call void @side_effect()
+; CHECK-NEXT:    br i1 [[EARLY29]], label [[EXIT]], label [[CONT29:%.*]]
+; CHECK:       cont29:
+; CHECK-NEXT:    [[EARLY30:%.*]] = icmp eq i64 [[IV]], [[LEN30]]
+; CHECK-NEXT:    call void @side_effect()
+; CHECK-NEXT:    br i1 [[EARLY30]], label [[EXIT]], label [[CONT30:%.*]]
+; CHECK:       cont30:
+; CHECK-NEXT:    [[EARLY31:%.*]] = icmp eq i64 [[IV]], [[LEN31]]
+; CHECK-NEXT:    call void @side_effect()
+; CHECK-NEXT:    br i1 [[EARLY31]], label [[EXIT]], label [[CONT31:%.*]]
+; CHECK:       cont31:
+; CHECK-NEXT:    [[EARLY32:%.*]] = icmp eq i64 [[IV]], [[LEN32]]
+; CHECK-NEXT:    call void @side_effect()
+; CHECK-NEXT:    br i1 [[EARLY32]], label [[EXIT]], label [[CONT32:%.*]]
+; CHECK:       cont32:
+; CHECK-NEXT:    [[EARLY33:%.*]] = icmp eq i64 [[IV]], [[LEN33]]
+; CHECK-NEXT:    call void @side_effect()
+; CHECK-NEXT:    br i1 [[EARLY33]], label [[EXIT]], label [[CONT33:%.*]]
+; CHECK:       cont33:
+; CHECK-NEXT:    [[EARLY34:%.*]] = icmp eq i64 [[IV]], [[LEN34]]
+; CHECK-NEXT:    call void @side_effect()
+; CHECK-NEXT:    br i1 [[EARLY34]], label [[EXIT]], label [[CONT34:%.*]]
+; CHECK:       cont34:
+; CHECK-NEXT:    [[EARLY35:%.*]] = icmp eq i64 [[IV]], [[LEN35]]
+; CHECK-NEXT:    call void @side_effect()
+; CHECK-NEXT:    br i1 [[EARLY35]], label [[EXIT]], label [[CONT35:%.*]]
+; CHECK:       cont35:
+; CHECK-NEXT:    [[EARLY36:%.*]] = icmp eq i64 [[IV]], [[LEN36]]
+; CHECK-NEXT:    call void @side_effect()
+; CHECK-NEXT:    br i1 [[EARLY36]], label [[EXIT]], label [[CONT36:%.*]]
+; CHECK:       cont36:
+; CHECK-NEXT:    [[EARLY37:%.*]] = icmp eq i64 [[IV]], [[LEN37]]
+; CHECK-NEXT:    call void @side_effect()
+; CHECK-NEXT:    br i1 [[EARLY37]], label [[EXIT]], label [[CONT37:%.*]]
+; CHECK:       cont37:
+; CHECK-NEXT:    [[EARLY38:%.*]] = icmp eq i64 [[IV]], [[LEN38]]
+; CHECK-NEXT:    call void @side_effect()
+; CHECK-NEXT:    br i1 [[EARLY38]], label [[EXIT]], label [[CONT38:%.*]]
+; CHECK:       cont38:
+; CHECK-NEXT:    [[EARLY39:%.*]] = icmp eq i64 [[IV]], [[LEN39]]
+; CHECK-NEXT:    call void @side_effect()
+; CHECK-NEXT:    br i1 [[EARLY39]], label [[EXIT]], label [[CONT39:%.*]]
+; CHECK:       cont39:
+; CHECK-NEXT:    br label [[BACKEDGE]]
+; CHECK:       backedge:
+; CHECK-NEXT:    call void @side_effect()
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp ult i64 [[IV]], 999
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    br i1 [[CMP2]], label [[LOOP]], label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
 entry:
   br label %loop
 loop:
diff --git a/llvm/test/Transforms/IndVarSimplify/eliminate-sat.ll b/llvm/test/Transforms/IndVarSimplify/eliminate-sat.ll
index 9fcfc7c9b349a..dc0e49efb091f 100644
--- a/llvm/test/Transforms/IndVarSimplify/eliminate-sat.ll
+++ b/llvm/test/Transforms/IndVarSimplify/eliminate-sat.ll
@@ -13,7 +13,7 @@ define void @uadd_sat(ptr %p) {
 ; CHECK:       loop:
 ; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[I_INC:%.*]], [[LOOP]] ]
 ; CHECK-NEXT:    [[SAT1:%.*]] = add nuw nsw i32 [[I]], 1
-; CHECK-NEXT:    store volatile i32 [[SAT1]], ptr [[P:%.*]]
+; CHECK-NEXT:    store volatile i32 [[SAT1]], ptr [[P:%.*]], align 4
 ; CHECK-NEXT:    [[I_INC]] = add nuw nsw i32 [[I]], 1
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 [[I_INC]], 100
 ; CHECK-NEXT:    br i1 [[CMP]], label [[LOOP]], label [[END:%.*]]
@@ -42,7 +42,7 @@ define void @sadd_sat(ptr %p) {
 ; CHECK:       loop:
 ; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[I_INC:%.*]], [[LOOP]] ]
 ; CHECK-NEXT:    [[SAT1:%.*]] = add nuw nsw i32 [[I]], 1
-; CHECK-NEXT:    store volatile i32 [[SAT1]], ptr [[P:%.*]]
+; CHECK-NEXT:    store volatile i32 [[SAT1]], ptr [[P:%.*]], align 4
 ; CHECK-NEXT:    [[I_INC]] = add nuw nsw i32 [[I]], 1
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 [[I_INC]], 100
 ; CHECK-NEXT:    br i1 [[CMP]], label [[LOOP]], label [[END:%.*]]
@@ -71,7 +71,7 @@ define void @usub_sat(ptr %p) {
 ; CHECK:       loop:
 ; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 1, [[ENTRY:%.*]] ], [ [[I_INC:%.*]], [[LOOP]] ]
 ; CHECK-NEXT:    [[SAT1:%.*]] = sub nuw nsw i32 [[I]], 1
-; CHECK-NEXT:    store volatile i32 [[SAT1]], ptr [[P:%.*]]
+; CHECK-NEXT:    store volatile i32 [[SAT1]], ptr [[P:%.*]], align 4
 ; CHECK-NEXT:    [[I_INC]] = add nuw nsw i32 [[I]], 1
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 [[I_INC]], 100
 ; CHECK-NEXT:    br i1 [[CMP]], label [[LOOP]], label [[END:%.*]]
@@ -100,7 +100,7 @@ define void @ssub_sat(ptr %p) {
 ; CHECK:       loop:
 ; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[I_INC:%.*]], [[LOOP]] ]
 ; CHECK-NEXT:    [[SAT1:%.*]] = sub nsw i32 [[I]], 1
-; CHECK-NEXT:    store volatile i32 [[SAT1]], ptr [[P:%.*]]
+; CHECK-NEXT:    store volatile i32 [[SAT1]], ptr [[P:%.*]], align 4
 ; CHECK-NEXT:    [[I_INC]] = add nuw nsw i32 [[I]], 1
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 [[I_INC]], 100
 ; CHECK-NEXT:    br i1 [[CMP]], label [[LOOP]], label [[END:%.*]]
diff --git a/llvm/test/Transforms/IndVarSimplify/monotonic_checks.ll b/llvm/test/Transforms/IndVarSimplify/monotonic_checks.ll
index a1c07b0a24638..1f8bf5fecb248 100644
--- a/llvm/test/Transforms/IndVarSimplify/monotonic_checks.ll
+++ b/llvm/test/Transforms/IndVarSimplify/monotonic_checks.ll
@@ -6,7 +6,7 @@
 define i32 @test_01(ptr %p) {
 ; CHECK-LABEL: @test_01(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[LEN:%.*]] = load i32, ptr [[P:%.*]], align 4, [[RNG0:!range !.*]]
+; CHECK-NEXT:    [[LEN:%.*]] = load i32, ptr [[P:%.*]], align 4, !range [[RNG0:![0-9]+]]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ [[LEN]], [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[BACKEDGE:%.*]] ]
@@ -45,7 +45,7 @@ exit:
 define i32 @test_01_neg(ptr %p) {
 ; CHECK-LABEL: @test_01_neg(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[LEN:%.*]] = load i32, ptr [[P:%.*]], align 4, [[RNG0]]
+; CHECK-NEXT:    [[LEN:%.*]] = load i32, ptr [[P:%.*]], align 4, !range [[RNG0]]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ [[LEN]], [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[BACKEDGE:%.*]] ]
@@ -86,7 +86,7 @@ exit:
 define i32 @test_02(ptr %p) {
 ; CHECK-LABEL: @test_02(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[LEN:%.*]] = load i32, ptr [[P:%.*]], align 4, [[RNG1:!range !.*]]
+; CHECK-NEXT:    [[LEN:%.*]] = load i32, ptr [[P:%.*]], align 4, !range [[RNG1:![0-9]+]]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ [[LEN]], [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[BACKEDGE:%.*]] ]
@@ -125,7 +125,7 @@ exit:
 define i32 @test_02_neg(ptr %p) {
 ; CHECK-LABEL: @test_02_neg(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[LEN:%.*]] = load i32, ptr [[P:%.*]], align 4, [[RNG1]]
+; CHECK-NEXT:    [[LEN:%.*]] = load i32, ptr [[P:%.*]], align 4, !range [[RNG1]]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ [[LEN]], [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[BACKEDGE:%.*]] ]
@@ -164,7 +164,7 @@ exit:
 define i32 @test_03(ptr %p) {
 ; CHECK-LABEL: @test_03(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[LEN:%.*]] = load i32, ptr [[P:%.*]], align 4, [[RNG2:!range !.*]]
+; CHECK-NEXT:    [[LEN:%.*]] = load i32, ptr [[P:%.*]], align 4, !range [[RNG2:![0-9]+]]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ [[LEN]], [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[BACKEDGE:%.*]] ]
@@ -202,7 +202,7 @@ exit:
 define i32 @test_04(ptr %p) {
 ; CHECK-LABEL: @test_04(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[LEN:%.*]] = load i32, ptr [[P:%.*]], align 4, [[RNG2]]
+; CHECK-NEXT:    [[LEN:%.*]] = load i32, ptr [[P:%.*]], align 4, !range [[RNG2]]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ [[LEN]], [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[BACKEDGE:%.*]] ]
diff --git a/llvm/test/Transforms/InstCombine/copy-access-metadata.ll b/llvm/test/Transforms/InstCombine/copy-access-metadata.ll
deleted file mode 100644
index c687f3796edcb..0000000000000
--- a/llvm/test/Transforms/InstCombine/copy-access-metadata.ll
+++ /dev/null
@@ -1,215 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
-; RUN: opt -S -passes=instcombine %s | FileCheck %s
-
-@test.data = private unnamed_addr constant [8 x i32] [i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7], align 4
-@test.ptrdata = private unnamed_addr constant [8 x ptr] [ptr null, ptr null, ptr null, ptr null, ptr null, ptr null, ptr null, ptr null], align 8
-
-; Verify that InstCombine copies range metadata when cloning a load as part of
-; replacing an alloca initialized via memcpy from a constant. OK
-define i32 @copy_range_metadata_after_memcpy(i64 %x) {
-; CHECK-LABEL: define i32 @copy_range_metadata_after_memcpy(
-; CHECK-SAME: i64 [[X:%.*]]) {
-; CHECK-NEXT:  [[ENTRY:.*:]]
-; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr @test.data, i64 [[X]]
-; CHECK-NEXT:    [[L:%.*]] = load i32, ptr [[ARRAYIDX]], align 4, !range [[RNG0:![0-9]+]]
-; CHECK-NEXT:    ret i32 [[L]]
-;
-entry:
-  %data = alloca [8 x i32], align 4
-  call void @llvm.memcpy.p0.p0.i64(ptr align 4 %data, ptr align 4 @test.data, i64 32, i1 false)
-  %arrayidx = getelementptr inbounds [8 x i32], ptr %data, i64 0, i64 %x
-  %l = load i32, ptr %arrayidx, align 4, !range !0
-  ret i32 %l
-}
-
-declare void @llvm.memcpy.p0.p0.i64(ptr nocapture writeonly, ptr nocapture readonly, i64, i1)
-
-!0 = !{i32 0, i32 100}
-
-; Verify TBAA metadata on a cloned load is preserved. OK
-define i32 @copy_tbaa_metadata_after_memcpy(i64 %x, ptr %sink) {
-; CHECK-LABEL: define i32 @copy_tbaa_metadata_after_memcpy(
-; CHECK-SAME: i64 [[X:%.*]], ptr [[SINK:%.*]]) {
-; CHECK-NEXT:  [[ENTRY:.*:]]
-; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr @test.data, i64 [[X]]
-; CHECK-NEXT:    [[L:%.*]] = load i32, ptr [[ARRAYIDX]], align 4, !tbaa [[SCALAR_TYPE_TBAA1:![0-9]+]]
-; CHECK-NEXT:    store i32 [[L]], ptr [[SINK]], align 4
-; CHECK-NEXT:    ret i32 [[L]]
-;
-entry:
-  %data = alloca [8 x i32], align 4
-  call void @llvm.memcpy.p0.p0.i64(ptr align 4 %data, ptr align 4 @test.data, i64 32, i1 false)
-  %arrayidx = getelementptr inbounds [8 x i32], ptr %data, i64 0, i64 %x
-  %l = load i32, ptr %arrayidx, align 4, !tbaa !1
-  store i32 %l, ptr %sink, align 4
-  ret i32 %l
-}
-
-!1 = !{!2, !2, i64 0}
-!2 = !{!"scalar type", !3}
-!3 = !{!"root"}
-
-; Verify dereferenceable_or_null metadata on a cloned load is preserved
-; when the loaded value type is a pointer. OK
-define ptr @copy_deref_or_null_metadata_after_memcpy(i64 %x) {
-; CHECK-LABEL: define ptr @copy_deref_or_null_metadata_after_memcpy(
-; CHECK-SAME: i64 [[X:%.*]]) {
-; CHECK-NEXT:  [[ENTRY:.*:]]
-; CHECK-NEXT:    ret ptr null
-;
-entry:
-  %data = alloca [8 x ptr], align 8
-  call void @llvm.memcpy.p0.p0.i64(ptr align 8 %data, ptr align 8 @test.ptrdata, i64 64, i1 false)
-  %arrayidx = getelementptr inbounds [8 x ptr], ptr %data, i64 0, i64 %x
-  %l = load ptr, ptr %arrayidx, align 8, !dereferenceable_or_null !4
-  ret ptr %l
-}
-
-!4 = !{i64 8}
-
-; Verify nonnull metadata on a cloned load is preserved
-; when the loaded value type is a pointer. OK
-define ptr @copy_nonnull_metadata_after_memcpy(i64 %x) {
-; CHECK-LABEL: define ptr @copy_nonnull_metadata_after_memcpy(
-; CHECK-SAME: i64 [[X:%.*]]) {
-; CHECK-NEXT:  [[ENTRY:.*:]]
-; CHECK-NEXT:    ret ptr null
-;
-entry:
-  %data = alloca [8 x ptr], align 8
-  call void @llvm.memcpy.p0.p0.i64(ptr align 8 %data, ptr align 8 @test.ptrdata, i64 64, i1 false)
-  %arrayidx = getelementptr inbounds [8 x ptr], ptr %data, i64 0, i64 %x
-  %l = load ptr, ptr %arrayidx, align 8, !nonnull !5
-  ret ptr %l
-}
-
-!5 = !{}
-
-; Verify invariant.load metadata on a cloned load is preserved. OK
-define i32 @copy_invariant_load_metadata_after_memcpy(i64 %x) {
-; CHECK-LABEL: define i32 @copy_invariant_load_metadata_after_memcpy(
-; CHECK-SAME: i64 [[X:%.*]]) {
-; CHECK-NEXT:  [[ENTRY:.*:]]
-; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr @test.data, i64 [[X]]
-; CHECK-NEXT:    [[L:%.*]] = load i32, ptr [[ARRAYIDX]], align 4, !invariant.load [[META4:![0-9]+]]
-; CHECK-NEXT:    ret i32 [[L]]
-;
-entry:
-  %data = alloca [8 x i32], align 4
-  call void @llvm.memcpy.p0.p0.i64(ptr align 4 %data, ptr align 4 @test.data, i64 32, i1 false)
-  %arrayidx = getelementptr inbounds [8 x i32], ptr %data, i64 0, i64 %x
-  %l = load i32, ptr %arrayidx, align 4, !invariant.load !5
-  ret i32 %l
-}
-
-; Verify alias.scope and noalias metadata on a cloned load are preserved. OK
-define i32 @copy_aliasscope_noalias_metadata_after_memcpy(i64 %x) {
-; CHECK-LABEL: define i32 @copy_aliasscope_noalias_metadata_after_memcpy(
-; CHECK-SAME: i64 [[X:%.*]]) {
-; CHECK-NEXT:  [[ENTRY:.*:]]
-; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr @test.data, i64 [[X]]
-; CHECK-NEXT:    [[L:%.*]] = load i32, ptr [[ARRAYIDX]], align 4, !alias.scope [[META5:![0-9]+]], !noalias [[META5]]
-; CHECK-NEXT:    ret i32 [[L]]
-;
-entry:
-  %data = alloca [8 x i32], align 4
-  call void @llvm.memcpy.p0.p0.i64(ptr align 4 %data, ptr align 4 @test.data, i64 32, i1 false)
-  %arrayidx = getelementptr inbounds [8 x i32], ptr %data, i64 0, i64 %x
-  %l = load i32, ptr %arrayidx, align 4, !alias.scope !6, !noalias !6
-  ret i32 %l
-}
-
-; Verify nontemporal metadata on a cloned load is preserved.OK
-define i32 @copy_nontemporal_metadata_after_memcpy(i64 %x) {
-; CHECK-LABEL: define i32 @copy_nontemporal_metadata_after_memcpy(
-; CHECK-SAME: i64 [[X:%.*]]) {
-; CHECK-NEXT:  [[ENTRY:.*:]]
-; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr @test.data, i64 [[X]]
-; CHECK-NEXT:    [[L:%.*]] = load i32, ptr [[ARRAYIDX]], align 4, !nontemporal [[META8:![0-9]+]]
-; CHECK-NEXT:    ret i32 [[L]]
-;
-entry:
-  %data = alloca [8 x i32], align 4
-  call void @llvm.memcpy.p0.p0.i64(ptr align 4 %data, ptr align 4 @test.data, i64 32, i1 false)
-  %arrayidx = getelementptr inbounds [8 x i32], ptr %data, i64 0, i64 %x
-  %l = load i32, ptr %arrayidx, align 4, !nontemporal !9
-  ret i32 %l
-}
-
-; Verify access group metadata on a cloned load is preserved. OK
-define i32 @copy_access_group_metadata_after_memcpy(i64 %x) {
-; CHECK-LABEL: define i32 @copy_access_group_metadata_after_memcpy(
-; CHECK-SAME: i64 [[X:%.*]]) {
-; CHECK-NEXT:  [[ENTRY:.*:]]
-; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr @test.data, i64 [[X]]
-; CHECK-NEXT:    [[L:%.*]] = load i32, ptr [[ARRAYIDX]], align 4, !llvm.access.group [[ACC_GRP9:![0-9]+]]
-; CHECK-NEXT:    ret i32 [[L]]
-;
-entry:
-  %data = alloca [8 x i32], align 4
-  call void @llvm.memcpy.p0.p0.i64(ptr align 4 %data, ptr align 4 @test.data, i64 32, i1 false)
-  %arrayidx = getelementptr inbounds [8 x i32], ptr %data, i64 0, i64 %x
-  %l = load i32, ptr %arrayidx, align 4, !llvm.access.group !10
-  ret i32 %l
-}
-
-; Verify noalias.addrspace metadata on a cloned load is preserved.
-define i32 @copy_noalias_addrspace_metadata_after_memcpy(i64 %x) {
-; CHECK-LABEL: define i32 @copy_noalias_addrspace_metadata_after_memcpy(
-; CHECK-SAME: i64 [[X:%.*]]) {
-; CHECK-NEXT:  [[ENTRY:.*:]]
-; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr @test.data, i64 [[X]]
-; CHECK-NEXT:    [[L:%.*]] = load i32, ptr [[ARRAYIDX]], align 4, !noalias.addrspace [[META10:![0-9]+]]
-; CHECK-NEXT:    ret i32 [[L]]
-;
-entry:
-  %data = alloca [8 x i32], align 4
-  call void @llvm.memcpy.p0.p0.i64(ptr align 4 %data, ptr align 4 @test.data, i64 32, i1 false)
-  %arrayidx = getelementptr inbounds [8 x i32], ptr %data, i64 0, i64 %x
-  %l = load i32, ptr %arrayidx, align 4, !noalias.addrspace !12
-  ret i32 %l
-}
-
-; Verify llvm.mem.parallel_loop_access metadata on a cloned load is preserved. OK
-define i32 @copy_mem_parallel_loop_access_metadata_after_memcpy(i64 %x) {
-; CHECK-LABEL: define i32 @copy_mem_parallel_loop_access_metadata_after_memcpy(
-; CHECK-SAME: i64 [[X:%.*]]) {
-; CHECK-NEXT:  [[ENTRY:.*:]]
-; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr @test.data, i64 [[X]]
-; CHECK-NEXT:    [[L:%.*]] = load i32, ptr [[ARRAYIDX]], align 4, !llvm.mem.parallel_loop_access [[META11:![0-9]+]]
-; CHECK-NEXT:    ret i32 [[L]]
-;
-entry:
-  %data = alloca [8 x i32], align 4
-  call void @llvm.memcpy.p0.p0.i64(ptr align 4 %data, ptr align 4 @test.data, i64 32, i1 false)
-  %arrayidx = getelementptr inbounds [8 x i32], ptr %data, i64 0, i64 %x
-  %l = load i32, ptr %arrayidx, align 4, !llvm.mem.parallel_loop_access !13
-  ret i32 %l
-}
-
-!6 = !{!7}
-!7 = distinct !{!7, !8}
-!8 = distinct !{!8}
-!9 = !{i32 1}
-!10 = distinct !{}
-!12 = !{i32 5, i32 6}
-!13 = !{!14}
-!14 = distinct !{}
-
-
-
-;.
-; CHECK: [[RNG0]] = !{i32 0, i32 100}
-; CHECK: [[SCALAR_TYPE_TBAA1]] = !{[[META2:![0-9]+]], [[META2]], i64 0}
-; CHECK: [[META2]] = !{!"scalar type", [[META3:![0-9]+]]}
-; CHECK: [[META3]] = !{!"root"}
-; CHECK: [[META4]] = !{}
-; CHECK: [[META5]] = !{[[META6:![0-9]+]]}
-; CHECK: [[META6]] = distinct !{[[META6]], [[META7:![0-9]+]]}
-; CHECK: [[META7]] = distinct !{[[META7]]}
-; CHECK: [[META8]] = !{i32 1}
-; CHECK: [[ACC_GRP9]] = distinct !{}
-; CHECK: [[META10]] = !{i32 5, i32 6}
-; CHECK: [[META11]] = !{[[META12:![0-9]+]]}
-; CHECK: [[META12]] = distinct !{}
-;.
diff --git a/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/copy-metadata-load-store.ll b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/copy-metadata-load-store.ll
deleted file mode 100644
index 7cb74c3cc2d2c..0000000000000
--- a/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/copy-metadata-load-store.ll
+++ /dev/null
@@ -1,159 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
-; RUN: opt -mtriple=amdgcn-amd-amdhsa -passes=load-store-vectorizer -S -o - %s | FileCheck %s
-
-; We expect the merged vector load to retain nontemporal and tbaa, and normalization to handle
-; other load-only metadata.
-define void @lsv_copy_load_metadata(ptr %p) {
-; CHECK-LABEL: define void @lsv_copy_load_metadata(
-; CHECK-SAME: ptr [[P:%.*]]) {
-; CHECK-NEXT:  [[ENTRY:.*:]]
-; CHECK-NEXT:    [[TMP0:%.*]] = load <2 x i32>, ptr [[P]], align 4, !tbaa [[CHAR_TBAA0:![0-9]+]], !invariant.load [[META3:![0-9]+]], !nontemporal [[META4:![0-9]+]]
-; CHECK-NEXT:    [[LD01:%.*]] = extractelement <2 x i32> [[TMP0]], i32 0
-; CHECK-NEXT:    [[LD1_MUT2:%.*]] = extractelement <2 x i32> [[TMP0]], i32 1
-; CHECK-NEXT:    [[LD1_MUT_BC:%.*]] = bitcast i32 [[LD1_MUT2]] to <2 x i16>
-; CHECK-NEXT:    ret void
-;
-entry:
-  %p1 = getelementptr i32, ptr %p, i64 1
-  %ld0 = load i32, ptr %p, align 4, !tbaa !0, !nontemporal !5, !invariant.load !6
-  %ld1 = load <2 x i16>, ptr %p1, align 4, !tbaa !0, !nontemporal !5, !invariant.load !6
-  ret void
-}
-
-; Check that metadata on stores is preserved when LSV normalizes mixed-typed
-; chains (exercises copyMetadataForAccess on stores).
-define void @lsv_copy_store_metadata(ptr %p) {
-; CHECK-LABEL: define void @lsv_copy_store_metadata(
-; CHECK-SAME: ptr [[P:%.*]]) {
-; CHECK-NEXT:  [[ENTRY:.*:]]
-; CHECK-NEXT:    store <2 x i32> <i32 7, i32 bitcast (<2 x i16> <i16 4, i16 5> to i32)>, ptr [[P]], align 4, !nontemporal [[META4]]
-; CHECK-NEXT:    ret void
-;
-entry:
-  %p1 = getelementptr i32, ptr %p, i64 1
-  store i32 7, ptr %p, align 4, !nontemporal !5
-  store <2 x i16> <i16 4, i16 5>, ptr %p1, align 4, !nontemporal !5
-  ret void
-}
-
-; Copy alias.scope and noalias metadata on vectorized stores.
-define void @lsv_copy_store_alias_metadata(ptr %p) {
-; CHECK-LABEL: define void @lsv_copy_store_alias_metadata(
-; CHECK-SAME: ptr [[P:%.*]]) {
-; CHECK-NEXT:  [[ENTRY:.*:]]
-; CHECK-NEXT:    store <2 x i32> <i32 1, i32 bitcast (<2 x i16> <i16 2, i16 3> to i32)>, ptr [[P]], align 4, !alias.scope [[META5:![0-9]+]], !noalias [[META5]]
-; CHECK-NEXT:    ret void
-;
-entry:
-  %p1 = getelementptr i32, ptr %p, i64 1
-  store i32 1, ptr %p, align 4, !alias.scope !11, !noalias !11
-  store <2 x i16> <i16 2, i16 3>, ptr %p1, align 4, !alias.scope !11, !noalias !11
-  ret void
-}
-
-; Copy access group metadata on vectorized stores.
-define void @lsv_copy_store_access_group(ptr %p) {
-; CHECK-LABEL: define void @lsv_copy_store_access_group(
-; CHECK-SAME: ptr [[P:%.*]]) {
-; CHECK-NEXT:  [[ENTRY:.*:]]
-; CHECK-NEXT:    store <2 x i32> <i32 9, i32 bitcast (<2 x i16> <i16 8, i16 7> to i32)>, ptr [[P]], align 4
-; CHECK-NEXT:    ret void
-;
-entry:
-  %p1 = getelementptr i32, ptr %p, i64 1
-  store i32 9, ptr %p, align 4, !llvm.access.group !14
-  store <2 x i16> <i16 8, i16 7>, ptr %p1, align 4, !llvm.access.group !14
-  ret void
-}
-
-; Copy noundef metadata on vectorized stores.
-define void @lsv_copy_store_noundef(ptr %p) {
-; CHECK-LABEL: define void @lsv_copy_store_noundef(
-; CHECK-SAME: ptr [[P:%.*]]) {
-; CHECK-NEXT:  [[ENTRY:.*:]]
-; CHECK-NEXT:    store <2 x i32> <i32 42, i32 bitcast (<2 x i16> <i16 6, i16 5> to i32)>, ptr [[P]], align 4
-; CHECK-NEXT:    ret void
-;
-entry:
-  %p1 = getelementptr i32, ptr %p, i64 1
-  store i32 42, ptr %p, align 4, !noundef !15
-  store <2 x i16> <i16 6, i16 5>, ptr %p1, align 4, !noundef !15
-  ret void
-}
-
-; Copy noalias.addrspace metadata on vectorized stores.
-define void @lsv_copy_store_noalias_addrspace(ptr %p) {
-; CHECK-LABEL: define void @lsv_copy_store_noalias_addrspace(
-; CHECK-SAME: ptr [[P:%.*]]) {
-; CHECK-NEXT:  [[ENTRY:.*:]]
-; CHECK-NEXT:    store <2 x i32> <i32 11, i32 bitcast (<2 x i16> <i16 10, i16 9> to i32)>, ptr [[P]], align 4
-; CHECK-NEXT:    ret void
-;
-entry:
-  %p1 = getelementptr i32, ptr %p, i64 1
-  store i32 11, ptr %p, align 4, !noalias.addrspace !16
-  store <2 x i16> <i16 10, i16 9>, ptr %p1, align 4, !noalias.addrspace !16
-  ret void
-}
-
-; Copy llvm.mem.parallel_loop_access metadata on vectorized stores.
-define void @lsv_copy_store_mem_parallel_loop_access(ptr %p) {
-; CHECK-LABEL: define void @lsv_copy_store_mem_parallel_loop_access(
-; CHECK-SAME: ptr [[P:%.*]]) {
-; CHECK-NEXT:  [[ENTRY:.*:]]
-; CHECK-NEXT:    store <2 x i32> <i32 13, i32 bitcast (<2 x i16> <i16 12, i16 11> to i32)>, ptr [[P]], align 4
-; CHECK-NEXT:    ret void
-;
-entry:
-  %p1 = getelementptr i32, ptr %p, i64 1
-  store i32 13, ptr %p, align 4, !llvm.mem.parallel_loop_access !17
-  store <2 x i16> <i16 12, i16 11>, ptr %p1, align 4, !llvm.mem.parallel_loop_access !17
-  ret void
-}
-
-; Normalized type is not a pointer in the following test, avoid copying
-; dereferenceable_or_null metadata.
-define void @lsv_no_copy_deref_or_null(ptr %p) {
-; CHECK-LABEL: define void @lsv_no_copy_deref_or_null(
-; CHECK-SAME: ptr [[P:%.*]]) {
-; CHECK-NEXT:  [[ENTRY:.*:]]
-; CHECK-NEXT:    [[TMP0:%.*]] = load <2 x i64>, ptr [[P]], align 8
-; CHECK-NEXT:    [[LD0_MUT1:%.*]] = extractelement <2 x i64> [[TMP0]], i32 0
-; CHECK-NEXT:    [[LD12:%.*]] = extractelement <2 x i64> [[TMP0]], i32 1
-; CHECK-NEXT:    [[LD0_MUT_BC:%.*]] = inttoptr i64 [[LD0_MUT1]] to ptr
-; CHECK-NEXT:    ret void
-;
-entry:
-  %p1 = getelementptr i32, ptr %p, i64 1
-  %ld0 = load ptr, ptr %p, align 4, !dereferenceable_or_null !7
-  %ld1 = load i64, ptr %p1, align 4
-  ret void
-}
-
-!0 = !{!3, !3, i64 0}
-!3 = !{!"omnipotent char", !4, i64 0}
-!4 = !{!"Simple C/C++ TBAA"}
-!5 = !{i32 1}
-!6 = !{}
-!7 = !{i64 8}
-!8 = !{i64 1, i64 256}
-!11 = !{!12}
-!12 = distinct !{!12, !13}
-!13 = distinct !{!13}
-!14 = distinct !{}
-!15 = !{}
-!16 = !{i32 5, i32 6}
-!17 = !{!18}
-!18 = distinct !{}
-attributes #0 = { nounwind }
-attributes #1 = { nounwind readnone }
-;.
-; CHECK: [[CHAR_TBAA0]] = !{[[META1:![0-9]+]], [[META1]], i64 0}
-; CHECK: [[META1]] = !{!"omnipotent char", [[META2:![0-9]+]], i64 0}
-; CHECK: [[META2]] = !{!"Simple C/C++ TBAA"}
-; CHECK: [[META3]] = !{}
-; CHECK: [[META4]] = !{i32 1}
-; CHECK: [[META5]] = !{[[META6:![0-9]+]]}
-; CHECK: [[META6]] = distinct !{[[META6]], [[META7:![0-9]+]]}
-; CHECK: [[META7]] = distinct !{[[META7]]}
-;.
diff --git a/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/merge-vectors-complex.ll b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/merge-vectors-complex.ll
index 64e8b1afb8c80..c53f4b6d7ff2b 100644
--- a/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/merge-vectors-complex.ll
+++ b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/merge-vectors-complex.ll
@@ -1,273 +1,57 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
 ; RUN: opt -mtriple=amdgcn-amd-amdhsa -passes=load-store-vectorizer -S -o - %s | FileCheck %s
 
-define void @no_merge_i16_half(ptr addrspace(1) %ptr1, ptr addrspace(2) %ptr2) {
-; CHECK-LABEL: define void @no_merge_i16_half(
+define void @merge_i32_v2i16_f32_v4i8(ptr addrspace(1) %ptr1, ptr addrspace(2) %ptr2) {
+; CHECK-LABEL: define void @merge_i32_v2i16_f32_v4i8(
 ; CHECK-SAME: ptr addrspace(1) [[PTR1:%.*]], ptr addrspace(2) [[PTR2:%.*]]) {
-; CHECK-NEXT:    [[GEP_1:%.*]] = getelementptr inbounds i16, ptr addrspace(1) [[PTR1]], i64 1
-; CHECK-NEXT:    [[LOAD_0:%.*]] = load i16, ptr addrspace(1) [[PTR1]], align 2
-; CHECK-NEXT:    [[LOAD_1:%.*]] = load half, ptr addrspace(1) [[GEP_1]], align 2
-; CHECK-NEXT:    [[STORE_GEP_1:%.*]] = getelementptr inbounds i16, ptr addrspace(2) [[PTR2]], i64 1
-; CHECK-NEXT:    store i16 [[LOAD_0]], ptr addrspace(2) [[PTR2]], align 2
-; CHECK-NEXT:    store half [[LOAD_1]], ptr addrspace(2) [[STORE_GEP_1]], align 2
-; CHECK-NEXT:    ret void
-;
-  %gep.1 = getelementptr inbounds i16, ptr addrspace(1) %ptr1, i64 1
-  %load.0 = load i16, ptr addrspace(1) %ptr1
-  %load.1 = load half, ptr addrspace(1) %gep.1
-  %store.gep.1 = getelementptr inbounds i16, ptr addrspace(2) %ptr2, i64 1
-  store i16 %load.0, ptr addrspace(2) %ptr2
-  store half %load.1, ptr addrspace(2) %store.gep.1
-  ret void
-}
-
-define void @no_merge_i16_float(ptr addrspace(1) %ptr1, ptr addrspace(2) %ptr2) {
-; CHECK-LABEL: define void @no_merge_i16_float(
-; CHECK-SAME: ptr addrspace(1) [[PTR1:%.*]], ptr addrspace(2) [[PTR2:%.*]]) {
-; CHECK-NEXT:    [[GEP_1:%.*]] = getelementptr inbounds i16, ptr addrspace(1) [[PTR1]], i64 1
-; CHECK-NEXT:    [[LOAD_0:%.*]] = load i16, ptr addrspace(1) [[PTR1]], align 2
-; CHECK-NEXT:    [[LOAD_1:%.*]] = load float, ptr addrspace(1) [[GEP_1]], align 4
-; CHECK-NEXT:    [[STORE_GEP_1:%.*]] = getelementptr inbounds i16, ptr addrspace(2) [[PTR2]], i64 1
-; CHECK-NEXT:    store i16 [[LOAD_0]], ptr addrspace(2) [[PTR2]], align 2
-; CHECK-NEXT:    store float [[LOAD_1]], ptr addrspace(2) [[STORE_GEP_1]], align 4
-; CHECK-NEXT:    ret void
-;
-  %gep.1 = getelementptr inbounds i16, ptr addrspace(1) %ptr1, i64 1
-  %load.0 = load i16, ptr addrspace(1) %ptr1
-  %load.1 = load float, ptr addrspace(1) %gep.1
-  %store.gep.1 = getelementptr inbounds i16, ptr addrspace(2) %ptr2, i64 1
-  store i16 %load.0, ptr addrspace(2) %ptr2
-  store float %load.1, ptr addrspace(2) %store.gep.1
-  ret void
-}
-
-define void @merge_i32_v2i16(ptr addrspace(1) %ptr1, ptr addrspace(2) %ptr2) {
-; CHECK-LABEL: define void @merge_i32_v2i16(
-; CHECK-SAME: ptr addrspace(1) [[PTR1:%.*]], ptr addrspace(2) [[PTR2:%.*]]) {
-; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i32>, ptr addrspace(1) [[PTR1]], align 4
-; CHECK-NEXT:    [[LOAD_01:%.*]] = extractelement <2 x i32> [[TMP1]], i32 0
-; CHECK-NEXT:    [[LOAD_1_MUT2:%.*]] = extractelement <2 x i32> [[TMP1]], i32 1
-; CHECK-NEXT:    [[LOAD_1_MUT_BC:%.*]] = bitcast i32 [[LOAD_1_MUT2]] to <2 x i16>
-; CHECK-NEXT:    [[LOAD_1_BC:%.*]] = bitcast <2 x i16> [[LOAD_1_MUT_BC]] to i32
-; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x i32> poison, i32 [[LOAD_01]], i32 0
-; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <2 x i32> [[TMP2]], i32 [[LOAD_1_BC]], i32 1
-; CHECK-NEXT:    store <2 x i32> [[TMP3]], ptr addrspace(2) [[PTR2]], align 4
-; CHECK-NEXT:    ret void
-;
-  %gep.1 = getelementptr inbounds i32, ptr addrspace(1) %ptr1, i64 1
-  %load.0 = load i32, ptr addrspace(1) %ptr1
-  %load.1 = load <2 x i16>, ptr addrspace(1) %gep.1
-  %store.gep.1 = getelementptr inbounds i32, ptr addrspace(2) %ptr2, i64 1
-  store i32 %load.0, ptr addrspace(2) %ptr2
-  store <2 x i16> %load.1, ptr addrspace(2) %store.gep.1
-  ret void
-}
-
-define void @no_merge_i32_ptr(ptr addrspace(1) %ptr1, ptr addrspace(2) %ptr2) {
-; CHECK-LABEL: define void @no_merge_i32_ptr(
-; CHECK-SAME: ptr addrspace(1) [[PTR1:%.*]], ptr addrspace(2) [[PTR2:%.*]]) {
-; CHECK-NEXT:    [[GEP_1:%.*]] = getelementptr inbounds i32, ptr addrspace(1) [[PTR1]], i64 1
-; CHECK-NEXT:    [[LOAD_0:%.*]] = load i32, ptr addrspace(1) [[PTR1]], align 4
-; CHECK-NEXT:    [[LOAD_1:%.*]] = load ptr, ptr addrspace(1) [[GEP_1]], align 8
-; CHECK-NEXT:    [[STORE_GEP_1:%.*]] = getelementptr inbounds i32, ptr addrspace(2) [[PTR2]], i64 1
-; CHECK-NEXT:    store i32 [[LOAD_0]], ptr addrspace(2) [[PTR2]], align 4
-; CHECK-NEXT:    store ptr [[LOAD_1]], ptr addrspace(2) [[STORE_GEP_1]], align 8
-; CHECK-NEXT:    ret void
-;
-  %gep.1 = getelementptr inbounds i32, ptr addrspace(1) %ptr1, i64 1
-  %load.0 = load i32, ptr addrspace(1) %ptr1
-  %load.1 = load ptr, ptr addrspace(1) %gep.1
-  %store.gep.1 = getelementptr inbounds i32, ptr addrspace(2) %ptr2, i64 1
-  store i32 %load.0, ptr addrspace(2) %ptr2
-  store ptr %load.1, ptr addrspace(2) %store.gep.1
-  ret void
-}
-
-define void @no_merge_i32_half(ptr addrspace(1) %ptr1, ptr addrspace(2) %ptr2) {
-; CHECK-LABEL: define void @no_merge_i32_half(
-; CHECK-SAME: ptr addrspace(1) [[PTR1:%.*]], ptr addrspace(2) [[PTR2:%.*]]) {
-; CHECK-NEXT:    [[GEP_1:%.*]] = getelementptr inbounds i32, ptr addrspace(1) [[PTR1]], i64 1
-; CHECK-NEXT:    [[LOAD_0:%.*]] = load i32, ptr addrspace(1) [[PTR1]], align 4
-; CHECK-NEXT:    [[LOAD_1:%.*]] = load half, ptr addrspace(1) [[GEP_1]], align 2
-; CHECK-NEXT:    [[STORE_GEP_1:%.*]] = getelementptr inbounds i32, ptr addrspace(2) [[PTR2]], i64 1
-; CHECK-NEXT:    store i32 [[LOAD_0]], ptr addrspace(2) [[PTR2]], align 4
-; CHECK-NEXT:    store half [[LOAD_1]], ptr addrspace(2) [[STORE_GEP_1]], align 2
-; CHECK-NEXT:    ret void
-;
-  %gep.1 = getelementptr inbounds i32, ptr addrspace(1) %ptr1, i64 1
-  %load.0 = load i32, ptr addrspace(1) %ptr1
-  %load.1 = load half, ptr addrspace(1) %gep.1
-  %store.gep.1 = getelementptr inbounds i32, ptr addrspace(2) %ptr2, i64 1
-  store i32 %load.0, ptr addrspace(2) %ptr2
-  store half %load.1, ptr addrspace(2) %store.gep.1
-  ret void
-}
-
-define void @merge_i32_float(ptr addrspace(1) %ptr1, ptr addrspace(2) %ptr2) {
-; CHECK-LABEL: define void @merge_i32_float(
-; CHECK-SAME: ptr addrspace(1) [[PTR1:%.*]], ptr addrspace(2) [[PTR2:%.*]]) {
-; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i32>, ptr addrspace(1) [[PTR1]], align 4
-; CHECK-NEXT:    [[LOAD_01:%.*]] = extractelement <2 x i32> [[TMP1]], i32 0
-; CHECK-NEXT:    [[LOAD_12:%.*]] = extractelement <2 x i32> [[TMP1]], i32 1
-; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i32 [[LOAD_12]] to float
-; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <2 x i32> poison, i32 [[LOAD_01]], i32 0
-; CHECK-NEXT:    [[TMP4:%.*]] = bitcast float [[TMP2]] to i32
-; CHECK-NEXT:    [[TMP5:%.*]] = insertelement <2 x i32> [[TMP3]], i32 [[TMP4]], i32 1
-; CHECK-NEXT:    store <2 x i32> [[TMP5]], ptr addrspace(2) [[PTR2]], align 4
-; CHECK-NEXT:    ret void
-;
-  %gep.1 = getelementptr inbounds i32, ptr addrspace(1) %ptr1, i64 1
-  %load.0 = load i32, ptr addrspace(1) %ptr1
-  %load.1 = load float, ptr addrspace(1) %gep.1
-  %store.gep.1 = getelementptr inbounds i32, ptr addrspace(2) %ptr2, i64 1
-  store i32 %load.0, ptr addrspace(2) %ptr2
-  store float %load.1, ptr addrspace(2) %store.gep.1
-  ret void
-}
-
-define void @no_merge_i32_double(ptr addrspace(1) %ptr1, ptr addrspace(2) %ptr2) {
-; CHECK-LABEL: define void @no_merge_i32_double(
-; CHECK-SAME: ptr addrspace(1) [[PTR1:%.*]], ptr addrspace(2) [[PTR2:%.*]]) {
-; CHECK-NEXT:    [[GEP_1:%.*]] = getelementptr inbounds i32, ptr addrspace(1) [[PTR1]], i64 1
-; CHECK-NEXT:    [[LOAD_0:%.*]] = load i32, ptr addrspace(1) [[PTR1]], align 4
-; CHECK-NEXT:    [[LOAD_1:%.*]] = load double, ptr addrspace(1) [[GEP_1]], align 8
-; CHECK-NEXT:    [[STORE_GEP_1:%.*]] = getelementptr inbounds i32, ptr addrspace(2) [[PTR2]], i64 1
-; CHECK-NEXT:    store i32 [[LOAD_0]], ptr addrspace(2) [[PTR2]], align 4
-; CHECK-NEXT:    store double [[LOAD_1]], ptr addrspace(2) [[STORE_GEP_1]], align 8
-; CHECK-NEXT:    ret void
-;
-  %gep.1 = getelementptr inbounds i32, ptr addrspace(1) %ptr1, i64 1
-  %load.0 = load i32, ptr addrspace(1) %ptr1
-  %load.1 = load double, ptr addrspace(1) %gep.1
-  %store.gep.1 = getelementptr inbounds i32, ptr addrspace(2) %ptr2, i64 1
-  store i32 %load.0, ptr addrspace(2) %ptr2
-  store double %load.1, ptr addrspace(2) %store.gep.1
-  ret void
-}
-
-define void @merge_i64_ptr(ptr addrspace(1) %ptr1, ptr addrspace(2) %ptr2) {
-; CHECK-LABEL: define void @merge_i64_ptr(
-; CHECK-SAME: ptr addrspace(1) [[PTR1:%.*]], ptr addrspace(2) [[PTR2:%.*]]) {
-; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr addrspace(1) [[PTR1]], align 8
-; CHECK-NEXT:    [[LOAD_01:%.*]] = extractelement <2 x i64> [[TMP1]], i32 0
-; CHECK-NEXT:    [[LOAD_12:%.*]] = extractelement <2 x i64> [[TMP1]], i32 1
-; CHECK-NEXT:    [[TMP2:%.*]] = inttoptr i64 [[LOAD_12]] to ptr
-; CHECK-NEXT:    [[STORE_GEP_1:%.*]] = getelementptr inbounds i64, ptr addrspace(2) [[PTR2]], i64 1
-; CHECK-NEXT:    store i64 [[LOAD_01]], ptr addrspace(2) [[PTR2]], align 8
-; CHECK-NEXT:    store ptr [[TMP2]], ptr addrspace(2) [[STORE_GEP_1]], align 8
-; CHECK-NEXT:    ret void
-;
-  %gep.1 = getelementptr inbounds i64, ptr addrspace(1) %ptr1, i64 1
-  %load.0 = load i64, ptr addrspace(1) %ptr1
-  %load.1 = load ptr, ptr addrspace(1) %gep.1
-  %store.gep.1 = getelementptr inbounds i64, ptr addrspace(2) %ptr2, i64 1
-  store i64 %load.0, ptr addrspace(2) %ptr2
-  store ptr %load.1, ptr addrspace(2) %store.gep.1
-  ret void
-}
-
-define void @no_merge_i64_float(ptr addrspace(1) %ptr1, ptr addrspace(2) %ptr2) {
-; CHECK-LABEL: define void @no_merge_i64_float(
-; CHECK-SAME: ptr addrspace(1) [[PTR1:%.*]], ptr addrspace(2) [[PTR2:%.*]]) {
-; CHECK-NEXT:    [[GEP_1:%.*]] = getelementptr inbounds i64, ptr addrspace(1) [[PTR1]], i64 1
-; CHECK-NEXT:    [[LOAD_0:%.*]] = load i64, ptr addrspace(1) [[PTR1]], align 8
-; CHECK-NEXT:    [[LOAD_1:%.*]] = load float, ptr addrspace(1) [[GEP_1]], align 4
-; CHECK-NEXT:    [[STORE_GEP_1:%.*]] = getelementptr inbounds i64, ptr addrspace(2) [[PTR2]], i64 1
-; CHECK-NEXT:    store i64 [[LOAD_0]], ptr addrspace(2) [[PTR2]], align 8
-; CHECK-NEXT:    store float [[LOAD_1]], ptr addrspace(2) [[STORE_GEP_1]], align 4
-; CHECK-NEXT:    ret void
-;
-  %gep.1 = getelementptr inbounds i64, ptr addrspace(1) %ptr1, i64 1
-  %load.0 = load i64, ptr addrspace(1) %ptr1
-  %load.1 = load float, ptr addrspace(1) %gep.1
-  %store.gep.1 = getelementptr inbounds i64, ptr addrspace(2) %ptr2, i64 1
-  store i64 %load.0, ptr addrspace(2) %ptr2
-  store float %load.1, ptr addrspace(2) %store.gep.1
-  ret void
-}
-
-define void @merge_i64_double(ptr addrspace(1) %ptr1, ptr addrspace(2) %ptr2) {
-; CHECK-LABEL: define void @merge_i64_double(
-; CHECK-SAME: ptr addrspace(1) [[PTR1:%.*]], ptr addrspace(2) [[PTR2:%.*]]) {
-; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr addrspace(1) [[PTR1]], align 8
-; CHECK-NEXT:    [[LOAD_01:%.*]] = extractelement <2 x i64> [[TMP1]], i32 0
-; CHECK-NEXT:    [[LOAD_12:%.*]] = extractelement <2 x i64> [[TMP1]], i32 1
-; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i64 [[LOAD_12]] to double
-; CHECK-NEXT:    [[STORE_GEP_1:%.*]] = getelementptr inbounds i64, ptr addrspace(2) [[PTR2]], i64 1
-; CHECK-NEXT:    store i64 [[LOAD_01]], ptr addrspace(2) [[PTR2]], align 8
-; CHECK-NEXT:    store double [[TMP2]], ptr addrspace(2) [[STORE_GEP_1]], align 8
-; CHECK-NEXT:    ret void
-;
-  %gep.1 = getelementptr inbounds i64, ptr addrspace(1) %ptr1, i64 1
-  %load.0 = load i64, ptr addrspace(1) %ptr1
-  %load.1 = load double, ptr addrspace(1) %gep.1
-  %store.gep.1 = getelementptr inbounds i64, ptr addrspace(2) %ptr2, i64 1
-  store i64 %load.0, ptr addrspace(2) %ptr2
-  store double %load.1, ptr addrspace(2) %store.gep.1
-  ret void
-}
-
-define void @merge_i64_v2i32(ptr addrspace(1) %ptr1, ptr addrspace(2) %ptr2) {
-; CHECK-LABEL: define void @merge_i64_v2i32(
-; CHECK-SAME: ptr addrspace(1) [[PTR1:%.*]], ptr addrspace(2) [[PTR2:%.*]]) {
-; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr addrspace(1) [[PTR1]], align 8
-; CHECK-NEXT:    [[LOAD_01:%.*]] = extractelement <2 x i64> [[TMP1]], i32 0
-; CHECK-NEXT:    [[LOAD_1_MUT2:%.*]] = extractelement <2 x i64> [[TMP1]], i32 1
-; CHECK-NEXT:    [[LOAD_1_MUT_BC:%.*]] = bitcast i64 [[LOAD_1_MUT2]] to <2 x i32>
-; CHECK-NEXT:    [[STORE_GEP_1:%.*]] = getelementptr inbounds i64, ptr addrspace(2) [[PTR2]], i64 1
-; CHECK-NEXT:    store i64 [[LOAD_01]], ptr addrspace(2) [[PTR2]], align 8
-; CHECK-NEXT:    [[LOAD_1_BC:%.*]] = bitcast <2 x i32> [[LOAD_1_MUT_BC]] to i64
-; CHECK-NEXT:    store i64 [[LOAD_1_BC]], ptr addrspace(2) [[STORE_GEP_1]], align 8
-; CHECK-NEXT:    ret void
-;
-  %gep.1 = getelementptr inbounds i64, ptr addrspace(1) %ptr1, i64 1
-  %load.0 = load i64, ptr addrspace(1) %ptr1
-  %load.1 = load <2 x i32>, ptr addrspace(1) %gep.1
-  %store.gep.1 = getelementptr inbounds i64, ptr addrspace(2) %ptr2, i64 1
-  store i64 %load.0, ptr addrspace(2) %ptr2
-  store <2 x i32> %load.1, ptr addrspace(2) %store.gep.1
-  ret void
-}
-
-define void @merge_i32_v2i16_v4i8(ptr addrspace(1) %ptr1) {
-; CHECK-LABEL: define void @merge_i32_v2i16_v4i8(
-; CHECK-SAME: ptr addrspace(1) [[PTR1:%.*]]) {
-; CHECK-NEXT:    [[LOAD_0:%.*]] = load i32, ptr addrspace(1) [[PTR1]], align 4
-; CHECK-NEXT:    [[GEP2:%.*]] = getelementptr inbounds i32, ptr addrspace(1) [[PTR1]], i64 1
-; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i32>, ptr addrspace(1) [[GEP2]], align 4
-; CHECK-NEXT:    [[LOAD2_MUT1:%.*]] = extractelement <2 x i32> [[TMP1]], i32 0
-; CHECK-NEXT:    [[LOAD4_MUT2:%.*]] = extractelement <2 x i32> [[TMP1]], i32 1
-; CHECK-NEXT:    [[LOAD2_MUT_BC:%.*]] = bitcast i32 [[LOAD2_MUT1]] to <2 x i16>
-; CHECK-NEXT:    [[LOAD4_MUT_BC:%.*]] = bitcast i32 [[LOAD4_MUT2]] to <4 x i8>
-; CHECK-NEXT:    [[GEP_3:%.*]] = getelementptr inbounds i32, ptr addrspace(1) [[PTR1]], i64 3
-; CHECK-NEXT:    [[LOAD_3:%.*]] = load float, ptr addrspace(1) [[GEP_3]], align 4
+; CHECK-NEXT:    [[GEP1:%.*]] = getelementptr inbounds i32, ptr addrspace(1) [[PTR1]], i64 0
+; CHECK-NEXT:    [[LOAD1:%.*]] = load i32, ptr addrspace(1) [[GEP1]], align 4
+; CHECK-NEXT:    [[GEP2:%.*]] = getelementptr inbounds <2 x i16>, ptr addrspace(1) [[PTR1]], i64 1
+; CHECK-NEXT:    [[LOAD2:%.*]] = load <2 x i16>, ptr addrspace(1) [[GEP2]], align 4
+; CHECK-NEXT:    [[GEP3:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[PTR1]], i64 2
+; CHECK-NEXT:    [[LOAD3:%.*]] = load float, ptr addrspace(1) [[GEP3]], align 4
+; CHECK-NEXT:    [[GEP4:%.*]] = getelementptr inbounds <4 x i8>, ptr addrspace(1) [[PTR1]], i64 3
+; CHECK-NEXT:    [[LOAD4:%.*]] = load <4 x i8>, ptr addrspace(1) [[GEP4]], align 4
+; CHECK-NEXT:    [[STORE_GEP1:%.*]] = getelementptr inbounds i32, ptr addrspace(2) [[PTR2]], i64 0
+; CHECK-NEXT:    store i32 [[LOAD1]], ptr addrspace(2) [[STORE_GEP1]], align 4
+; CHECK-NEXT:    [[STORE_GEP2:%.*]] = getelementptr inbounds <2 x i16>, ptr addrspace(2) [[PTR2]], i64 1
+; CHECK-NEXT:    store <2 x i16> [[LOAD2]], ptr addrspace(2) [[STORE_GEP2]], align 4
+; CHECK-NEXT:    [[STORE_GEP3:%.*]] = getelementptr inbounds float, ptr addrspace(2) [[PTR2]], i64 2
+; CHECK-NEXT:    store float [[LOAD3]], ptr addrspace(2) [[STORE_GEP3]], align 4
+; CHECK-NEXT:    [[STORE_GEP4:%.*]] = getelementptr inbounds <4 x i8>, ptr addrspace(2) [[PTR2]], i64 3
+; CHECK-NEXT:    store <4 x i8> [[LOAD4]], ptr addrspace(2) [[STORE_GEP4]], align 4
 ; CHECK-NEXT:    ret void
 ;
-  %load.0 = load i32, ptr addrspace(1) %ptr1, align 4
-  %gep.1 = getelementptr inbounds i32, ptr addrspace(1) %ptr1, i64 1
-  %load.1 = load <2 x i16>, ptr addrspace(1) %gep.1, align 4
-  %gep.2 = getelementptr inbounds i32, ptr addrspace(1) %ptr1, i64 2
-  %load.2 = load <4 x i8>, ptr addrspace(1) %gep.2, align 4
-  %gep.3 = getelementptr inbounds i32, ptr addrspace(1) %ptr1, i64 3
-  %load.3 = load float, ptr addrspace(1) %gep.3, align 4
+  %gep1 = getelementptr inbounds i32, ptr addrspace(1) %ptr1, i64 0
+  %load1 = load i32, ptr addrspace(1) %gep1, align 4
+  %gep2 = getelementptr inbounds <2 x i16>, ptr addrspace(1) %ptr1, i64 1
+  %load2 = load <2 x i16>, ptr addrspace(1) %gep2, align 4
+  %gep3 = getelementptr inbounds float, ptr addrspace(1) %ptr1, i64 2
+  %load3 = load float, ptr addrspace(1) %gep3, align 4
+  %gep4 = getelementptr inbounds <4 x i8>, ptr addrspace(1) %ptr1, i64 3
+  %load4 = load <4 x i8>, ptr addrspace(1) %gep4, align 4
+  %store.gep1 = getelementptr inbounds i32, ptr addrspace(2) %ptr2, i64 0
+  store i32 %load1, ptr addrspace(2) %store.gep1, align 4
+  %store.gep2 = getelementptr inbounds <2 x i16>, ptr addrspace(2) %ptr2, i64 1
+  store <2 x i16> %load2, ptr addrspace(2) %store.gep2, align 4
+  %store.gep3 = getelementptr inbounds float, ptr addrspace(2) %ptr2, i64 2
+  store float %load3, ptr addrspace(2) %store.gep3, align 4
+  %store.gep4 = getelementptr inbounds <4 x i8>, ptr addrspace(2) %ptr2, i64 3
+  store <4 x i8> %load4, ptr addrspace(2) %store.gep4, align 4
   ret void
 }
 
-define void @merge_float_v2f16_type(ptr addrspace(1) %ptr1, ptr addrspace(2) %ptr2) {
-; CHECK-LABEL: define void @merge_float_v2f16_type(
+define void @merge_f32_v2f16_type(ptr addrspace(1) %ptr1, ptr addrspace(2) %ptr2) {
+; CHECK-LABEL: define void @merge_f32_v2f16_type(
 ; CHECK-SAME: ptr addrspace(1) [[PTR1:%.*]], ptr addrspace(2) [[PTR2:%.*]]) {
 ; CHECK-NEXT:    [[GEP1:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[PTR1]], i64 0
-; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i32>, ptr addrspace(1) [[GEP1]], align 4
-; CHECK-NEXT:    [[LOAD1_MUT1:%.*]] = extractelement <2 x i32> [[TMP1]], i32 0
-; CHECK-NEXT:    [[LOAD2_MUT2:%.*]] = extractelement <2 x i32> [[TMP1]], i32 1
-; CHECK-NEXT:    [[LOAD1_TOORIG:%.*]] = bitcast i32 [[LOAD1_MUT1]] to float
-; CHECK-NEXT:    [[LOAD2_TOORIG:%.*]] = bitcast i32 [[LOAD2_MUT2]] to <2 x half>
+; CHECK-NEXT:    [[LOAD1:%.*]] = load float, ptr addrspace(1) [[GEP1]], align 4
+; CHECK-NEXT:    [[GEP2:%.*]] = getelementptr inbounds <2 x half>, ptr addrspace(1) [[PTR1]], i64 1
+; CHECK-NEXT:    [[LOAD2:%.*]] = load <2 x half>, ptr addrspace(1) [[GEP2]], align 4
 ; CHECK-NEXT:    [[STORE_GEP1:%.*]] = getelementptr inbounds i32, ptr addrspace(2) [[PTR2]], i64 0
-; CHECK-NEXT:    [[LOAD1_BC:%.*]] = bitcast float [[LOAD1_TOORIG]] to i32
-; CHECK-NEXT:    [[LOAD2_BC:%.*]] = bitcast <2 x half> [[LOAD2_TOORIG]] to i32
-; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x i32> poison, i32 [[LOAD1_BC]], i32 0
-; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <2 x i32> [[TMP2]], i32 [[LOAD2_BC]], i32 1
-; CHECK-NEXT:    store <2 x i32> [[TMP3]], ptr addrspace(2) [[STORE_GEP1]], align 4
+; CHECK-NEXT:    store float [[LOAD1]], ptr addrspace(2) [[STORE_GEP1]], align 4
+; CHECK-NEXT:    [[STORE_GEP2:%.*]] = getelementptr inbounds <2 x half>, ptr addrspace(2) [[PTR2]], i64 1
+; CHECK-NEXT:    store <2 x half> [[LOAD2]], ptr addrspace(2) [[STORE_GEP2]], align 4
 ; CHECK-NEXT:    ret void
 ;
   %gep1 = getelementptr inbounds float, ptr addrspace(1) %ptr1, i64 0
@@ -304,3 +88,27 @@ define void @merge_v2f16_bfloat_type(ptr addrspace(1) %ptr1, ptr addrspace(2) %p
   store <2 x half> %load2, ptr addrspace(2) %store.gep2, align 4
   ret void
 }
+
+define void @no_merge_mixed_ptr_addrspaces(ptr addrspace(1) %ptr1, ptr addrspace(2) %ptr2) {
+; CHECK-LABEL: define void @no_merge_mixed_ptr_addrspaces(
+; CHECK-SAME: ptr addrspace(1) [[PTR1:%.*]], ptr addrspace(2) [[PTR2:%.*]]) {
+; CHECK-NEXT:    [[GEP1:%.*]] = getelementptr inbounds ptr addrspace(1), ptr addrspace(1) [[PTR1]], i64 0
+; CHECK-NEXT:    [[LOAD1:%.*]] = load ptr addrspace(1), ptr addrspace(1) [[GEP1]], align 4
+; CHECK-NEXT:    [[GEP2:%.*]] = getelementptr inbounds ptr addrspace(2), ptr addrspace(1) [[PTR1]], i64 1
+; CHECK-NEXT:    [[LOAD2:%.*]] = load ptr addrspace(2), ptr addrspace(1) [[GEP2]], align 4
+; CHECK-NEXT:    [[STORE_GEP1:%.*]] = getelementptr inbounds i32, ptr addrspace(2) [[PTR2]], i64 0
+; CHECK-NEXT:    store ptr addrspace(1) [[LOAD1]], ptr addrspace(2) [[STORE_GEP1]], align 4
+; CHECK-NEXT:    [[STORE_GEP2:%.*]] = getelementptr inbounds ptr addrspace(2), ptr addrspace(2) [[PTR2]], i64 1
+; CHECK-NEXT:    store ptr addrspace(2) [[LOAD2]], ptr addrspace(2) [[STORE_GEP2]], align 4
+; CHECK-NEXT:    ret void
+;
+  %gep1 = getelementptr inbounds ptr addrspace(1), ptr addrspace(1) %ptr1, i64 0
+  %load1 = load ptr addrspace(1), ptr addrspace(1) %gep1, align 4
+  %gep2 = getelementptr inbounds ptr addrspace(2), ptr addrspace(1) %ptr1, i64 1
+  %load2 = load ptr addrspace(2), ptr addrspace(1) %gep2, align 4
+  %store.gep1 = getelementptr inbounds i32, ptr addrspace(2) %ptr2, i64 0
+  store ptr addrspace(1) %load1, ptr addrspace(2) %store.gep1, align 4
+  %store.gep2 = getelementptr inbounds ptr addrspace(2), ptr addrspace(2) %ptr2, i64 1
+  store ptr addrspace(2) %load2, ptr addrspace(2) %store.gep2, align 4
+  ret void
+}
diff --git a/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/merge-vectors.ll b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/merge-vectors.ll
index e6f2be25030c5..d6b51039d5b44 100644
--- a/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/merge-vectors.ll
+++ b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/merge-vectors.ll
@@ -130,302 +130,24 @@ entry:
   ret void
 }
 
+; Ideally this would be merged
 define amdgpu_kernel void @merge_load_i32_v2i16(ptr addrspace(1) nocapture %a) #0 {
 ; CHECK-LABEL: define amdgpu_kernel void @merge_load_i32_v2i16(
 ; CHECK-SAME: ptr addrspace(1) captures(none) [[A:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:  [[ENTRY:.*:]]
-; CHECK-NEXT:    [[TMP0:%.*]] = load <2 x i32>, ptr addrspace(1) [[A]], align 4
-; CHECK-NEXT:    [[LD_01:%.*]] = extractelement <2 x i32> [[TMP0]], i32 0
-; CHECK-NEXT:    [[LD_1_MUT2:%.*]] = extractelement <2 x i32> [[TMP0]], i32 1
-; CHECK-NEXT:    [[LD_1_TOORIG:%.*]] = bitcast i32 [[LD_1_MUT2]] to <2 x i16>
-; CHECK-NEXT:    ret void
-;
-entry:
-  %a.1 = getelementptr inbounds i32, ptr addrspace(1) %a, i32 1
-
-  %ld.0 = load i32, ptr addrspace(1) %a
-  %ld.1 = load <2 x i16>, ptr addrspace(1) %a.1
-
-  ret void
-}
-
-define amdgpu_kernel void @no_merge_load_i32_v2i8(ptr addrspace(1) nocapture %a) #0 {
-; CHECK-LABEL: define amdgpu_kernel void @no_merge_load_i32_v2i8(
-; CHECK-SAME: ptr addrspace(1) captures(none) [[A:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    [[A_1:%.*]] = getelementptr inbounds i32, ptr addrspace(1) [[A]], i32 1
 ; CHECK-NEXT:    [[LD_0:%.*]] = load i32, ptr addrspace(1) [[A]], align 4
-; CHECK-NEXT:    [[LD_1:%.*]] = load <2 x i8>, ptr addrspace(1) [[A_1]], align 2
+; CHECK-NEXT:    [[LD_1:%.*]] = load <2 x i16>, ptr addrspace(1) [[A_1]], align 4
 ; CHECK-NEXT:    ret void
 ;
 entry:
   %a.1 = getelementptr inbounds i32, ptr addrspace(1) %a, i32 1
 
   %ld.0 = load i32, ptr addrspace(1) %a
-  %ld.1 = load <2 x i8>, ptr addrspace(1) %a.1
-
-  ret void
-}
-
-define void @test_normalize_loads(ptr %p) {
-; CHECK-OOB-RELAXED-LABEL: define void @test_normalize_loads(
-; CHECK-OOB-RELAXED-SAME: ptr [[P:%.*]]) #[[ATTR1:[0-9]+]] {
-; CHECK-OOB-RELAXED-NEXT:  [[ENTRY:.*:]]
-; CHECK-OOB-RELAXED-NEXT:    [[TMP0:%.*]] = load <2 x i32>, ptr [[P]], align 4
-; CHECK-OOB-RELAXED-NEXT:    [[L01:%.*]] = extractelement <2 x i32> [[TMP0]], i32 0
-; CHECK-OOB-RELAXED-NEXT:    [[L1_MUT2:%.*]] = extractelement <2 x i32> [[TMP0]], i32 1
-; CHECK-OOB-RELAXED-NEXT:    [[L1_MUT_BC:%.*]] = bitcast i32 [[L1_MUT2]] to <2 x i16>
-; CHECK-OOB-RELAXED-NEXT:    [[L0_EXT:%.*]] = zext i32 [[L01]] to i64
-; CHECK-OOB-RELAXED-NEXT:    [[L1_CAST:%.*]] = bitcast <2 x i16> [[L1_MUT_BC]] to i32
-; CHECK-OOB-RELAXED-NEXT:    [[L1_EXT:%.*]] = zext i32 [[L1_CAST]] to i64
-; CHECK-OOB-RELAXED-NEXT:    [[ADD:%.*]] = add i64 [[L0_EXT]], [[L1_EXT]]
-; CHECK-OOB-RELAXED-NEXT:    store i64 [[ADD]], ptr null, align 8
-; CHECK-OOB-RELAXED-NEXT:    ret void
-;
-; CHECK-OOB-STRICT-LABEL: define void @test_normalize_loads(
-; CHECK-OOB-STRICT-SAME: ptr [[P:%.*]]) {
-; CHECK-OOB-STRICT-NEXT:  [[ENTRY:.*:]]
-; CHECK-OOB-STRICT-NEXT:    [[TMP0:%.*]] = load <2 x i32>, ptr [[P]], align 4
-; CHECK-OOB-STRICT-NEXT:    [[L01:%.*]] = extractelement <2 x i32> [[TMP0]], i32 0
-; CHECK-OOB-STRICT-NEXT:    [[L1_MUT2:%.*]] = extractelement <2 x i32> [[TMP0]], i32 1
-; CHECK-OOB-STRICT-NEXT:    [[L1_MUT_BC:%.*]] = bitcast i32 [[L1_MUT2]] to <2 x i16>
-; CHECK-OOB-STRICT-NEXT:    [[L0_EXT:%.*]] = zext i32 [[L01]] to i64
-; CHECK-OOB-STRICT-NEXT:    [[L1_CAST:%.*]] = bitcast <2 x i16> [[L1_MUT_BC]] to i32
-; CHECK-OOB-STRICT-NEXT:    [[L1_EXT:%.*]] = zext i32 [[L1_CAST]] to i64
-; CHECK-OOB-STRICT-NEXT:    [[ADD:%.*]] = add i64 [[L0_EXT]], [[L1_EXT]]
-; CHECK-OOB-STRICT-NEXT:    store i64 [[ADD]], ptr null, align 8
-; CHECK-OOB-STRICT-NEXT:    ret void
-;
-entry:
-  %p1 = getelementptr i32, ptr %p, i64 1
-  %l0 = load i32, ptr %p
-  %l1 = load <2 x i16>, ptr %p1
-  %l0_ext = zext i32 %l0 to i64
-  %l1_cast = bitcast <2 x i16> %l1 to i32
-  %l1_ext = zext i32 %l1_cast to i64
-  %add = add i64 %l0_ext, %l1_ext
-  store i64 %add, ptr null
-  ret void
-}
-
-define void @test_normalize_stores(ptr %p) {
-; CHECK-OOB-RELAXED-LABEL: define void @test_normalize_stores(
-; CHECK-OOB-RELAXED-SAME: ptr [[P:%.*]]) #[[ATTR1]] {
-; CHECK-OOB-RELAXED-NEXT:  [[ENTRY:.*:]]
-; CHECK-OOB-RELAXED-NEXT:    store <2 x i32> <i32 123, i32 bitcast (<2 x i16> <i16 4, i16 5> to i32)>, ptr [[P]], align 4
-; CHECK-OOB-RELAXED-NEXT:    ret void
-;
-; CHECK-OOB-STRICT-LABEL: define void @test_normalize_stores(
-; CHECK-OOB-STRICT-SAME: ptr [[P:%.*]]) {
-; CHECK-OOB-STRICT-NEXT:  [[ENTRY:.*:]]
-; CHECK-OOB-STRICT-NEXT:    store <2 x i32> <i32 123, i32 bitcast (<2 x i16> <i16 4, i16 5> to i32)>, ptr [[P]], align 4
-; CHECK-OOB-STRICT-NEXT:    ret void
-;
-entry:
-  %p1 = getelementptr i32, ptr %p, i64 1
-  store i32 123, ptr %p
-  store <2 x i16> <i16 4, i16 5>, ptr %p1
-  ret void
-}
-
-; TODO: Fix the below test
-; Check that metadata on loads is preserved when LSV normalizes mixed-typed
-; chains (exercises copyMetadataForAccess on loads).
-define void @lsv_copy_load_metadata(ptr %p) {
-; CHECK-OOB-RELAXED-LABEL: define void @lsv_copy_load_metadata(
-; CHECK-OOB-RELAXED-SAME: ptr [[P:%.*]]) #[[ATTR0:[0-9]+]] {
-; CHECK-OOB-RELAXED-NEXT:  [[ENTRY:.*:]]
-; CHECK-OOB-RELAXED-NEXT:    [[TMP0:%.*]] = load <2 x i32>, ptr [[P]], align 4, !tbaa [[TBAA0:![0-9]+]], !invariant.load [[META3:![0-9]+]], !nontemporal [[META4:![0-9]+]]
-; CHECK-OOB-RELAXED-NEXT:    [[L01:%.*]] = extractelement <2 x i32> [[TMP0]], i32 0
-; CHECK-OOB-RELAXED-NEXT:    [[L1_MUT2:%.*]] = extractelement <2 x i32> [[TMP0]], i32 1
-; CHECK-OOB-RELAXED-NEXT:    [[L1_MUT_BC:%.*]] = bitcast i32 [[L1_MUT2]] to <2 x i16>
-; CHECK-OOB-RELAXED-NEXT:    ret void
-;
-; CHECK-OOB-STRICT-LABEL: define void @lsv_copy_load_metadata(
-; CHECK-OOB-STRICT-SAME: ptr [[P:%.*]]) {
-; CHECK-OOB-STRICT-NEXT:  [[ENTRY:.*:]]
-; CHECK-OOB-STRICT-NEXT:    [[TMP0:%.*]] = load <2 x i32>, ptr [[P]], align 4, !tbaa [[TBAA0:![0-9]+]], !invariant.load [[META3:![0-9]+]], !nontemporal [[META4:![0-9]+]]
-; CHECK-OOB-STRICT-NEXT:    [[L01:%.*]] = extractelement <2 x i32> [[TMP0]], i32 0
-; CHECK-OOB-STRICT-NEXT:    [[L1_MUT2:%.*]] = extractelement <2 x i32> [[TMP0]], i32 1
-; CHECK-OOB-STRICT-NEXT:    [[L1_MUT_BC:%.*]] = bitcast i32 [[L1_MUT2]] to <2 x i16>
-; CHECK-OOB-STRICT-NEXT:    ret void
-;
-entry:
-  %p1 = getelementptr i32, ptr %p, i64 1
-  %ld0 = load i32, ptr %p, align 4, !tbaa !0, !nontemporal !5, !invariant.load !6
-  %ld1 = load <2 x i16>, ptr %p1, align 4, !tbaa !0, !nontemporal !5, !invariant.load !6
-  ret void
-}
+  %ld.1 = load <2 x i16>, ptr addrspace(1) %a.1
 
-; Check that metadata on stores is preserved when LSV normalizes mixed-typed
-; chains (exercises copyMetadataForAccess on stores).
-define void @lsv_copy_store_metadata(ptr %p) {
-; CHECK-OOB-RELAXED-LABEL: define void @lsv_copy_store_metadata(
-; CHECK-OOB-RELAXED-SAME: ptr [[P:%.*]]) #[[ATTR0]] {
-; CHECK-OOB-RELAXED-NEXT:  [[ENTRY:.*:]]
-; CHECK-OOB-RELAXED-NEXT:    store <2 x i32> <i32 7, i32 bitcast (<2 x i16> <i16 4, i16 5> to i32)>, ptr [[P]], align 4, !nontemporal [[META4]]
-; CHECK-OOB-RELAXED-NEXT:    ret void
-;
-; CHECK-OOB-STRICT-LABEL: define void @lsv_copy_store_metadata(
-; CHECK-OOB-STRICT-SAME: ptr [[P:%.*]]) {
-; CHECK-OOB-STRICT-NEXT:  [[ENTRY:.*:]]
-; CHECK-OOB-STRICT-NEXT:    store <2 x i32> <i32 7, i32 bitcast (<2 x i16> <i16 4, i16 5> to i32)>, ptr [[P]], align 4, !nontemporal [[META4]]
-; CHECK-OOB-STRICT-NEXT:    ret void
-;
-entry:
-  %p1 = getelementptr i32, ptr %p, i64 1
-  store i32 7, ptr %p, align 4, !nontemporal !5
-  store <2 x i16> <i16 4, i16 5>, ptr %p1, align 4, !nontemporal !5
   ret void
 }
 
-!0 = !{!3, !3, i64 0}
-!3 = !{!"omnipotent char", !4, i64 0}
-!4 = !{!"Simple C/C++ TBAA"}
-!5 = !{i32 1}
-!6 = !{}
 attributes #0 = { nounwind }
 attributes #1 = { nounwind readnone }
-
-
-; Non power-of-two combined span (12 bytes) must not merge chains.
-define void @no_merge_non_pot_span(ptr addrspace(1) %p) {
-; CHECK-OOB-RELAXED-LABEL: define void @no_merge_non_pot_span(
-; CHECK-OOB-RELAXED-SAME: ptr addrspace(1) [[P:%.*]]) #[[ATTR1]] {
-; CHECK-OOB-RELAXED-NEXT:  [[ENTRY:.*:]]
-; CHECK-OOB-RELAXED-NEXT:    [[L0:%.*]] = load i32, ptr addrspace(1) [[P]], align 4
-; CHECK-OOB-RELAXED-NEXT:    [[P8:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[P]], i64 8
-; CHECK-OOB-RELAXED-NEXT:    [[L1:%.*]] = load float, ptr addrspace(1) [[P8]], align 4
-; CHECK-OOB-RELAXED-NEXT:    ret void
-;
-; CHECK-OOB-STRICT-LABEL: define void @no_merge_non_pot_span(
-; CHECK-OOB-STRICT-SAME: ptr addrspace(1) [[P:%.*]]) {
-; CHECK-OOB-STRICT-NEXT:  [[ENTRY:.*:]]
-; CHECK-OOB-STRICT-NEXT:    [[L0:%.*]] = load i32, ptr addrspace(1) [[P]], align 4
-; CHECK-OOB-STRICT-NEXT:    [[P8:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[P]], i64 8
-; CHECK-OOB-STRICT-NEXT:    [[L1:%.*]] = load float, ptr addrspace(1) [[P8]], align 4
-; CHECK-OOB-STRICT-NEXT:    ret void
-;
-entry:
-  %l0 = load i32, ptr addrspace(1) %p, align 4
-  %p8 = getelementptr inbounds i8, ptr addrspace(1) %p, i64 8
-  %l1 = load float, ptr addrspace(1) %p8, align 4
-  ret void
-}
-
-define void @no_merge_diff_ptrop(ptr addrspace(1) %ptr1, ptr addrspace(2) %ptr2) {
-; CHECK-OOB-RELAXED-LABEL: define void @no_merge_diff_ptrop(
-; CHECK-OOB-RELAXED-SAME: ptr addrspace(1) [[PTR1:%.*]], ptr addrspace(2) [[PTR2:%.*]]) #[[ATTR1]] {
-; CHECK-OOB-RELAXED-NEXT:    [[LOAD_0:%.*]] = load i32, ptr addrspace(1) [[PTR1]], align 4
-; CHECK-OOB-RELAXED-NEXT:    [[LOAD_1:%.*]] = load i32, ptr addrspace(2) [[PTR2]], align 4
-; CHECK-OOB-RELAXED-NEXT:    ret void
-;
-; CHECK-OOB-STRICT-LABEL: define void @no_merge_diff_ptrop(
-; CHECK-OOB-STRICT-SAME: ptr addrspace(1) [[PTR1:%.*]], ptr addrspace(2) [[PTR2:%.*]]) {
-; CHECK-OOB-STRICT-NEXT:    [[LOAD_0:%.*]] = load i32, ptr addrspace(1) [[PTR1]], align 4
-; CHECK-OOB-STRICT-NEXT:    [[LOAD_1:%.*]] = load i32, ptr addrspace(2) [[PTR2]], align 4
-; CHECK-OOB-STRICT-NEXT:    ret void
-;
-  %load.0 = load i32, ptr addrspace(1) %ptr1
-  %load.1 = load i32, ptr addrspace(2) %ptr2
-  ret void
-}
-
-define void @no_merge_load_store(ptr addrspace(1) %ptr1) {
-; CHECK-OOB-RELAXED-LABEL: define void @no_merge_load_store(
-; CHECK-OOB-RELAXED-SAME: ptr addrspace(1) [[PTR1:%.*]]) #[[ATTR1]] {
-; CHECK-OOB-RELAXED-NEXT:    [[LOAD_0:%.*]] = load i32, ptr addrspace(1) [[PTR1]], align 4
-; CHECK-OOB-RELAXED-NEXT:    store i32 111, ptr addrspace(1) [[PTR1]], align 4
-; CHECK-OOB-RELAXED-NEXT:    ret void
-;
-; CHECK-OOB-STRICT-LABEL: define void @no_merge_load_store(
-; CHECK-OOB-STRICT-SAME: ptr addrspace(1) [[PTR1:%.*]]) {
-; CHECK-OOB-STRICT-NEXT:    [[LOAD_0:%.*]] = load i32, ptr addrspace(1) [[PTR1]], align 4
-; CHECK-OOB-STRICT-NEXT:    store i32 111, ptr addrspace(1) [[PTR1]], align 4
-; CHECK-OOB-STRICT-NEXT:    ret void
-;
-  %load.0 = load i32, ptr addrspace(1) %ptr1
-  store i32 111, ptr addrspace(1) %ptr1
-  ret void
-}
-
-; Stores in this test should not be vectorized as the total byte span
-; from the end of %gep.a to the end of %gep.b is not a power of 2. This
-; is a necessary condition for splitChainByAlignment.
-define void @check_contiguity_of_base_ptrs(ptr addrspace(1) %ptr) {
-; CHECK-OOB-RELAXED-LABEL: define void @check_contiguity_of_base_ptrs(
-; CHECK-OOB-RELAXED-SAME: ptr addrspace(1) [[PTR:%.*]]) #[[ATTR1]] {
-; CHECK-OOB-RELAXED-NEXT:    store i32 274, ptr addrspace(1) [[PTR]], align 4
-; CHECK-OOB-RELAXED-NEXT:    [[GEP_A:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(1) [[PTR]], i64 4
-; CHECK-OOB-RELAXED-NEXT:    store i64 3610770474484254748, ptr addrspace(1) [[GEP_A]], align 8
-; CHECK-OOB-RELAXED-NEXT:    [[GEP_B:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(1) [[PTR]], i64 12
-; CHECK-OOB-RELAXED-NEXT:    store <2 x i32> <i32 1819043144, i32 1867980911>, ptr addrspace(1) [[GEP_B]], align 4
-; CHECK-OOB-RELAXED-NEXT:    ret void
-;
-; CHECK-OOB-STRICT-LABEL: define void @check_contiguity_of_base_ptrs(
-; CHECK-OOB-STRICT-SAME: ptr addrspace(1) [[PTR:%.*]]) {
-; CHECK-OOB-STRICT-NEXT:    store i32 274, ptr addrspace(1) [[PTR]], align 4
-; CHECK-OOB-STRICT-NEXT:    [[GEP_A:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(1) [[PTR]], i64 4
-; CHECK-OOB-STRICT-NEXT:    store i64 3610770474484254748, ptr addrspace(1) [[GEP_A]], align 8
-; CHECK-OOB-STRICT-NEXT:    [[GEP_B:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(1) [[PTR]], i64 12
-; CHECK-OOB-STRICT-NEXT:    store <2 x i32> <i32 1819043144, i32 1867980911>, ptr addrspace(1) [[GEP_B]], align 4
-; CHECK-OOB-STRICT-NEXT:    ret void
-;
-  store i32 274, ptr addrspace(1) %ptr, align 4
-  %gep.a = getelementptr inbounds nuw i8, ptr addrspace(1) %ptr, i64 4
-  store i64 3610770474484254748, ptr addrspace(1) %gep.a, align 8
-  %gep.b = getelementptr inbounds nuw i8, ptr addrspace(1) %ptr, i64 12
-  store <2 x i32> <i32 1819043144, i32 1867980911>, ptr addrspace(1) %gep.b, align 4
-  ret void
-}
-
-; Offset is unknown in the following test, LSV should fail to vectorize.
-define amdgpu_kernel void @assert_computeLeaderDelta(ptr addrspace(1) %a, i64 %idx) {
-; CHECK-OOB-RELAXED-LABEL: define amdgpu_kernel void @assert_computeLeaderDelta(
-; CHECK-OOB-RELAXED-SAME: ptr addrspace(1) [[A:%.*]], i64 [[IDX:%.*]]) #[[ATTR1]] {
-; CHECK-OOB-RELAXED-NEXT:  [[ENTRY:.*:]]
-; CHECK-OOB-RELAXED-NEXT:    [[LD0:%.*]] = load i32, ptr addrspace(1) [[A]], align 4
-; CHECK-OOB-RELAXED-NEXT:    [[P1:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[A]], i64 [[IDX]]
-; CHECK-OOB-RELAXED-NEXT:    [[LD1:%.*]] = load <2 x i16>, ptr addrspace(1) [[P1]], align 2
-; CHECK-OOB-RELAXED-NEXT:    ret void
-;
-; CHECK-OOB-STRICT-LABEL: define amdgpu_kernel void @assert_computeLeaderDelta(
-; CHECK-OOB-STRICT-SAME: ptr addrspace(1) [[A:%.*]], i64 [[IDX:%.*]]) {
-; CHECK-OOB-STRICT-NEXT:  [[ENTRY:.*:]]
-; CHECK-OOB-STRICT-NEXT:    [[LD0:%.*]] = load i32, ptr addrspace(1) [[A]], align 4
-; CHECK-OOB-STRICT-NEXT:    [[P1:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[A]], i64 [[IDX]]
-; CHECK-OOB-STRICT-NEXT:    [[LD1:%.*]] = load <2 x i16>, ptr addrspace(1) [[P1]], align 2
-; CHECK-OOB-STRICT-NEXT:    ret void
-;
-entry:
-  %ld0 = load i32, ptr addrspace(1) %a, align 4
-  %p1 = getelementptr inbounds i8, ptr addrspace(1) %a, i64 %idx
-  %ld1 = load <2 x i16>, ptr addrspace(1) %p1, align 2
-  ret void
-}
-
-
-; Overlapping ranges after rebasing should prevent merging across chains.
-define void @no_merge_overlap_after_rebase(ptr addrspace(1) %p) {
-; CHECK-OOB-RELAXED-LABEL: define void @no_merge_overlap_after_rebase(
-; CHECK-OOB-RELAXED-SAME: ptr addrspace(1) [[P:%.*]]) #[[ATTR1]] {
-; CHECK-OOB-RELAXED-NEXT:  [[ENTRY:.*:]]
-; CHECK-OOB-RELAXED-NEXT:    [[L0:%.*]] = load i32, ptr addrspace(1) [[P]], align 4
-; CHECK-OOB-RELAXED-NEXT:    [[P1:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[P]], i64 2
-; CHECK-OOB-RELAXED-NEXT:    [[L1:%.*]] = load <2 x i16>, ptr addrspace(1) [[P1]], align 2
-; CHECK-OOB-RELAXED-NEXT:    ret void
-;
-; CHECK-OOB-STRICT-LABEL: define void @no_merge_overlap_after_rebase(
-; CHECK-OOB-STRICT-SAME: ptr addrspace(1) [[P:%.*]]) {
-; CHECK-OOB-STRICT-NEXT:  [[ENTRY:.*:]]
-; CHECK-OOB-STRICT-NEXT:    [[L0:%.*]] = load i32, ptr addrspace(1) [[P]], align 4
-; CHECK-OOB-STRICT-NEXT:    [[P1:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[P]], i64 2
-; CHECK-OOB-STRICT-NEXT:    [[L1:%.*]] = load <2 x i16>, ptr addrspace(1) [[P1]], align 2
-; CHECK-OOB-STRICT-NEXT:    ret void
-;
-entry:
-  %l0 = load i32, ptr addrspace(1) %p, align 4
-  %p1 = getelementptr inbounds i8, ptr addrspace(1) %p, i64 2
-  %l1 = load <2 x i16>, ptr addrspace(1) %p1, align 2
-  ret void
-}
diff --git a/llvm/test/Transforms/LoopStrengthReduce/AArch64/prefer-all.ll b/llvm/test/Transforms/LoopStrengthReduce/AArch64/prefer-all.ll
index 1944a9c800355..5fe72ea0d4fea 100644
--- a/llvm/test/Transforms/LoopStrengthReduce/AArch64/prefer-all.ll
+++ b/llvm/test/Transforms/LoopStrengthReduce/AArch64/prefer-all.ll
@@ -230,8 +230,6 @@ exit:
 
 ; The control-flow before and after the load of qval shouldn't prevent postindex
 ; addressing from happening.
-; FIXME: We choose postindex addressing, but the scevgep is placed in for.inc so
-; during codegen we will fail to actually generate a postindex load.
 define void @middle_block_load(ptr %p, ptr %q, i64 %n) {
 ; CHECK-LABEL: define void @middle_block_load(
 ; CHECK-SAME: ptr [[P:%.*]], ptr [[Q:%.*]], i64 [[N:%.*]]) {
@@ -254,6 +252,7 @@ define void @middle_block_load(ptr %p, ptr %q, i64 %n) {
 ; CHECK:       [[IF_END]]:
 ; CHECK-NEXT:    [[QVAL:%.*]] = load i32, ptr [[LSR_IV1]], align 4
 ; CHECK-NEXT:    [[CMP2:%.*]] = icmp sgt i32 [[QVAL]], 0
+; CHECK-NEXT:    [[SCEVGEP]] = getelementptr i8, ptr [[LSR_IV1]], i64 4
 ; CHECK-NEXT:    br i1 [[CMP2]], label %[[IF_THEN2:.*]], label %[[IF_ELSE2:.*]]
 ; CHECK:       [[IF_THEN2]]:
 ; CHECK-NEXT:    tail call void @otherfn1()
@@ -263,7 +262,6 @@ define void @middle_block_load(ptr %p, ptr %q, i64 %n) {
 ; CHECK-NEXT:    br label %[[FOR_INC]]
 ; CHECK:       [[FOR_INC]]:
 ; CHECK-NEXT:    [[LSR_IV_NEXT]] = add i64 [[LSR_IV]], -1
-; CHECK-NEXT:    [[SCEVGEP]] = getelementptr i8, ptr [[LSR_IV1]], i64 4
 ; CHECK-NEXT:    [[CMP3:%.*]] = icmp eq i64 [[LSR_IV_NEXT]], 0
 ; CHECK-NEXT:    br i1 [[CMP3]], label %[[EXIT:.*]], label %[[FOR_BODY]]
 ; CHECK:       [[EXIT]]:
diff --git a/llvm/test/Transforms/LoopUnroll/runtime-loop-multiexit-dom-verify.ll b/llvm/test/Transforms/LoopUnroll/runtime-loop-multiexit-dom-verify.ll
index de54852313456..7be3a94b90e46 100644
--- a/llvm/test/Transforms/LoopUnroll/runtime-loop-multiexit-dom-verify.ll
+++ b/llvm/test/Transforms/LoopUnroll/runtime-loop-multiexit-dom-verify.ll
@@ -265,8 +265,8 @@ define void @test4(i16 %c3) {
 ; CHECK-NEXT:    br label [[EXITING_PROL:%.*]]
 ; CHECK:       exiting.prol:
 ; CHECK-NEXT:    switch i16 [[C3:%.*]], label [[DEFAULT_LOOPEXIT_LOOPEXIT1:%.*]] [
-; CHECK-NEXT:    i16 45, label [[OTHEREXIT_LOOPEXIT2:%.*]]
-; CHECK-NEXT:    i16 95, label [[LATCH_PROL]]
+; CHECK-NEXT:      i16 45, label [[OTHEREXIT_LOOPEXIT2:%.*]]
+; CHECK-NEXT:      i16 95, label [[LATCH_PROL]]
 ; CHECK-NEXT:    ]
 ; CHECK:       latch.prol:
 ; CHECK-NEXT:    [[INDVARS_IV_NEXT_PROL]] = add nuw nsw i64 [[INDVARS_IV_PROL]], 1
@@ -288,29 +288,29 @@ define void @test4(i16 %c3) {
 ; CHECK-NEXT:    br label [[EXITING:%.*]]
 ; CHECK:       exiting:
 ; CHECK-NEXT:    switch i16 [[C3]], label [[DEFAULT_LOOPEXIT_LOOPEXIT:%.*]] [
-; CHECK-NEXT:    i16 45, label [[OTHEREXIT_LOOPEXIT:%.*]]
-; CHECK-NEXT:    i16 95, label [[LATCH:%.*]]
+; CHECK-NEXT:      i16 45, label [[OTHEREXIT_LOOPEXIT:%.*]]
+; CHECK-NEXT:      i16 95, label [[LATCH:%.*]]
 ; CHECK-NEXT:    ]
 ; CHECK:       latch:
 ; CHECK-NEXT:    br label [[EXITING_1:%.*]]
 ; CHECK:       exiting.1:
 ; CHECK-NEXT:    switch i16 [[C3]], label [[DEFAULT_LOOPEXIT_LOOPEXIT]] [
-; CHECK-NEXT:    i16 45, label [[OTHEREXIT_LOOPEXIT]]
-; CHECK-NEXT:    i16 95, label [[LATCH_1:%.*]]
+; CHECK-NEXT:      i16 45, label [[OTHEREXIT_LOOPEXIT]]
+; CHECK-NEXT:      i16 95, label [[LATCH_1:%.*]]
 ; CHECK-NEXT:    ]
 ; CHECK:       latch.1:
 ; CHECK-NEXT:    br label [[EXITING_2:%.*]]
 ; CHECK:       exiting.2:
 ; CHECK-NEXT:    switch i16 [[C3]], label [[DEFAULT_LOOPEXIT_LOOPEXIT]] [
-; CHECK-NEXT:    i16 45, label [[OTHEREXIT_LOOPEXIT]]
-; CHECK-NEXT:    i16 95, label [[LATCH_2:%.*]]
+; CHECK-NEXT:      i16 45, label [[OTHEREXIT_LOOPEXIT]]
+; CHECK-NEXT:      i16 95, label [[LATCH_2:%.*]]
 ; CHECK-NEXT:    ]
 ; CHECK:       latch.2:
 ; CHECK-NEXT:    br label [[EXITING_3:%.*]]
 ; CHECK:       exiting.3:
 ; CHECK-NEXT:    switch i16 [[C3]], label [[DEFAULT_LOOPEXIT_LOOPEXIT]] [
-; CHECK-NEXT:    i16 45, label [[OTHEREXIT_LOOPEXIT]]
-; CHECK-NEXT:    i16 95, label [[LATCH_3]]
+; CHECK-NEXT:      i16 45, label [[OTHEREXIT_LOOPEXIT]]
+; CHECK-NEXT:      i16 95, label [[LATCH_3]]
 ; CHECK-NEXT:    ]
 ; CHECK:       latch.3:
 ; CHECK-NEXT:    [[INDVARS_IV_NEXT_3]] = add nuw nsw i64 [[INDVARS_IV]], 4
diff --git a/llvm/test/Transforms/LoopUnrollAndJam/unroll-and-jam.ll b/llvm/test/Transforms/LoopUnrollAndJam/unroll-and-jam.ll
index 9ee51cfbcb590..a3d2fcb5ab946 100644
--- a/llvm/test/Transforms/LoopUnrollAndJam/unroll-and-jam.ll
+++ b/llvm/test/Transforms/LoopUnrollAndJam/unroll-and-jam.ll
@@ -638,18 +638,18 @@ define i32 @test6() #0 {
 ; CHECK:       [[FOR_LATCH]]:
 ; CHECK-NEXT:    br i1 false, label %[[FOR_OUTER]], label %[[FOR_END_UNR_LCSSA:.*]], !llvm.loop [[LOOP7:![0-9]+]]
 ; CHECK:       [[FOR_END_UNR_LCSSA]]:
-; CHECK-NEXT:    [[DOTLCSSA_LCSSA_PH_PH:%.*]] = phi i32 [ 2, %[[FOR_LATCH]] ]
-; CHECK-NEXT:    [[INC_LCSSA_LCSSA_PH_PH:%.*]] = phi i32 [ 7, %[[FOR_LATCH]] ]
-; CHECK-NEXT:    [[P0_UNR_PH:%.*]] = phi i32 [ 2, %[[FOR_LATCH]] ]
+; CHECK-NEXT:    [[DOTLCSSA_LCSSA_PH:%.*]] = phi i32 [ 2, %[[FOR_LATCH]] ]
+; CHECK-NEXT:    [[INC_LCSSA_LCSSA_PH:%.*]] = phi i32 [ 7, %[[FOR_LATCH]] ]
+; CHECK-NEXT:    [[P0_UNR:%.*]] = phi i32 [ 2, %[[FOR_LATCH]] ]
 ; CHECK-NEXT:    br i1 true, label %[[FOR_OUTER_EPIL_PREHEADER]], label %[[FOR_END:.*]]
 ; CHECK:       [[FOR_OUTER_EPIL_PREHEADER]]:
-; CHECK-NEXT:    [[P0_UNR:%.*]] = phi i32 [ [[F_PROMOTED10]], %[[ENTRY]] ], [ [[P0_UNR_PH]], %[[FOR_END_UNR_LCSSA]] ]
+; CHECK-NEXT:    [[P0_EPIL_INIT:%.*]] = phi i32 [ [[F_PROMOTED10]], %[[ENTRY]] ], [ [[P0_UNR]], %[[FOR_END_UNR_LCSSA]] ]
 ; CHECK-NEXT:    call void @llvm.assume(i1 true)
 ; CHECK-NEXT:    br label %[[FOR_OUTER_EPIL:.*]]
 ; CHECK:       [[FOR_OUTER_EPIL]]:
 ; CHECK-NEXT:    br label %[[FOR_INNER_EPIL:.*]]
 ; CHECK:       [[FOR_INNER_EPIL]]:
-; CHECK-NEXT:    [[P1_EPIL:%.*]] = phi i32 [ [[P0_UNR]], %[[FOR_OUTER_EPIL]] ], [ 2, %[[FOR_INNER_EPIL]] ]
+; CHECK-NEXT:    [[P1_EPIL:%.*]] = phi i32 [ [[P0_EPIL_INIT]], %[[FOR_OUTER_EPIL]] ], [ 2, %[[FOR_INNER_EPIL]] ]
 ; CHECK-NEXT:    [[INC_SINK8_EPIL:%.*]] = phi i32 [ 0, %[[FOR_OUTER_EPIL]] ], [ [[INC_EPIL:%.*]], %[[FOR_INNER_EPIL]] ]
 ; CHECK-NEXT:    [[INC_EPIL]] = add nuw nsw i32 [[INC_SINK8_EPIL]], 1
 ; CHECK-NEXT:    [[EXITCOND_EPIL:%.*]] = icmp ne i32 [[INC_EPIL]], 7
@@ -658,8 +658,8 @@ define i32 @test6() #0 {
 ; CHECK-NEXT:    [[DOTLCSSA_EPIL:%.*]] = phi i32 [ [[P1_EPIL]], %[[FOR_INNER_EPIL]] ]
 ; CHECK-NEXT:    br label %[[FOR_END]]
 ; CHECK:       [[FOR_END]]:
-; CHECK-NEXT:    [[DOTLCSSA_LCSSA:%.*]] = phi i32 [ [[DOTLCSSA_LCSSA_PH_PH]], %[[FOR_END_UNR_LCSSA]] ], [ [[DOTLCSSA_EPIL]], %[[FOR_LATCH_EPIL]] ]
-; CHECK-NEXT:    [[INC_LCSSA_LCSSA:%.*]] = phi i32 [ [[INC_LCSSA_LCSSA_PH_PH]], %[[FOR_END_UNR_LCSSA]] ], [ 7, %[[FOR_LATCH_EPIL]] ]
+; CHECK-NEXT:    [[DOTLCSSA_LCSSA:%.*]] = phi i32 [ [[DOTLCSSA_LCSSA_PH]], %[[FOR_END_UNR_LCSSA]] ], [ [[DOTLCSSA_EPIL]], %[[FOR_LATCH_EPIL]] ]
+; CHECK-NEXT:    [[INC_LCSSA_LCSSA:%.*]] = phi i32 [ [[INC_LCSSA_LCSSA_PH]], %[[FOR_END_UNR_LCSSA]] ], [ 7, %[[FOR_LATCH_EPIL]] ]
 ; CHECK-NEXT:    ret i32 0
 ;
 entry:
@@ -1324,9 +1324,9 @@ define signext i16 @test10(i32 %k) #0 {
 ; CHECK-NEXT:    [[STOREMERGE_4_LCSSA_3:%.*]] = phi i64 [ [[STOREMERGE_4_3:%.*]], %[[FOR_INC21_3]] ]
 ; CHECK-NEXT:    br i1 false, label %[[FOR_BODY]], label %[[FOR_END26_UNR_LCSSA:.*]], !llvm.loop [[LOOP13:![0-9]+]]
 ; CHECK:       [[FOR_END26_UNR_LCSSA]]:
-; CHECK-NEXT:    [[DEC_LCSSA_LCSSA_PH_PH:%.*]] = phi i64 [ 0, %[[FOR_INC24]] ]
-; CHECK-NEXT:    [[STOREMERGE_4_LCSSA_LCSSA_PH_PH:%.*]] = phi i64 [ [[STOREMERGE_4_LCSSA_3]], %[[FOR_INC24]] ]
-; CHECK-NEXT:    [[STOREMERGE_5_LCSSA_LCSSA_PH_PH:%.*]] = phi i32 [ 0, %[[FOR_INC24]] ]
+; CHECK-NEXT:    [[DEC_LCSSA_LCSSA_PH:%.*]] = phi i64 [ 0, %[[FOR_INC24]] ]
+; CHECK-NEXT:    [[STOREMERGE_4_LCSSA_LCSSA_PH:%.*]] = phi i64 [ [[STOREMERGE_4_LCSSA_3]], %[[FOR_INC24]] ]
+; CHECK-NEXT:    [[STOREMERGE_5_LCSSA_LCSSA_PH:%.*]] = phi i32 [ 0, %[[FOR_INC24]] ]
 ; CHECK-NEXT:    br i1 true, label %[[FOR_BODY_EPIL_PREHEADER]], label %[[FOR_END26:.*]]
 ; CHECK:       [[FOR_BODY_EPIL_PREHEADER]]:
 ; CHECK-NEXT:    call void @llvm.assume(i1 true)
@@ -1353,9 +1353,9 @@ define signext i16 @test10(i32 %k) #0 {
 ; CHECK-NEXT:    [[STOREMERGE_4_LCSSA_EPIL:%.*]] = phi i64 [ [[STOREMERGE_4_EPIL]], %[[FOR_INC21_EPIL]] ]
 ; CHECK-NEXT:    br label %[[FOR_END26]]
 ; CHECK:       [[FOR_END26]]:
-; CHECK-NEXT:    [[DEC_LCSSA_LCSSA:%.*]] = phi i64 [ [[DEC_LCSSA_LCSSA_PH_PH]], %[[FOR_END26_UNR_LCSSA]] ], [ 0, %[[FOR_INC24_EPIL]] ]
-; CHECK-NEXT:    [[STOREMERGE_4_LCSSA_LCSSA:%.*]] = phi i64 [ [[STOREMERGE_4_LCSSA_LCSSA_PH_PH]], %[[FOR_END26_UNR_LCSSA]] ], [ [[STOREMERGE_4_LCSSA_EPIL]], %[[FOR_INC24_EPIL]] ]
-; CHECK-NEXT:    [[STOREMERGE_5_LCSSA_LCSSA:%.*]] = phi i32 [ [[STOREMERGE_5_LCSSA_LCSSA_PH_PH]], %[[FOR_END26_UNR_LCSSA]] ], [ 0, %[[FOR_INC24_EPIL]] ]
+; CHECK-NEXT:    [[DEC_LCSSA_LCSSA:%.*]] = phi i64 [ [[DEC_LCSSA_LCSSA_PH]], %[[FOR_END26_UNR_LCSSA]] ], [ 0, %[[FOR_INC24_EPIL]] ]
+; CHECK-NEXT:    [[STOREMERGE_4_LCSSA_LCSSA:%.*]] = phi i64 [ [[STOREMERGE_4_LCSSA_LCSSA_PH]], %[[FOR_END26_UNR_LCSSA]] ], [ [[STOREMERGE_4_LCSSA_EPIL]], %[[FOR_INC24_EPIL]] ]
+; CHECK-NEXT:    [[STOREMERGE_5_LCSSA_LCSSA:%.*]] = phi i32 [ [[STOREMERGE_5_LCSSA_LCSSA_PH]], %[[FOR_END26_UNR_LCSSA]] ], [ 0, %[[FOR_INC24_EPIL]] ]
 ; CHECK-NEXT:    store i64 [[DEC_LCSSA_LCSSA]], ptr @g, align 8
 ; CHECK-NEXT:    ret i16 0
 ; CHECK:       [[FOR_BODY2_SPLIT2_1]]:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/force-target-instruction-cost.ll b/llvm/test/Transforms/LoopVectorize/AArch64/force-target-instruction-cost.ll
index 21b21774d18cf..91c65ba8f6267 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/force-target-instruction-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/force-target-instruction-cost.ll
@@ -380,7 +380,7 @@ for.end:
   ret void
 }
 
-define void @loop_with_freeze_and_conditional_srem(ptr %dst, ptr %keyinfo, ptr %invariant.ptr, i32 %divisor) #1 {
+define void @loop_with_freeze_and_conditional_srem(ptr %dst, ptr %keyinfo, ptr %invariant.ptr, i32 %divisor) {
 ; COMMON-LABEL: define void @loop_with_freeze_and_conditional_srem(
 ; COMMON-SAME: ptr [[DST:%.*]], ptr [[KEYINFO:%.*]], ptr [[INVARIANT_PTR:%.*]], i32 [[DIVISOR:%.*]]) {
 ; COMMON-NEXT:  [[ENTRY:.*]]:
@@ -433,7 +433,165 @@ exit:                                             ; preds = %loop.latch
   ret void
 }
 
+define void @interleave_group(ptr %dst) #1 {
+; COST1-LABEL: define void @interleave_group(
+; COST1-SAME: ptr [[DST:%.*]]) #[[ATTR1:[0-9]+]] {
+; COST1-NEXT:  [[ITER_CHECK:.*:]]
+; COST1-NEXT:    br i1 false, label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
+; COST1:       [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
+; COST1-NEXT:    br i1 false, label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]]
+; COST1:       [[VECTOR_PH]]:
+; COST1-NEXT:    br label %[[VECTOR_BODY:.*]]
+; COST1:       [[VECTOR_BODY]]:
+; COST1-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COST1-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 16
+; COST1-NEXT:    [[TMP1:%.*]] = mul i64 [[INDEX]], 3
+; COST1-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP0]], 3
+; COST1-NEXT:    [[TMP3:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP1]]
+; COST1-NEXT:    [[TMP4:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP2]]
+; COST1-NEXT:    store <48 x i8> zeroinitializer, ptr [[TMP3]], align 1
+; COST1-NEXT:    store <48 x i8> zeroinitializer, ptr [[TMP4]], align 1
+; COST1-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 32
+; COST1-NEXT:    [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 96
+; COST1-NEXT:    br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
+; COST1:       [[MIDDLE_BLOCK]]:
+; COST1-NEXT:    br i1 false, [[EXIT:label %.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
+; COST1:       [[VEC_EPILOG_ITER_CHECK]]:
+; COST1-NEXT:    br i1 false, label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF4]]
+; COST1:       [[VEC_EPILOG_PH]]:
+; COST1-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 96, %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; COST1-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[BC_RESUME_VAL]], i64 0
+; COST1-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; COST1-NEXT:    [[INDUCTION:%.*]] = add <4 x i64> [[BROADCAST_SPLAT]], <i64 0, i64 1, i64 2, i64 3>
+; COST1-NEXT:    br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
+; COST1:       [[VEC_EPILOG_VECTOR_BODY]]:
+; COST1-NEXT:    [[INDEX1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT2:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; COST1-NEXT:    [[VEC_IND:%.*]] = phi <4 x i64> [ [[INDUCTION]], %[[VEC_EPILOG_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; COST1-NEXT:    [[TMP6:%.*]] = mul <4 x i64> [[VEC_IND]], splat (i64 3)
+; COST1-NEXT:    [[TMP7:%.*]] = extractelement <4 x i64> [[TMP6]], i32 0
+; COST1-NEXT:    [[TMP8:%.*]] = extractelement <4 x i64> [[TMP6]], i32 1
+; COST1-NEXT:    [[TMP9:%.*]] = extractelement <4 x i64> [[TMP6]], i32 2
+; COST1-NEXT:    [[TMP10:%.*]] = extractelement <4 x i64> [[TMP6]], i32 3
+; COST1-NEXT:    [[TMP11:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP7]]
+; COST1-NEXT:    [[TMP12:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP8]]
+; COST1-NEXT:    [[TMP13:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP9]]
+; COST1-NEXT:    [[TMP14:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP10]]
+; COST1-NEXT:    [[TMP15:%.*]] = getelementptr i8, ptr [[TMP11]], i64 2
+; COST1-NEXT:    [[TMP16:%.*]] = getelementptr i8, ptr [[TMP12]], i64 2
+; COST1-NEXT:    [[TMP17:%.*]] = getelementptr i8, ptr [[TMP13]], i64 2
+; COST1-NEXT:    [[TMP18:%.*]] = getelementptr i8, ptr [[TMP14]], i64 2
+; COST1-NEXT:    store i8 0, ptr [[TMP15]], align 1
+; COST1-NEXT:    store i8 0, ptr [[TMP16]], align 1
+; COST1-NEXT:    store i8 0, ptr [[TMP17]], align 1
+; COST1-NEXT:    store i8 0, ptr [[TMP18]], align 1
+; COST1-NEXT:    [[TMP19:%.*]] = getelementptr i8, ptr [[TMP11]], i64 1
+; COST1-NEXT:    [[TMP20:%.*]] = getelementptr i8, ptr [[TMP12]], i64 1
+; COST1-NEXT:    [[TMP21:%.*]] = getelementptr i8, ptr [[TMP13]], i64 1
+; COST1-NEXT:    [[TMP22:%.*]] = getelementptr i8, ptr [[TMP14]], i64 1
+; COST1-NEXT:    store i8 0, ptr [[TMP19]], align 1
+; COST1-NEXT:    store i8 0, ptr [[TMP20]], align 1
+; COST1-NEXT:    store i8 0, ptr [[TMP21]], align 1
+; COST1-NEXT:    store i8 0, ptr [[TMP22]], align 1
+; COST1-NEXT:    store i8 0, ptr [[TMP11]], align 1
+; COST1-NEXT:    store i8 0, ptr [[TMP12]], align 1
+; COST1-NEXT:    store i8 0, ptr [[TMP13]], align 1
+; COST1-NEXT:    store i8 0, ptr [[TMP14]], align 1
+; COST1-NEXT:    [[INDEX_NEXT2]] = add nuw i64 [[INDEX1]], 4
+; COST1-NEXT:    [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
+; COST1-NEXT:    [[TMP23:%.*]] = icmp eq i64 [[INDEX_NEXT2]], 100
+; COST1-NEXT:    br i1 [[TMP23]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP17:![0-9]+]]
+; COST1:       [[VEC_EPILOG_MIDDLE_BLOCK]]:
+; COST1-NEXT:    br i1 false, [[EXIT]], label %[[VEC_EPILOG_SCALAR_PH]]
+; COST1:       [[VEC_EPILOG_SCALAR_PH]]:
+;
+; COST10-LABEL: define void @interleave_group(
+; COST10-SAME: ptr [[DST:%.*]]) #[[ATTR1:[0-9]+]] {
+; COST10-NEXT:  [[ITER_CHECK:.*:]]
+; COST10-NEXT:    br i1 false, label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
+; COST10:       [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
+; COST10-NEXT:    br i1 false, label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]]
+; COST10:       [[VECTOR_PH]]:
+; COST10-NEXT:    br label %[[VECTOR_BODY:.*]]
+; COST10:       [[VECTOR_BODY]]:
+; COST10-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COST10-NEXT:    [[TMP0:%.*]] = mul i64 [[INDEX]], 3
+; COST10-NEXT:    [[TMP1:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP0]]
+; COST10-NEXT:    store <48 x i8> zeroinitializer, ptr [[TMP1]], align 1
+; COST10-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
+; COST10-NEXT:    [[TMP2:%.*]] = icmp eq i64 [[INDEX_NEXT]], 96
+; COST10-NEXT:    br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
+; COST10:       [[MIDDLE_BLOCK]]:
+; COST10-NEXT:    br i1 false, [[EXIT:label %.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
+; COST10:       [[VEC_EPILOG_ITER_CHECK]]:
+; COST10-NEXT:    br i1 false, label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF4]]
+; COST10:       [[VEC_EPILOG_PH]]:
+; COST10-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 96, %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; COST10-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[BC_RESUME_VAL]], i64 0
+; COST10-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; COST10-NEXT:    [[INDUCTION:%.*]] = add <4 x i64> [[BROADCAST_SPLAT]], <i64 0, i64 1, i64 2, i64 3>
+; COST10-NEXT:    br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
+; COST10:       [[VEC_EPILOG_VECTOR_BODY]]:
+; COST10-NEXT:    [[INDEX1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT2:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; COST10-NEXT:    [[VEC_IND:%.*]] = phi <4 x i64> [ [[INDUCTION]], %[[VEC_EPILOG_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; COST10-NEXT:    [[TMP3:%.*]] = mul <4 x i64> [[VEC_IND]], splat (i64 3)
+; COST10-NEXT:    [[TMP4:%.*]] = extractelement <4 x i64> [[TMP3]], i32 0
+; COST10-NEXT:    [[TMP5:%.*]] = extractelement <4 x i64> [[TMP3]], i32 1
+; COST10-NEXT:    [[TMP6:%.*]] = extractelement <4 x i64> [[TMP3]], i32 2
+; COST10-NEXT:    [[TMP7:%.*]] = extractelement <4 x i64> [[TMP3]], i32 3
+; COST10-NEXT:    [[TMP8:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP4]]
+; COST10-NEXT:    [[TMP9:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP5]]
+; COST10-NEXT:    [[TMP10:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP6]]
+; COST10-NEXT:    [[TMP11:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP7]]
+; COST10-NEXT:    [[TMP12:%.*]] = getelementptr i8, ptr [[TMP8]], i64 2
+; COST10-NEXT:    [[TMP13:%.*]] = getelementptr i8, ptr [[TMP9]], i64 2
+; COST10-NEXT:    [[TMP14:%.*]] = getelementptr i8, ptr [[TMP10]], i64 2
+; COST10-NEXT:    [[TMP15:%.*]] = getelementptr i8, ptr [[TMP11]], i64 2
+; COST10-NEXT:    store i8 0, ptr [[TMP12]], align 1
+; COST10-NEXT:    store i8 0, ptr [[TMP13]], align 1
+; COST10-NEXT:    store i8 0, ptr [[TMP14]], align 1
+; COST10-NEXT:    store i8 0, ptr [[TMP15]], align 1
+; COST10-NEXT:    [[TMP16:%.*]] = getelementptr i8, ptr [[TMP8]], i64 1
+; COST10-NEXT:    [[TMP17:%.*]] = getelementptr i8, ptr [[TMP9]], i64 1
+; COST10-NEXT:    [[TMP18:%.*]] = getelementptr i8, ptr [[TMP10]], i64 1
+; COST10-NEXT:    [[TMP19:%.*]] = getelementptr i8, ptr [[TMP11]], i64 1
+; COST10-NEXT:    store i8 0, ptr [[TMP16]], align 1
+; COST10-NEXT:    store i8 0, ptr [[TMP17]], align 1
+; COST10-NEXT:    store i8 0, ptr [[TMP18]], align 1
+; COST10-NEXT:    store i8 0, ptr [[TMP19]], align 1
+; COST10-NEXT:    store i8 0, ptr [[TMP8]], align 1
+; COST10-NEXT:    store i8 0, ptr [[TMP9]], align 1
+; COST10-NEXT:    store i8 0, ptr [[TMP10]], align 1
+; COST10-NEXT:    store i8 0, ptr [[TMP11]], align 1
+; COST10-NEXT:    [[INDEX_NEXT2]] = add nuw i64 [[INDEX1]], 4
+; COST10-NEXT:    [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
+; COST10-NEXT:    [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT2]], 100
+; COST10-NEXT:    br i1 [[TMP20]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP17:![0-9]+]]
+; COST10:       [[VEC_EPILOG_MIDDLE_BLOCK]]:
+; COST10-NEXT:    br i1 false, [[EXIT]], label %[[VEC_EPILOG_SCALAR_PH]]
+; COST10:       [[VEC_EPILOG_SCALAR_PH]]:
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %iv.3 = mul i64 %iv, 3
+  %gep.0 = getelementptr i8, ptr %dst, i64 %iv.3
+  %gep.2 = getelementptr i8, ptr %gep.0, i64 2
+  store i8 0, ptr %gep.2, align 1
+  %gep.1 = getelementptr i8, ptr %gep.0, i64 1
+  store i8 0, ptr %gep.1, align 1
+  store i8 0, ptr %gep.0, align 1
+  %iv.next = add i64 %iv, 1
+  %ec = icmp eq i64 %iv, 100
+  br i1 %ec, label %exit, label %loop
+
+exit:
+  ret void
+}
+
 attributes #0 = { "target-features"="+neon,+sve" vscale_range(1,16) }
+attributes #1 = { "target-cpu"="neoverse-512tvb" }
 
 declare void @llvm.assume(i1 noundef)
 declare i64 @llvm.umin.i64(i64, i64)
diff --git a/llvm/test/Transforms/LoopVectorize/hoist-predicated-loads-with-predicated-stores.ll b/llvm/test/Transforms/LoopVectorize/hoist-predicated-loads-with-predicated-stores.ll
index ae772da8862b3..cdbe9bb555834 100644
--- a/llvm/test/Transforms/LoopVectorize/hoist-predicated-loads-with-predicated-stores.ll
+++ b/llvm/test/Transforms/LoopVectorize/hoist-predicated-loads-with-predicated-stores.ll
@@ -755,7 +755,7 @@ define void @sink_multiple_store_groups_noalias_via_scev(ptr %dst, ptr %src) {
 ; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    br label %[[VECTOR_MEMCHECK:.*]]
 ; CHECK:       [[VECTOR_MEMCHECK]]:
-; CHECK-NEXT:    [[SCEVGEP:%.*]] = getelementptr i8, ptr [[DST]], i64 12688
+; CHECK-NEXT:    [[SCEVGEP:%.*]] = getelementptr i8, ptr [[DST]], i64 12696
 ; CHECK-NEXT:    [[SCEVGEP8:%.*]] = getelementptr i8, ptr [[SRC]], i64 12828
 ; CHECK-NEXT:    [[BOUND1:%.*]] = icmp ult ptr [[DST]], [[SCEVGEP8]]
 ; CHECK-NEXT:    [[BOUND2:%.*]] = icmp ult ptr [[SRC]], [[SCEVGEP]]
@@ -764,88 +764,59 @@ define void @sink_multiple_store_groups_noalias_via_scev(ptr %dst, ptr %src) {
 ; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
-; CHECK-NEXT:    [[INDEX1:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE9:.*]] ]
+; CHECK-NEXT:    [[INDEX1:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE7:.*]] ]
 ; CHECK-NEXT:    [[INDEX:%.*]] = mul i64 [[INDEX1]], 16
 ; CHECK-NEXT:    [[IV:%.*]] = add i64 [[INDEX]], 0
 ; CHECK-NEXT:    [[TMP17:%.*]] = add i64 [[INDEX]], 16
 ; CHECK-NEXT:    [[GEP_SRC:%.*]] = getelementptr double, ptr [[SRC]], i64 [[IV]]
 ; CHECK-NEXT:    [[TMP22:%.*]] = getelementptr double, ptr [[SRC]], i64 [[TMP17]]
-; CHECK-NEXT:    [[TMP23:%.*]] = insertelement <2 x ptr> poison, ptr [[GEP_SRC]], i32 0
-; CHECK-NEXT:    [[TMP24:%.*]] = insertelement <2 x ptr> [[TMP23]], ptr [[TMP22]], i32 1
 ; CHECK-NEXT:    [[GEP_FLAG:%.*]] = getelementptr i8, ptr [[GEP_SRC]], i64 152
 ; CHECK-NEXT:    [[TMP26:%.*]] = getelementptr i8, ptr [[TMP22]], i64 152
 ; CHECK-NEXT:    [[TMP27:%.*]] = load i32, ptr [[GEP_FLAG]], align 4, !alias.scope [[META78:![0-9]+]]
 ; CHECK-NEXT:    [[TMP28:%.*]] = load i32, ptr [[TMP26]], align 4, !alias.scope [[META78]]
 ; CHECK-NEXT:    [[TMP29:%.*]] = insertelement <2 x i32> poison, i32 [[TMP27]], i32 0
 ; CHECK-NEXT:    [[TMP30:%.*]] = insertelement <2 x i32> [[TMP29]], i32 [[TMP28]], i32 1
-; CHECK-NEXT:    [[TMP31:%.*]] = icmp eq <2 x i32> [[TMP30]], zeroinitializer
+; CHECK-NEXT:    [[TMP10:%.*]] = icmp eq <2 x i32> [[TMP30]], zeroinitializer
 ; CHECK-NEXT:    [[TMP13:%.*]] = load double, ptr [[GEP_SRC]], align 8, !alias.scope [[META78]]
 ; CHECK-NEXT:    [[TMP14:%.*]] = load double, ptr [[TMP22]], align 8, !alias.scope [[META78]]
 ; CHECK-NEXT:    [[TMP15:%.*]] = insertelement <2 x double> poison, double [[TMP13]], i32 0
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = insertelement <2 x double> [[TMP15]], double [[TMP14]], i32 1
-; CHECK-NEXT:    [[TMP33:%.*]] = xor <2 x i1> [[TMP31]], splat (i1 true)
+; CHECK-NEXT:    [[TMP16:%.*]] = xor <2 x i1> [[TMP10]], splat (i1 true)
 ; CHECK-NEXT:    [[TMP34:%.*]] = fadd <2 x double> [[WIDE_LOAD]], splat (double 8.000000e+00)
-; CHECK-NEXT:    [[GEP_DST1_ELSE:%.*]] = getelementptr double, ptr [[DST]], i64 [[IV]]
-; CHECK-NEXT:    [[TMP37:%.*]] = getelementptr double, ptr [[DST]], i64 [[TMP17]]
-; CHECK-NEXT:    [[TMP38:%.*]] = insertelement <2 x ptr> poison, ptr [[GEP_DST1_ELSE]], i32 0
-; CHECK-NEXT:    [[TMP39:%.*]] = insertelement <2 x ptr> [[TMP38]], ptr [[TMP37]], i32 1
-; CHECK-NEXT:    [[TMP40:%.*]] = extractelement <2 x i1> [[TMP33]], i32 0
-; CHECK-NEXT:    br i1 [[TMP40]], label %[[PRED_LOAD_IF:.*]], label %[[PRED_LOAD_CONTINUE:.*]]
-; CHECK:       [[PRED_LOAD_IF]]:
-; CHECK-NEXT:    [[TMP41:%.*]] = extractelement <2 x double> [[TMP34]], i32 0
-; CHECK-NEXT:    store double [[TMP41]], ptr [[GEP_DST1_ELSE]], align 8, !alias.scope [[META81:![0-9]+]], !noalias [[META78]]
-; CHECK-NEXT:    [[GEP_SRC_16:%.*]] = getelementptr i8, ptr [[GEP_SRC]], i64 16
-; CHECK-NEXT:    [[TMP43:%.*]] = load double, ptr [[GEP_SRC_16]], align 8, !alias.scope [[META78]]
-; CHECK-NEXT:    [[TMP44:%.*]] = insertelement <2 x double> poison, double [[TMP43]], i32 0
-; CHECK-NEXT:    br label %[[PRED_LOAD_CONTINUE]]
-; CHECK:       [[PRED_LOAD_CONTINUE]]:
-; CHECK-NEXT:    [[TMP45:%.*]] = phi <2 x double> [ poison, %[[VECTOR_BODY]] ], [ [[TMP44]], %[[PRED_LOAD_IF]] ]
-; CHECK-NEXT:    [[TMP46:%.*]] = extractelement <2 x i1> [[TMP33]], i32 1
-; CHECK-NEXT:    br i1 [[TMP46]], label %[[PRED_LOAD_IF2:.*]], label %[[PRED_LOAD_CONTINUE3:.*]]
-; CHECK:       [[PRED_LOAD_IF2]]:
-; CHECK-NEXT:    [[TMP47:%.*]] = extractelement <2 x double> [[TMP34]], i32 1
-; CHECK-NEXT:    store double [[TMP47]], ptr [[TMP37]], align 8, !alias.scope [[META81]], !noalias [[META78]]
-; CHECK-NEXT:    [[TMP48:%.*]] = getelementptr i8, ptr [[TMP22]], i64 16
-; CHECK-NEXT:    [[TMP49:%.*]] = load double, ptr [[TMP48]], align 8, !alias.scope [[META78]]
-; CHECK-NEXT:    [[TMP50:%.*]] = insertelement <2 x double> [[TMP45]], double [[TMP49]], i32 1
-; CHECK-NEXT:    br label %[[PRED_LOAD_CONTINUE3]]
-; CHECK:       [[PRED_LOAD_CONTINUE3]]:
-; CHECK-NEXT:    [[TMP51:%.*]] = phi <2 x double> [ [[TMP45]], %[[PRED_LOAD_CONTINUE]] ], [ [[TMP50]], %[[PRED_LOAD_IF2]] ]
-; CHECK-NEXT:    [[TMP53:%.*]] = fmul <2 x double> splat (double 2.000000e+01), [[TMP51]]
-; CHECK-NEXT:    [[TMP54:%.*]] = extractelement <2 x i1> [[TMP33]], i32 0
-; CHECK-NEXT:    br i1 [[TMP54]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
+; CHECK-NEXT:    [[TMP24:%.*]] = extractelement <2 x i1> [[TMP16]], i32 0
+; CHECK-NEXT:    br i1 [[TMP24]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
 ; CHECK:       [[PRED_STORE_IF]]:
-; CHECK-NEXT:    [[GEP_DST2_ELSE:%.*]] = getelementptr i8, ptr [[GEP_DST1_ELSE]], i64 8
-; CHECK-NEXT:    [[TMP56:%.*]] = extractelement <2 x double> [[TMP53]], i32 0
-; CHECK-NEXT:    store double [[TMP56]], ptr [[GEP_DST2_ELSE]], align 8, !alias.scope [[META81]], !noalias [[META78]]
+; CHECK-NEXT:    [[TMP18:%.*]] = getelementptr double, ptr [[DST]], i64 [[IV]]
+; CHECK-NEXT:    [[TMP19:%.*]] = extractelement <2 x double> [[TMP34]], i32 0
+; CHECK-NEXT:    store double [[TMP19]], ptr [[TMP18]], align 8, !alias.scope [[META81:![0-9]+]], !noalias [[META78]]
 ; CHECK-NEXT:    br label %[[PRED_STORE_CONTINUE]]
 ; CHECK:       [[PRED_STORE_CONTINUE]]:
-; CHECK-NEXT:    [[TMP57:%.*]] = extractelement <2 x i1> [[TMP33]], i32 1
-; CHECK-NEXT:    br i1 [[TMP57]], label %[[PRED_STORE_IF4:.*]], label %[[PRED_STORE_CONTINUE5:.*]]
+; CHECK-NEXT:    [[TMP20:%.*]] = extractelement <2 x i1> [[TMP16]], i32 1
+; CHECK-NEXT:    br i1 [[TMP20]], label %[[PRED_STORE_IF2:.*]], label %[[PRED_STORE_CONTINUE3:.*]]
+; CHECK:       [[PRED_STORE_IF2]]:
+; CHECK-NEXT:    [[TMP21:%.*]] = getelementptr double, ptr [[DST]], i64 [[TMP17]]
+; CHECK-NEXT:    [[TMP33:%.*]] = extractelement <2 x double> [[TMP34]], i32 1
+; CHECK-NEXT:    store double [[TMP33]], ptr [[TMP21]], align 8, !alias.scope [[META81]], !noalias [[META78]]
+; CHECK-NEXT:    br label %[[PRED_STORE_CONTINUE3]]
+; CHECK:       [[PRED_STORE_CONTINUE3]]:
+; CHECK-NEXT:    [[TMP23:%.*]] = extractelement <2 x i1> [[TMP10]], i32 0
+; CHECK-NEXT:    br i1 [[TMP23]], label %[[PRED_STORE_IF4:.*]], label %[[PRED_STORE_CONTINUE5:.*]]
 ; CHECK:       [[PRED_STORE_IF4]]:
-; CHECK-NEXT:    [[TMP58:%.*]] = getelementptr i8, ptr [[TMP37]], i64 8
-; CHECK-NEXT:    [[TMP59:%.*]] = extractelement <2 x double> [[TMP53]], i32 1
-; CHECK-NEXT:    store double [[TMP59]], ptr [[TMP58]], align 8, !alias.scope [[META81]], !noalias [[META78]]
+; CHECK-NEXT:    [[TMP31:%.*]] = getelementptr double, ptr [[DST]], i64 [[IV]]
+; CHECK-NEXT:    store double [[TMP13]], ptr [[TMP31]], align 8, !alias.scope [[META81]], !noalias [[META78]]
+; CHECK-NEXT:    [[TMP37:%.*]] = getelementptr i8, ptr [[TMP31]], i64 16
+; CHECK-NEXT:    store double 1.000000e+01, ptr [[TMP37]], align 8, !alias.scope [[META81]], !noalias [[META78]]
 ; CHECK-NEXT:    br label %[[PRED_STORE_CONTINUE5]]
 ; CHECK:       [[PRED_STORE_CONTINUE5]]:
-; CHECK-NEXT:    [[TMP60:%.*]] = extractelement <2 x i1> [[TMP31]], i32 0
-; CHECK-NEXT:    br i1 [[TMP60]], label %[[PRED_STORE_IF6:.*]], label %[[PRED_STORE_CONTINUE7:.*]]
+; CHECK-NEXT:    [[TMP25:%.*]] = extractelement <2 x i1> [[TMP10]], i32 1
+; CHECK-NEXT:    br i1 [[TMP25]], label %[[PRED_STORE_IF6:.*]], label %[[PRED_STORE_CONTINUE7]]
 ; CHECK:       [[PRED_STORE_IF6]]:
-; CHECK-NEXT:    [[TMP62:%.*]] = getelementptr double, ptr [[DST]], i64 [[IV]]
-; CHECK-NEXT:    store double [[TMP13]], ptr [[TMP62]], align 8, !alias.scope [[META81]], !noalias [[META78]]
-; CHECK-NEXT:    [[TMP64:%.*]] = getelementptr i8, ptr [[TMP62]], i64 8
-; CHECK-NEXT:    store double 1.000000e+01, ptr [[TMP64]], align 8, !alias.scope [[META81]], !noalias [[META78]]
+; CHECK-NEXT:    [[TMP32:%.*]] = getelementptr double, ptr [[DST]], i64 [[TMP17]]
+; CHECK-NEXT:    store double [[TMP14]], ptr [[TMP32]], align 8, !alias.scope [[META81]], !noalias [[META78]]
+; CHECK-NEXT:    [[TMP47:%.*]] = getelementptr i8, ptr [[TMP32]], i64 16
+; CHECK-NEXT:    store double 1.000000e+01, ptr [[TMP47]], align 8, !alias.scope [[META81]], !noalias [[META78]]
 ; CHECK-NEXT:    br label %[[PRED_STORE_CONTINUE7]]
 ; CHECK:       [[PRED_STORE_CONTINUE7]]:
-; CHECK-NEXT:    [[TMP66:%.*]] = extractelement <2 x i1> [[TMP31]], i32 1
-; CHECK-NEXT:    br i1 [[TMP66]], label %[[PRED_STORE_IF8:.*]], label %[[PRED_STORE_CONTINUE9]]
-; CHECK:       [[PRED_STORE_IF8]]:
-; CHECK-NEXT:    [[TMP68:%.*]] = getelementptr double, ptr [[DST]], i64 [[TMP17]]
-; CHECK-NEXT:    store double [[TMP14]], ptr [[TMP68]], align 8, !alias.scope [[META81]], !noalias [[META78]]
-; CHECK-NEXT:    [[TMP70:%.*]] = getelementptr i8, ptr [[TMP68]], i64 8
-; CHECK-NEXT:    store double 1.000000e+01, ptr [[TMP70]], align 8, !alias.scope [[META81]], !noalias [[META78]]
-; CHECK-NEXT:    br label %[[PRED_STORE_CONTINUE9]]
-; CHECK:       [[PRED_STORE_CONTINUE9]]:
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX1]], 2
 ; CHECK-NEXT:    [[TMP52:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
 ; CHECK-NEXT:    br i1 [[TMP52]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP83:![0-9]+]]
@@ -865,6 +836,117 @@ loop:
   %v.1 = load double, ptr %gep.src, align 8
   br i1 %cmp, label %then, label %else
 
+then:
+  %gep.dst1.then = getelementptr double, ptr %dst, i64 %iv
+  store double %v.1, ptr %gep.dst1.then, align 8
+  %gep.dst2.then = getelementptr i8, ptr %gep.dst1.then, i64 16
+  store double 10.0, ptr %gep.dst2.then, align 8
+  br label %loop.latch
+
+else:
+  %r.1 = fadd double %v.1, 8.0
+  %gep.dst1.else = getelementptr double, ptr %dst, i64 %iv
+  store double %r.1, ptr %gep.dst1.else, align 8
+  br label %loop.latch
+
+loop.latch:
+  %iv.next = add i64 %iv, 16
+  %exit.cond = icmp eq i64 %iv.next, 1600
+  br i1 %exit.cond, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+; Same as @sink_multiple_store_groups_noalias_via_scev, but the offset between
+; store groups is only 8, which means the alias across VFs.
+define void @sink_multiple_store_groups_alias_via_scev(ptr %dst, ptr %src) {
+; CHECK-LABEL: define void @sink_multiple_store_groups_alias_via_scev(
+; CHECK-SAME: ptr [[DST:%.*]], ptr [[SRC:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br label %[[VECTOR_MEMCHECK:.*]]
+; CHECK:       [[VECTOR_MEMCHECK]]:
+; CHECK-NEXT:    [[SCEVGEP:%.*]] = getelementptr i8, ptr [[DST]], i64 12688
+; CHECK-NEXT:    [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[SRC]], i64 12828
+; CHECK-NEXT:    [[BOUND0:%.*]] = icmp ult ptr [[DST]], [[SCEVGEP1]]
+; CHECK-NEXT:    [[BOUND1:%.*]] = icmp ult ptr [[SRC]], [[SCEVGEP]]
+; CHECK-NEXT:    [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
+; CHECK-NEXT:    br i1 [[FOUND_CONFLICT]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE7:.*]] ]
+; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], 16
+; CHECK-NEXT:    [[IV:%.*]] = add i64 [[OFFSET_IDX]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = add i64 [[OFFSET_IDX]], 16
+; CHECK-NEXT:    [[GEP_SRC:%.*]] = getelementptr double, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr double, ptr [[SRC]], i64 [[TMP1]]
+; CHECK-NEXT:    [[GEP_FLAG:%.*]] = getelementptr i8, ptr [[GEP_SRC]], i64 152
+; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr i8, ptr [[TMP3]], i64 152
+; CHECK-NEXT:    [[TMP8:%.*]] = load i32, ptr [[GEP_FLAG]], align 4, !alias.scope [[META85:![0-9]+]]
+; CHECK-NEXT:    [[TMP9:%.*]] = load i32, ptr [[TMP7]], align 4, !alias.scope [[META85]]
+; CHECK-NEXT:    [[TMP10:%.*]] = insertelement <2 x i32> poison, i32 [[TMP8]], i32 0
+; CHECK-NEXT:    [[TMP11:%.*]] = insertelement <2 x i32> [[TMP10]], i32 [[TMP9]], i32 1
+; CHECK-NEXT:    [[TMP12:%.*]] = icmp eq <2 x i32> [[TMP11]], zeroinitializer
+; CHECK-NEXT:    [[TMP13:%.*]] = load double, ptr [[GEP_SRC]], align 8, !alias.scope [[META85]]
+; CHECK-NEXT:    [[TMP14:%.*]] = load double, ptr [[TMP3]], align 8, !alias.scope [[META85]]
+; CHECK-NEXT:    [[TMP15:%.*]] = insertelement <2 x double> poison, double [[TMP13]], i32 0
+; CHECK-NEXT:    [[TMP16:%.*]] = insertelement <2 x double> [[TMP15]], double [[TMP14]], i32 1
+; CHECK-NEXT:    [[TMP17:%.*]] = xor <2 x i1> [[TMP12]], splat (i1 true)
+; CHECK-NEXT:    [[TMP18:%.*]] = fadd <2 x double> [[TMP16]], splat (double 8.000000e+00)
+; CHECK-NEXT:    [[TMP36:%.*]] = extractelement <2 x i1> [[TMP17]], i32 0
+; CHECK-NEXT:    br i1 [[TMP36]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
+; CHECK:       [[PRED_STORE_IF]]:
+; CHECK-NEXT:    [[TMP20:%.*]] = getelementptr double, ptr [[DST]], i64 [[IV]]
+; CHECK-NEXT:    [[TMP19:%.*]] = extractelement <2 x double> [[TMP18]], i32 0
+; CHECK-NEXT:    store double [[TMP19]], ptr [[TMP20]], align 8, !alias.scope [[META88:![0-9]+]], !noalias [[META85]]
+; CHECK-NEXT:    br label %[[PRED_STORE_CONTINUE]]
+; CHECK:       [[PRED_STORE_CONTINUE]]:
+; CHECK-NEXT:    [[TMP39:%.*]] = extractelement <2 x i1> [[TMP17]], i32 1
+; CHECK-NEXT:    br i1 [[TMP39]], label %[[PRED_STORE_IF2:.*]], label %[[PRED_STORE_CONTINUE3:.*]]
+; CHECK:       [[PRED_STORE_IF2]]:
+; CHECK-NEXT:    [[TMP21:%.*]] = getelementptr double, ptr [[DST]], i64 [[TMP1]]
+; CHECK-NEXT:    [[TMP22:%.*]] = extractelement <2 x double> [[TMP18]], i32 1
+; CHECK-NEXT:    store double [[TMP22]], ptr [[TMP21]], align 8, !alias.scope [[META88]], !noalias [[META85]]
+; CHECK-NEXT:    br label %[[PRED_STORE_CONTINUE3]]
+; CHECK:       [[PRED_STORE_CONTINUE3]]:
+; CHECK-NEXT:    [[TMP42:%.*]] = extractelement <2 x i1> [[TMP12]], i32 0
+; CHECK-NEXT:    br i1 [[TMP42]], label %[[PRED_STORE_IF4:.*]], label %[[PRED_STORE_CONTINUE5:.*]]
+; CHECK:       [[PRED_STORE_IF4]]:
+; CHECK-NEXT:    [[TMP43:%.*]] = getelementptr double, ptr [[DST]], i64 [[IV]]
+; CHECK-NEXT:    store double [[TMP13]], ptr [[TMP43]], align 8, !alias.scope [[META88]], !noalias [[META85]]
+; CHECK-NEXT:    [[TMP44:%.*]] = getelementptr i8, ptr [[TMP43]], i64 8
+; CHECK-NEXT:    store double 1.000000e+01, ptr [[TMP44]], align 8, !alias.scope [[META88]], !noalias [[META85]]
+; CHECK-NEXT:    br label %[[PRED_STORE_CONTINUE5]]
+; CHECK:       [[PRED_STORE_CONTINUE5]]:
+; CHECK-NEXT:    [[TMP45:%.*]] = extractelement <2 x i1> [[TMP12]], i32 1
+; CHECK-NEXT:    br i1 [[TMP45]], label %[[PRED_STORE_IF6:.*]], label %[[PRED_STORE_CONTINUE7]]
+; CHECK:       [[PRED_STORE_IF6]]:
+; CHECK-NEXT:    [[TMP46:%.*]] = getelementptr double, ptr [[DST]], i64 [[TMP1]]
+; CHECK-NEXT:    store double [[TMP14]], ptr [[TMP46]], align 8, !alias.scope [[META88]], !noalias [[META85]]
+; CHECK-NEXT:    [[TMP47:%.*]] = getelementptr i8, ptr [[TMP46]], i64 8
+; CHECK-NEXT:    store double 1.000000e+01, ptr [[TMP47]], align 8, !alias.scope [[META88]], !noalias [[META85]]
+; CHECK-NEXT:    br label %[[PRED_STORE_CONTINUE7]]
+; CHECK:       [[PRED_STORE_CONTINUE7]]:
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; CHECK-NEXT:    [[TMP48:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
+; CHECK-NEXT:    br i1 [[TMP48]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP90:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    br [[EXIT:label %.*]]
+; CHECK:       [[SCALAR_PH]]:
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]
+  %gep.src = getelementptr double, ptr %src, i64 %iv
+  %gep.flag = getelementptr i8, ptr %gep.src, i64 152
+  %c = load i32, ptr %gep.flag, align 4
+  %cmp = icmp eq i32 %c, 0
+  %v.1 = load double, ptr %gep.src, align 8
+  br i1 %cmp, label %then, label %else
+
 then:
   %gep.dst1.then = getelementptr double, ptr %dst, i64 %iv
   store double %v.1, ptr %gep.dst1.then, align 8
@@ -876,11 +958,6 @@ else:
   %r.1 = fadd double %v.1, 8.0
   %gep.dst1.else = getelementptr double, ptr %dst, i64 %iv
   store double %r.1, ptr %gep.dst1.else, align 8
-  %gep.src.16 = getelementptr i8, ptr %gep.src, i64 16
-  %v.3 = load double, ptr %gep.src.16, align 8
-  %r.2 = fmul double 20.0, %v.3
-  %gep.dst2.else = getelementptr i8, ptr %gep.dst1.else, i64 8
-  store double %r.2, ptr %gep.dst2.else, align 8
   br label %loop.latch
 
 loop.latch:
@@ -988,20 +1065,20 @@ define void @test_three_stores_with_different_predicates(ptr %dst, ptr %src, ptr
 ; CHECK-NEXT:    [[TMP0:%.*]] = add i32 [[INDEX]], 0
 ; CHECK-NEXT:    [[TMP1:%.*]] = add i32 [[INDEX]], 1
 ; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[COND]], i32 [[TMP0]]
-; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <2 x i32>, ptr [[TMP2]], align 4, !alias.scope [[META85:![0-9]+]]
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <2 x i32>, ptr [[TMP2]], align 4, !alias.scope [[META92:![0-9]+]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = icmp ule <2 x i32> [[WIDE_LOAD]], splat (i32 11)
 ; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <2 x i1> [[TMP3]], i32 0
 ; CHECK-NEXT:    br i1 [[TMP4]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
 ; CHECK:       [[PRED_STORE_IF]]:
 ; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[DST]], i32 [[TMP0]]
-; CHECK-NEXT:    store i32 1, ptr [[TMP5]], align 4, !alias.scope [[META88:![0-9]+]], !noalias [[META85]]
+; CHECK-NEXT:    store i32 1, ptr [[TMP5]], align 4, !alias.scope [[META95:![0-9]+]], !noalias [[META92]]
 ; CHECK-NEXT:    br label %[[PRED_STORE_CONTINUE]]
 ; CHECK:       [[PRED_STORE_CONTINUE]]:
 ; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <2 x i1> [[TMP3]], i32 1
 ; CHECK-NEXT:    br i1 [[TMP6]], label %[[PRED_STORE_IF2:.*]], label %[[PRED_STORE_CONTINUE3:.*]]
 ; CHECK:       [[PRED_STORE_IF2]]:
 ; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[DST]], i32 [[TMP1]]
-; CHECK-NEXT:    store i32 1, ptr [[TMP7]], align 4, !alias.scope [[META88]], !noalias [[META85]]
+; CHECK-NEXT:    store i32 1, ptr [[TMP7]], align 4, !alias.scope [[META95]], !noalias [[META92]]
 ; CHECK-NEXT:    br label %[[PRED_STORE_CONTINUE3]]
 ; CHECK:       [[PRED_STORE_CONTINUE3]]:
 ; CHECK-NEXT:    [[TMP8:%.*]] = xor <2 x i1> [[TMP3]], splat (i1 true)
@@ -1012,14 +1089,14 @@ define void @test_three_stores_with_different_predicates(ptr %dst, ptr %src, ptr
 ; CHECK-NEXT:    br i1 [[TMP12]], label %[[PRED_STORE_IF4:.*]], label %[[PRED_STORE_CONTINUE5:.*]]
 ; CHECK:       [[PRED_STORE_IF4]]:
 ; CHECK-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i32, ptr [[DST]], i32 [[TMP0]]
-; CHECK-NEXT:    store i32 2, ptr [[TMP13]], align 4, !alias.scope [[META88]], !noalias [[META85]]
+; CHECK-NEXT:    store i32 2, ptr [[TMP13]], align 4, !alias.scope [[META95]], !noalias [[META92]]
 ; CHECK-NEXT:    br label %[[PRED_STORE_CONTINUE5]]
 ; CHECK:       [[PRED_STORE_CONTINUE5]]:
 ; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <2 x i1> [[TMP11]], i32 1
 ; CHECK-NEXT:    br i1 [[TMP14]], label %[[PRED_STORE_IF6:.*]], label %[[PRED_STORE_CONTINUE7:.*]]
 ; CHECK:       [[PRED_STORE_IF6]]:
 ; CHECK-NEXT:    [[TMP15:%.*]] = getelementptr inbounds i32, ptr [[DST]], i32 [[TMP1]]
-; CHECK-NEXT:    store i32 2, ptr [[TMP15]], align 4, !alias.scope [[META88]], !noalias [[META85]]
+; CHECK-NEXT:    store i32 2, ptr [[TMP15]], align 4, !alias.scope [[META95]], !noalias [[META92]]
 ; CHECK-NEXT:    br label %[[PRED_STORE_CONTINUE7]]
 ; CHECK:       [[PRED_STORE_CONTINUE7]]:
 ; CHECK-NEXT:    [[TMP16:%.*]] = icmp ule <2 x i32> [[WIDE_LOAD]], splat (i32 9)
@@ -1028,19 +1105,19 @@ define void @test_three_stores_with_different_predicates(ptr %dst, ptr %src, ptr
 ; CHECK-NEXT:    br i1 [[TMP18]], label %[[PRED_STORE_IF8:.*]], label %[[PRED_STORE_CONTINUE9:.*]]
 ; CHECK:       [[PRED_STORE_IF8]]:
 ; CHECK-NEXT:    [[TMP19:%.*]] = getelementptr inbounds i32, ptr [[DST]], i32 [[TMP0]]
-; CHECK-NEXT:    store i32 3, ptr [[TMP19]], align 4, !alias.scope [[META88]], !noalias [[META85]]
+; CHECK-NEXT:    store i32 3, ptr [[TMP19]], align 4, !alias.scope [[META95]], !noalias [[META92]]
 ; CHECK-NEXT:    br label %[[PRED_STORE_CONTINUE9]]
 ; CHECK:       [[PRED_STORE_CONTINUE9]]:
 ; CHECK-NEXT:    [[TMP20:%.*]] = extractelement <2 x i1> [[TMP17]], i32 1
 ; CHECK-NEXT:    br i1 [[TMP20]], label %[[PRED_STORE_IF10:.*]], label %[[PRED_STORE_CONTINUE11]]
 ; CHECK:       [[PRED_STORE_IF10]]:
 ; CHECK-NEXT:    [[TMP21:%.*]] = getelementptr inbounds i32, ptr [[DST]], i32 [[TMP1]]
-; CHECK-NEXT:    store i32 3, ptr [[TMP21]], align 4, !alias.scope [[META88]], !noalias [[META85]]
+; CHECK-NEXT:    store i32 3, ptr [[TMP21]], align 4, !alias.scope [[META95]], !noalias [[META92]]
 ; CHECK-NEXT:    br label %[[PRED_STORE_CONTINUE11]]
 ; CHECK:       [[PRED_STORE_CONTINUE11]]:
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
 ; CHECK-NEXT:    [[TMP22:%.*]] = icmp eq i32 [[INDEX_NEXT]], 100
-; CHECK-NEXT:    br i1 [[TMP22]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP90:![0-9]+]]
+; CHECK-NEXT:    br i1 [[TMP22]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP97:![0-9]+]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    br [[EXIT:label %.*]]
 ; CHECK:       [[SCALAR_PH]]:
diff --git a/llvm/test/Transforms/LowerConstantIntrinsics/builtin-object-size-idxsize.ll b/llvm/test/Transforms/LowerConstantIntrinsics/builtin-object-size-idxsize.ll
new file mode 100644
index 0000000000000..3fb2789ce0e49
--- /dev/null
+++ b/llvm/test/Transforms/LowerConstantIntrinsics/builtin-object-size-idxsize.ll
@@ -0,0 +1,262 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -passes=lower-constant-intrinsics -S < %s | FileCheck %s
+
+; Some extra tests using 16-bit pointers and 16-bit index type size. This
+; allows us to for example test what happens when the index type used in a
+; getelementptr does not match with the index type size (e.g. when not running
+; full opt pipeline before the lower-constant-intrinsics pass).
+
+target datalayout = "e-p:16:16:16"
+
+
+define i32 @possible_out_of_bounds_gep_i8(i1 %c0, i1 %c1) {
+; CHECK-LABEL: define i32 @possible_out_of_bounds_gep_i8(
+; CHECK-SAME: i1 [[C0:%.*]], i1 [[C1:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[OBJ:%.*]] = alloca [5 x i8], align 1
+; CHECK-NEXT:    [[OFFSET:%.*]] = select i1 [[C0]], i8 2, i8 10
+; CHECK-NEXT:    [[PTR_SLIDE:%.*]] = getelementptr i8, ptr [[OBJ]], i8 [[OFFSET]]
+; CHECK-NEXT:    [[RES:%.*]] = select i1 [[C1]], i32 3, i32 0
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+entry:
+  %obj = alloca [5 x i8]
+  %offset = select i1 %c0, i8 2, i8 10
+  %ptr.slide = getelementptr i8, ptr %obj, i8 %offset
+  %objsize_max = call i32 @llvm.objectsize.i32.p0(ptr %ptr.slide, i1 false, i1 true, i1 false)
+  %objsize_min = call i32 @llvm.objectsize.i32.p0(ptr %ptr.slide, i1 true, i1 true, i1 false)
+  %res = select i1 %c1, i32 %objsize_max, i32 %objsize_min
+  ret i32 %res
+}
+
+define i32 @possible_out_of_bounds_gep_i16(i1 %c0, i1 %c1) {
+; CHECK-LABEL: define i32 @possible_out_of_bounds_gep_i16(
+; CHECK-SAME: i1 [[C0:%.*]], i1 [[C1:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[OBJ:%.*]] = alloca [5 x i8], align 1
+; CHECK-NEXT:    [[OFFSET:%.*]] = select i1 [[C0]], i16 2, i16 10
+; CHECK-NEXT:    [[PTR_SLIDE:%.*]] = getelementptr i8, ptr [[OBJ]], i16 [[OFFSET]]
+; CHECK-NEXT:    [[RES:%.*]] = select i1 [[C1]], i32 3, i32 0
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+entry:
+  %obj = alloca [5 x i8]
+  %offset = select i1 %c0, i16 2, i16 10
+  %ptr.slide = getelementptr i8, ptr %obj, i16 %offset
+  %objsize_max = call i32 @llvm.objectsize.i32.p0(ptr %ptr.slide, i1 false, i1 true, i1 false)
+  %objsize_min = call i32 @llvm.objectsize.i32.p0(ptr %ptr.slide, i1 true, i1 true, i1 false)
+  %res = select i1 %c1, i32 %objsize_max, i32 %objsize_min
+  ret i32 %res
+}
+
+define i32 @possible_out_of_bounds_gep_i32(i1 %c0, i1 %c1) {
+; CHECK-LABEL: define i32 @possible_out_of_bounds_gep_i32(
+; CHECK-SAME: i1 [[C0:%.*]], i1 [[C1:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[OBJ:%.*]] = alloca [5 x i8], align 1
+; CHECK-NEXT:    [[OFFSET:%.*]] = select i1 [[C0]], i32 2, i32 10
+; CHECK-NEXT:    [[PTR_SLIDE:%.*]] = getelementptr i8, ptr [[OBJ]], i32 [[OFFSET]]
+; CHECK-NEXT:    [[RES:%.*]] = select i1 [[C1]], i32 3, i32 0
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+entry:
+  %obj = alloca [5 x i8]
+  %offset = select i1 %c0, i32 2, i32 10
+  %ptr.slide = getelementptr i8, ptr %obj, i32 %offset
+  %objsize_max = call i32 @llvm.objectsize.i32.p0(ptr %ptr.slide, i1 false, i1 true, i1 false)
+  %objsize_min = call i32 @llvm.objectsize.i32.p0(ptr %ptr.slide, i1 true, i1 true, i1 false)
+  %res = select i1 %c1, i32 %objsize_max, i32 %objsize_min
+  ret i32 %res
+}
+
+; SROA would produce IR like this if applied to @possible_out_of_bounds_gep_i16.
+; FIXME: The %objsize_min result here is invalid.
+define i32 @possible_out_of_bounds_gep_i16_sroa(i1 %c0, i1 %c1) {
+; CHECK-LABEL: define i32 @possible_out_of_bounds_gep_i16_sroa(
+; CHECK-SAME: i1 [[C0:%.*]], i1 [[C1:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[OBJ:%.*]] = alloca [5 x i8], align 1
+; CHECK-NEXT:    [[DOTSROA_GEP:%.*]] = getelementptr i8, ptr [[OBJ]], i16 2
+; CHECK-NEXT:    [[DOTSROA_GEP1:%.*]] = getelementptr i8, ptr [[OBJ]], i16 10
+; CHECK-NEXT:    [[OFFSET_SROA_SEL:%.*]] = select i1 [[C0]], ptr [[DOTSROA_GEP]], ptr [[DOTSROA_GEP1]]
+; CHECK-NEXT:    [[RES:%.*]] = select i1 [[C1]], i32 3, i32 65531
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+entry:
+  %obj = alloca [5 x i8], align 1
+  %.sroa.gep = getelementptr i8, ptr %obj, i16 2
+  %.sroa.gep1 = getelementptr i8, ptr %obj, i16 10
+  %offset.sroa.sel = select i1 %c0, ptr %.sroa.gep, ptr %.sroa.gep1
+  %objsize_max = call i32 @llvm.objectsize.i32.p0(ptr %offset.sroa.sel, i1 false, i1 true, i1 false)
+  %objsize_min = call i32 @llvm.objectsize.i32.p0(ptr %offset.sroa.sel, i1 true, i1 true, i1 false)
+  %res = select i1 %c1, i32 %objsize_max, i32 %objsize_min
+  ret i32 %res
+}
+
+; Indices are truncated to the pointer size in a gep. So "i32 -65526" should
+; be treated as "i16 10" and we expect same result as for
+; @possible_out_of_bounds_gep_i16 above.
+; FIXME: The result here is incorrect (max/min is swapped).
+define i32 @possible_out_of_bounds_gep_i32_trunc(i1 %c0, i1 %c1) {
+; CHECK-LABEL: define i32 @possible_out_of_bounds_gep_i32_trunc(
+; CHECK-SAME: i1 [[C0:%.*]], i1 [[C1:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[OBJ:%.*]] = alloca [5 x i8], align 1
+; CHECK-NEXT:    [[OFFSET:%.*]] = select i1 [[C0]], i32 2, i32 -65526
+; CHECK-NEXT:    [[PTR_SLIDE:%.*]] = getelementptr i8, ptr [[OBJ]], i32 [[OFFSET]]
+; CHECK-NEXT:    [[RES:%.*]] = select i1 [[C1]], i32 0, i32 3
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+entry:
+  %obj = alloca [5 x i8]
+  %offset = select i1 %c0, i32 2, i32 -65526  ; 0xffff000a
+  %ptr.slide = getelementptr i8, ptr %obj, i32 %offset
+  %objsize_max = call i32 @llvm.objectsize.i32.p0(ptr %ptr.slide, i1 false, i1 true, i1 false)
+  %objsize_min = call i32 @llvm.objectsize.i32.p0(ptr %ptr.slide, i1 true, i1 true, i1 false)
+  %res = select i1 %c1, i32 %objsize_max, i32 %objsize_min
+  ret i32 %res
+}
+
+define i32 @out_of_bounds_gep_i8(i1 %c0, i1 %c1) {
+; CHECK-LABEL: define i32 @out_of_bounds_gep_i8(
+; CHECK-SAME: i1 [[C0:%.*]], i1 [[C1:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[OBJ:%.*]] = alloca [5 x i8], align 1
+; CHECK-NEXT:    [[PTR_SLIDE:%.*]] = getelementptr i8, ptr [[OBJ]], i8 -128
+; CHECK-NEXT:    [[RES:%.*]] = select i1 [[C1]], i32 -1, i32 0
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+entry:
+  %obj = alloca [5 x i8]
+  %ptr.slide = getelementptr i8, ptr %obj, i8 -128
+  %objsize_max = call i32 @llvm.objectsize.i32.p0(ptr %ptr.slide, i1 false, i1 true, i1 false)
+  %objsize_min = call i32 @llvm.objectsize.i32.p0(ptr %ptr.slide, i1 true, i1 true, i1 false)
+  %res = select i1 %c1, i32 %objsize_max, i32 %objsize_min
+  ret i32 %res
+}
+
+define i32 @out_of_bounds_gep_i32(i1 %c0, i1 %c1) {
+; CHECK-LABEL: define i32 @out_of_bounds_gep_i32(
+; CHECK-SAME: i1 [[C0:%.*]], i1 [[C1:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[OBJ:%.*]] = alloca [5 x i8], align 1
+; CHECK-NEXT:    [[PTR_SLIDE:%.*]] = getelementptr i8, ptr [[OBJ]], i32 10
+; CHECK-NEXT:    ret i32 0
+;
+entry:
+  %obj = alloca [5 x i8]
+  %ptr.slide = getelementptr i8, ptr %obj, i32 10
+  %objsize_max = call i32 @llvm.objectsize.i32.p0(ptr %ptr.slide, i1 false, i1 true, i1 false)
+  %objsize_min = call i32 @llvm.objectsize.i32.p0(ptr %ptr.slide, i1 true, i1 true, i1 false)
+  %res = select i1 %c1, i32 %objsize_max, i32 %objsize_min
+  ret i32 %res
+}
+
+define i32 @out_of_bounds_gep_i32_trunc(i1 %c0, i1 %c1) {
+; CHECK-LABEL: define i32 @out_of_bounds_gep_i32_trunc(
+; CHECK-SAME: i1 [[C0:%.*]], i1 [[C1:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[OBJ:%.*]] = alloca [5 x i8], align 1
+; CHECK-NEXT:    [[PTR_SLIDE:%.*]] = getelementptr i8, ptr [[OBJ]], i32 -65526
+; CHECK-NEXT:    ret i32 0
+;
+entry:
+  %obj = alloca [5 x i8]
+  %ptr.slide = getelementptr i8, ptr %obj, i32 -65526
+  %objsize_max = call i32 @llvm.objectsize.i32.p0(ptr %ptr.slide, i1 false, i1 true, i1 false)
+  %objsize_min = call i32 @llvm.objectsize.i32.p0(ptr %ptr.slide, i1 true, i1 true, i1 false)
+  %res = select i1 %c1, i32 %objsize_max, i32 %objsize_min
+  ret i32 %res
+}
+
+; In this test the index will be out-of-bounds, but the current analysis won't
+; detect that. The analysis will find out that %offset is in the range [-2,
+; 10] which includes valid offsets that aren't out-of-bounds. Therefore we can
+; expect to get -1 for %objsize_max even if an advanced analysis should be
+; able to derive that we are out-of-bounds (returning 0 also for
+; %objsize_max).
+define i32 @out_of_bounds_gep_i16_pos_neg(i1 %c0, i1 %c1) {
+; CHECK-LABEL: define i32 @out_of_bounds_gep_i16_pos_neg(
+; CHECK-SAME: i1 [[C0:%.*]], i1 [[C1:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[OBJ:%.*]] = alloca [5 x i8], align 1
+; CHECK-NEXT:    [[OFFSET:%.*]] = select i1 [[C0]], i32 10, i32 -2
+; CHECK-NEXT:    [[PTR_SLIDE:%.*]] = getelementptr i8, ptr [[OBJ]], i32 [[OFFSET]]
+; CHECK-NEXT:    [[RES:%.*]] = select i1 [[C1]], i32 -1, i32 0
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+entry:
+  %obj = alloca [5 x i8]
+  %offset = select i1 %c0, i32 10, i32 -2
+  %ptr.slide = getelementptr i8, ptr %obj, i32 %offset
+  %objsize_max = call i32 @llvm.objectsize.i32.p0(ptr %ptr.slide, i1 false, i1 true, i1 false)
+  %objsize_min = call i32 @llvm.objectsize.i32.p0(ptr %ptr.slide, i1 true, i1 true, i1 false)
+  %res = select i1 %c1, i32 %objsize_max, i32 %objsize_min
+  ret i32 %res
+}
+
+; With 16-bit index size %offset is either 32767 or -32768. Thus, when
+; aggregating the possible offsets it we know that it is in the range [-32768,
+; 32767], which includes valid offsets that aren't out-of-bounds. This is
+; similar to the out_of_bounds_gep_i16_pos_neg test above, and the current
+; implementation is expected to derive the result -1 for %objsize_max.
+define i32 @out_of_bounds_gep_i32_trunc_select(i1 %c0, i1 %c1) {
+; CHECK-LABEL: define i32 @out_of_bounds_gep_i32_trunc_select(
+; CHECK-SAME: i1 [[C0:%.*]], i1 [[C1:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[OBJ:%.*]] = alloca [5 x i8], align 1
+; CHECK-NEXT:    [[OFFSET:%.*]] = select i1 [[C0]], i32 32767, i32 32768
+; CHECK-NEXT:    [[PTR_SLIDE:%.*]] = getelementptr i8, ptr [[OBJ]], i32 [[OFFSET]]
+; CHECK-NEXT:    ret i32 0
+;
+entry:
+  %obj = alloca [5 x i8]
+  %offset = select i1 %c0, i32 32767, i32 32768
+  %ptr.slide = getelementptr i8, ptr %obj, i32 %offset
+  %objsize_max = call i32 @llvm.objectsize.i32.p0(ptr %ptr.slide, i1 false, i1 true, i1 false)
+  %objsize_min = call i32 @llvm.objectsize.i32.p0(ptr %ptr.slide, i1 true, i1 true, i1 false)
+  %res = select i1 %c1, i32 %objsize_max, i32 %objsize_min
+  ret i32 %res
+}
+
+; FIXME: Is 3 really correct for %objsize_min here?
+define i32 @possible_out_of_bounds_gep_i8_neg(i1 %c0, i1 %c1) {
+; CHECK-LABEL: define i32 @possible_out_of_bounds_gep_i8_neg(
+; CHECK-SAME: i1 [[C0:%.*]], i1 [[C1:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[OBJ:%.*]] = alloca [5 x i8], align 1
+; CHECK-NEXT:    [[OFFSET:%.*]] = select i1 [[C0]], i8 2, i8 -10
+; CHECK-NEXT:    [[PTR_SLIDE:%.*]] = getelementptr i8, ptr [[OBJ]], i8 [[OFFSET]]
+; CHECK-NEXT:    [[RES:%.*]] = select i1 [[C1]], i32 -1, i32 3
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+entry:
+  %obj = alloca [5 x i8]
+  %offset = select i1 %c0, i8 2, i8 -10
+  %ptr.slide = getelementptr i8, ptr %obj, i8 %offset
+  %objsize_max = call i32 @llvm.objectsize.i32.p0(ptr %ptr.slide, i1 false, i1 true, i1 false)
+  %objsize_min = call i32 @llvm.objectsize.i32.p0(ptr %ptr.slide, i1 true, i1 true, i1 false)
+  %res = select i1 %c1, i32 %objsize_max, i32 %objsize_min
+  ret i32 %res
+}
+
+; FIXME: Is 3 really correct for %objsize_min here?
+define i32 @possible_out_of_bounds_gep_i16_neg(i1 %c0, i1 %c1) {
+; CHECK-LABEL: define i32 @possible_out_of_bounds_gep_i16_neg(
+; CHECK-SAME: i1 [[C0:%.*]], i1 [[C1:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[OBJ:%.*]] = alloca [5 x i8], align 1
+; CHECK-NEXT:    [[OFFSET:%.*]] = select i1 [[C0]], i16 2, i16 -10
+; CHECK-NEXT:    [[PTR_SLIDE:%.*]] = getelementptr i8, ptr [[OBJ]], i16 [[OFFSET]]
+; CHECK-NEXT:    [[RES:%.*]] = select i1 [[C1]], i32 -1, i32 3
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+entry:
+  %obj = alloca [5 x i8]
+  %offset = select i1 %c0, i16 2, i16 -10
+  %ptr.slide = getelementptr i8, ptr %obj, i16 %offset
+  %objsize_max = call i32 @llvm.objectsize.i32.p0(ptr %ptr.slide, i1 false, i1 true, i1 false)
+  %objsize_min = call i32 @llvm.objectsize.i32.p0(ptr %ptr.slide, i1 true, i1 true, i1 false)
+  %res = select i1 %c1, i32 %objsize_max, i32 %objsize_min
+  ret i32 %res
+}
diff --git a/llvm/test/Transforms/LowerConstantIntrinsics/builtin-object-size-phi.ll b/llvm/test/Transforms/LowerConstantIntrinsics/builtin-object-size-phi.ll
index 564311da64a81..d294650d7f3e2 100644
--- a/llvm/test/Transforms/LowerConstantIntrinsics/builtin-object-size-phi.ll
+++ b/llvm/test/Transforms/LowerConstantIntrinsics/builtin-object-size-phi.ll
@@ -200,8 +200,12 @@ if.end:
   ret i64 %size
 }
 
-define i64 @pick_negative_offset_different_width(i32 %n) {
-; CHECK-LABEL: @pick_negative_offset_different_width(
+; FIXME: The result here looks weird. Either we reference into buffer0 with an
+;        oob offset. Or we reference buffer1 (8 bytes) with a 4 byte
+;        offset. The result 5 is wrong in both cases. Probably better to
+;        return -1 here since we do not know if we have an oob pointer.
+define i64 @pick_negative_offset_different_width_index_maybe_too_small(i32 %n, i1 %c) {
+; CHECK-LABEL: @pick_negative_offset_different_width_index_maybe_too_small(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[BUFFER0:%.*]] = alloca i8, i64 4, align 1
 ; CHECK-NEXT:    [[BUFFER1:%.*]] = alloca i8, i64 8, align 1
@@ -216,7 +220,8 @@ define i64 @pick_negative_offset_different_width(i32 %n) {
 ; CHECK:       if.end:
 ; CHECK-NEXT:    [[P:%.*]] = phi ptr [ [[OFFSETED0]], [[IF_ELSE]] ], [ [[OFFSETED1]], [[IF_END]] ]
 ; CHECK-NEXT:    [[POFFSETED:%.*]] = getelementptr i8, ptr [[P]], i64 -2
-; CHECK-NEXT:    ret i64 5
+; CHECK-NEXT:    [[SIZE:%.*]] = select i1 [[C:%.*]], i64 5, i64 0
+; CHECK-NEXT:    ret i64 [[SIZE]]
 ;
 entry:
   %buffer0 = alloca i8, i64 4
@@ -235,7 +240,51 @@ if.else:
 if.end:
   %p = phi ptr [ %offseted0, %if.then ], [ %offseted1, %if.else ]
   %poffseted = getelementptr i8, ptr %p, i64 -2
-  %size = call i64 @llvm.objectsize.i64.p0(ptr %poffseted, i1 false, i1 false, i1 false)
+  %sizemax = call i64 @llvm.objectsize.i64.p0(ptr %poffseted, i1 false, i1 false, i1 false)
+  %sizemin = call i64 @llvm.objectsize.i64.p0(ptr %poffseted, i1 true, i1 false, i1 false)
+  %size = select i1 %c, i64 %sizemax, i64 %sizemin
+  ret i64 %size
+}
+
+define i64 @pick_negative_offset_different_width_index_maybe_too_large(i32 %n, i1 %c) {
+; CHECK-LABEL: @pick_negative_offset_different_width_index_maybe_too_large(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[BUFFER0:%.*]] = alloca i8, i64 4, align 1
+; CHECK-NEXT:    [[BUFFER1:%.*]] = alloca i8, i64 8, align 1
+; CHECK-NEXT:    [[COND:%.*]] = icmp eq i32 [[N:%.*]], 0
+; CHECK-NEXT:    br i1 [[COND]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    [[OFFSETED0:%.*]] = getelementptr i8, ptr [[BUFFER0]], i64 1
+; CHECK-NEXT:    br label [[IF_END:%.*]]
+; CHECK:       if.else:
+; CHECK-NEXT:    [[OFFSETED1:%.*]] = getelementptr i8, ptr [[BUFFER1]], i64 6
+; CHECK-NEXT:    br label [[IF_END]]
+; CHECK:       if.end:
+; CHECK-NEXT:    [[P:%.*]] = phi ptr [ [[OFFSETED0]], [[IF_THEN]] ], [ [[OFFSETED1]], [[IF_ELSE]] ]
+; CHECK-NEXT:    [[POFFSETED:%.*]] = getelementptr i8, ptr [[P]], i64 2
+; CHECK-NEXT:    [[SIZE:%.*]] = select i1 [[C:%.*]], i64 1, i64 0
+; CHECK-NEXT:    ret i64 [[SIZE]]
+;
+entry:
+  %buffer0 = alloca i8, i64 4
+  %buffer1 = alloca i8, i64 8
+  %cond = icmp eq i32 %n, 0
+  br i1 %cond, label %if.then, label %if.else
+
+if.then:
+  %offseted0 = getelementptr i8, ptr %buffer0, i64 1
+  br label %if.end
+
+if.else:
+  %offseted1 = getelementptr i8, ptr %buffer1, i64 6
+  br label %if.end
+
+if.end:
+  %p = phi ptr [ %offseted0, %if.then ], [ %offseted1, %if.else ]
+  %poffseted = getelementptr i8, ptr %p, i64 2
+  %sizemax = call i64 @llvm.objectsize.i64.p0(ptr %poffseted, i1 false, i1 false, i1 false)
+  %sizemin = call i64 @llvm.objectsize.i64.p0(ptr %poffseted, i1 true, i1 false, i1 false)
+  %size = select i1 %c, i64 %sizemax, i64 %sizemin
   ret i64 %size
 }
 
diff --git a/llvm/test/Transforms/LowerTypeTests/function.ll b/llvm/test/Transforms/LowerTypeTests/function.ll
index ab3cfb6acccf8..fa7c7bbcdabd3 100644
--- a/llvm/test/Transforms/LowerTypeTests/function.ll
+++ b/llvm/test/Transforms/LowerTypeTests/function.ll
@@ -1,7 +1,7 @@
-; RUN: opt -S -passes=lowertypetests -mtriple=i686-unknown-linux-gnu %s | FileCheck --check-prefixes=X86,X86-LINUX,NATIVE,JT8 %s
-; RUN: opt -S -passes=lowertypetests -mtriple=x86_64-unknown-linux-gnu %s | FileCheck --check-prefixes=X86,X86-LINUX,NATIVE,JT8 %s
-; RUN: opt -S -passes=lowertypetests -mtriple=i686-pc-win32 %s | FileCheck --check-prefixes=X86,X86-WIN32,NATIVE,JT8 %s
-; RUN: opt -S -passes=lowertypetests -mtriple=x86_64-pc-win32 %s | FileCheck --check-prefixes=X86,X86-WIN32,NATIVE,JT8 %s
+; RUN: opt -S -passes=lowertypetests -mtriple=i686-unknown-linux-gnu %s | FileCheck --check-prefixes=X86,NATIVE,JT8 %s
+; RUN: opt -S -passes=lowertypetests -mtriple=x86_64-unknown-linux-gnu %s | FileCheck --check-prefixes=X86,NATIVE,JT8 %s
+; RUN: opt -S -passes=lowertypetests -mtriple=i686-pc-win32 %s | FileCheck --check-prefixes=X86,NATIVE,JT8 %s
+; RUN: opt -S -passes=lowertypetests -mtriple=x86_64-pc-win32 %s | FileCheck --check-prefixes=X86,NATIVE,JT8 %s
 ; RUN: opt -S -passes=lowertypetests -mtriple=riscv32-unknown-linux-gnu %s | FileCheck --check-prefixes=RISCV,NATIVE,JT8 %s
 ; RUN: opt -S -passes=lowertypetests -mtriple=riscv64-unknown-linux-gnu %s | FileCheck --check-prefixes=RISCV,NATIVE,JT8 %s
 ; RUN: opt -S -passes=lowertypetests -mtriple=wasm32-unknown-unknown %s | FileCheck --check-prefix=WASM32 %s
@@ -114,8 +114,7 @@ define i1 @foo(ptr %p) {
 
 ; NATIVE-SAME: "s"(ptr @g.cfi)
 
-; X86-LINUX: attributes #[[ATTR]] = { naked nocf_check noinline }
-; X86-WIN32: attributes #[[ATTR]] = { nocf_check noinline }
+; X86: attributes #[[ATTR]] = { naked nocf_check noinline }
 ; ARM: attributes #[[ATTR]] = { naked noinline
 ; THUMB: attributes #[[ATTR]] = { naked noinline "target-cpu"="cortex-a8" "target-features"="+thumb-mode" }
 ; THUMBV6M: attributes #[[ATTR]] = { naked noinline "target-features"="+thumb-mode" }
diff --git a/llvm/test/tools/llvm-exegesis/RISCV/rvv/filter.test b/llvm/test/tools/llvm-exegesis/RISCV/rvv/filter.test
index 858569e4b0ef5..3815e281876e2 100644
--- a/llvm/test/tools/llvm-exegesis/RISCV/rvv/filter.test
+++ b/llvm/test/tools/llvm-exegesis/RISCV/rvv/filter.test
@@ -1,7 +1,9 @@
+# TODO(mshockwave): We use a fixed seed for this test because sometimes it
+# will fail to generate any snippet because it is unable to assign unique
+# def and use registers.
 # RUN: llvm-exegesis -mtriple=riscv64 -mcpu=sifive-x280 -benchmark-phase=assemble-measured-code --mode=inverse_throughput --opcode-name=PseudoVNCLIPU_WX_M1_MASK \
-# RUN:    --riscv-filter-config='vtype = {VXRM: rod, AVL: VLMAX, SEW: e(8|16), Policy: ta/mu}' --max-configs-per-opcode=1000 --min-instructions=10 | FileCheck %s
-# Sometimes it'll fail to generate any snippet because it's unable to assign unique def and use registers.
-# ALLOW_RETRIES: 2
+# RUN:    --riscv-filter-config='vtype = {VXRM: rod, AVL: VLMAX, SEW: e(8|16), Policy: ta/mu}' --max-configs-per-opcode=1000 --min-instructions=10 \
+# RUN:    -random-generator-seed=5 | FileCheck %s
 
 # CHECK: config:          'vtype = {VXRM: rod, AVL: VLMAX, SEW: e8, Policy: ta/mu}'
 # CHECK: config:          'vtype = {VXRM: rod, AVL: VLMAX, SEW: e16, Policy: ta/mu}'
diff --git a/llvm/unittests/ADT/SetVectorTest.cpp b/llvm/unittests/ADT/SetVectorTest.cpp
index ff3c876deb458..6230472553c38 100644
--- a/llvm/unittests/ADT/SetVectorTest.cpp
+++ b/llvm/unittests/ADT/SetVectorTest.cpp
@@ -52,7 +52,7 @@ TEST(SetVector, ContainsTest) {
 }
 
 TEST(SetVector, ConstPtrKeyTest) {
-  SetVector<int *, SmallVector<int *, 8>, SmallPtrSet<const int *, 8>> S, T;
+  SetVector<int *> S, T;
   int i, j, k, m, n;
 
   S.insert(&i);
diff --git a/llvm/unittests/CodeGen/GlobalISel/InstructionSelectTest.cpp b/llvm/unittests/CodeGen/GlobalISel/InstructionSelectTest.cpp
index 7fbccf7160e17..223798342b3ee 100644
--- a/llvm/unittests/CodeGen/GlobalISel/InstructionSelectTest.cpp
+++ b/llvm/unittests/CodeGen/GlobalISel/InstructionSelectTest.cpp
@@ -59,10 +59,8 @@ TEST_F(AArch64GISelMITest, TestInstructionSelectErase) {
     GTEST_SKIP();
 
   legacy::PassManager PM;
-  std::unique_ptr<TargetPassConfig> TPC(TM->createPassConfig(PM));
 
   EraseMockInstructionSelector ISel;
-  ISel.TPC = TPC.get();
   for (auto &MI : *EntryMBB) {
     ISel.MIs.push_back(&MI);
   }
diff --git a/llvm/utils/gn/secondary/lldb/source/Target/BUILD.gn b/llvm/utils/gn/secondary/lldb/source/Target/BUILD.gn
index 679373d741661..ac63bbc6ee3b3 100644
--- a/llvm/utils/gn/secondary/lldb/source/Target/BUILD.gn
+++ b/llvm/utils/gn/secondary/lldb/source/Target/BUILD.gn
@@ -35,6 +35,7 @@ static_library("Target") {
   sources = [
     "ABI.cpp",
     "AssertFrameRecognizer.cpp",
+    "BorrowedStackFrame.cpp",
     "CoreFileMemoryRanges.cpp",
     "DynamicRegisterInfo.cpp",
     "ExecutionContext.cpp",
diff --git a/llvm/utils/profcheck-xfail.txt b/llvm/utils/profcheck-xfail.txt
index 835025d1e319e..980f99687c4cc 100644
--- a/llvm/utils/profcheck-xfail.txt
+++ b/llvm/utils/profcheck-xfail.txt
@@ -17,7 +17,6 @@ CodeGen/RISCV/zmmul.ll
 CodeGen/WebAssembly/memory-interleave.ll
 CodeGen/X86/AMX/amx-low-intrinsics.ll
 CodeGen/X86/masked_gather_scatter.ll
-CodeGen/X86/nocfivalue.ll
 DebugInfo/AArch64/ir-outliner.ll
 DebugInfo/assignment-tracking/X86/hotcoldsplit.ll
 DebugInfo/Generic/block-asan.ll
@@ -148,9 +147,6 @@ Transforms/ExpandLargeFpConvert/X86/expand-large-fp-convert-ui129tofp.ll
 Transforms/ExpandMemCmp/AArch64/memcmp.ll
 Transforms/ExpandMemCmp/X86/memcmp.ll
 Transforms/ExpandMemCmp/X86/memcmp-x32.ll
-Transforms/ExpandVariadics/expand-va-intrinsic-split-linkage.ll
-Transforms/ExpandVariadics/expand-va-intrinsic-split-simple.ll
-Transforms/ExpandVariadics/intrinsics.ll
 Transforms/FixIrreducible/basic.ll
 Transforms/FixIrreducible/bug45623.ll
 Transforms/FixIrreducible/callbr.ll
@@ -472,9 +468,6 @@ Transforms/LoopDeletion/invalidate-scev-after-hoisting.ll
 Transforms/LoopIdiom/AArch64/byte-compare-index.ll
 Transforms/LoopIdiom/AArch64/find-first-byte.ll
 Transforms/LoopIdiom/RISCV/byte-compare-index.ll
-Transforms/LoopUnroll/peel-last-iteration-expansion-cost.ll
-Transforms/LoopUnroll/peel-last-iteration-with-guards.ll
-Transforms/LoopUnroll/peel-last-iteration-with-variable-trip-count.ll
 Transforms/LowerAtomic/atomic-load.ll
 Transforms/LowerAtomic/atomic-swap.ll
 Transforms/LowerConstantIntrinsics/builtin-object-size-phi.ll
@@ -505,41 +498,15 @@ Transforms/LowerSwitch/do-not-handle-impossible-values.ll
 Transforms/LowerSwitch/feature.ll
 Transforms/LowerSwitch/fold-popular-case-to-unreachable-default.ll
 Transforms/LowerSwitch/pr59316.ll
-Transforms/LowerTypeTests/aarch64-jumptable.ll
-Transforms/LowerTypeTests/blockaddress-2.ll
-Transforms/LowerTypeTests/blockaddress.ll
-Transforms/LowerTypeTests/cfi-annotation.ll
 Transforms/LowerTypeTests/cfi-coff-comdat-rename.ll
-Transforms/LowerTypeTests/cfi-direct-call1.ll
-Transforms/LowerTypeTests/cfi-icall-alias.ll
-Transforms/LowerTypeTests/cfi-nounwind-direct-call.ll
-Transforms/LowerTypeTests/cfi-unwind-direct-call.ll
-Transforms/LowerTypeTests/export-alias.ll
-Transforms/LowerTypeTests/export-cross-dso-cfi.ll
-Transforms/LowerTypeTests/export-icall.ll
-Transforms/LowerTypeTests/export-rename-local.ll
-Transforms/LowerTypeTests/export-symver.ll
-Transforms/LowerTypeTests/function-arm-thumb.ll
-Transforms/LowerTypeTests/function-disjoint.ll
-Transforms/LowerTypeTests/function-ext.ll
 Transforms/LowerTypeTests/function.ll
-Transforms/LowerTypeTests/function-thumb-bti.ll
 Transforms/LowerTypeTests/function-weak.ll
-Transforms/LowerTypeTests/icall-branch-funnel.ll
 Transforms/LowerTypeTests/import.ll
-Transforms/LowerTypeTests/nocfivalue.ll
-Transforms/LowerTypeTests/pr37625.ll
-Transforms/LowerTypeTests/section.ll
 Transforms/LowerTypeTests/simple.ll
-Transforms/LowerTypeTests/x86-jumptable.ll
-Transforms/MemCpyOpt/memset-memcpy-dbgloc.ll
-Transforms/MemCpyOpt/memset-memcpy-redundant-memset.ll
-Transforms/MemCpyOpt/opaque-ptr.ll
 Transforms/MergeFunc/2011-02-08-RemoveEqual.ll
 Transforms/MergeFunc/apply_function_attributes.ll
 Transforms/MergeFunc/call-and-invoke-with-ranges-attr.ll
 Transforms/MergeFunc/call-and-invoke-with-ranges.ll
-Transforms/MergeFunc/cfi-thunk-merging.ll
 Transforms/MergeFunc/comdat.ll
 Transforms/MergeFunc/crash-cast-arrays.ll
 Transforms/MergeFunc/crash.ll
@@ -572,10 +539,6 @@ Transforms/MergeFunc/ranges-multiple.ll
 Transforms/MergeFunc/self-referential-global.ll
 Transforms/MergeFunc/unnamed-addr-reprocessing.ll
 Transforms/MergeFunc/vector-GEP-crash.ll
-Transforms/MergeICmps/X86/alias-merge-blocks.ll
-Transforms/MergeICmps/X86/entry-block-shuffled-2.ll
-Transforms/MergeICmps/X86/entry-block-shuffled.ll
-Transforms/MergeICmps/X86/pr59740.ll
 Transforms/OpenMP/always_inline_device.ll
 Transforms/OpenMP/custom_state_machines.ll
 Transforms/OpenMP/custom_state_machines_remarks.ll
diff --git a/mlir/include/mlir/Dialect/AMDGPU/IR/AMDGPU.td b/mlir/include/mlir/Dialect/AMDGPU/IR/AMDGPU.td
index 3581b07dc4e3e..16eaf28ddd95b 100644
--- a/mlir/include/mlir/Dialect/AMDGPU/IR/AMDGPU.td
+++ b/mlir/include/mlir/Dialect/AMDGPU/IR/AMDGPU.td
@@ -1229,15 +1229,13 @@ def AMDGPU_ScaledMFMAOp :
 
 def AMDGPU_MakeDmaBaseOp :
     AMDGPU_Op<"make_dma_base", [Pure, AttrSizedOperandSegments]>,
-    Arguments<(ins
-                   Arg<AnyMemRef, "buffer to read from">:$src,
-                   Variadic<Index>:$src_indices,
-                   Arg<AnyMemRef, "buffer to write to">:$dst,
-                   Variadic<Index>:$dst_indices)>,
+    Arguments<(ins Arg<AnyMemRef>:$global,
+                   Variadic<Index>:$global_indices,
+                   Arg<AnyMemRef>:$lds,
+                   Variadic<Index>:$lds_indices)>,
     Results<(outs AMDGPU_TDMBaseType: $base)> {
 
   // TODO:
-  // * Add verifiers such that one of the memrefs is from LDS and the other global.
   // * Add verifiers to make sure that the number of indices do not exceed the number of dimensions.
 
   let summary = "Pair of based addresses used when moving tiles between LDS and global memory.";
@@ -1251,7 +1249,7 @@ def AMDGPU_MakeDmaBaseOp :
     For example:
 
     ```mlir
-      %base = amdgpu.make_dma_base %src[%idx0], %dst[%idx1] : memref<8xi32>, memref<8xi32, #gpu.address_space<workgroup>> -> !amdgpu.tdm_base<i32>
+      %base = amdgpu.make_dma_base %global[%idx0, %idx1], %lds[%idx2, %idx3] : memref<64x64xi32>, memref<64x64xi32, #gpu.address_space<workgroup>> -> !amdgpu.tdm_base<i32>
       %descriptor = amdgpu.make_dma_descriptor %base globalSize [2, 2] globalStride [2, 1] sharedSize [2, 2] : !amdgpu.tdm_base<i32> -> !amdgpu.tdm_descriptor
       amdgpu.tensor_load_to_lds %descriptor : !amdgpu.tdm_descriptor
     ```
@@ -1259,27 +1257,31 @@ def AMDGPU_MakeDmaBaseOp :
     to
 
     ```mlir
-       // pseudocode
-       %base_0 = llvm.mlir.undef : !llvm.struct<(ptr, ptr)>
-       %base_1 = llvm.insertvalue %global_addr, %base_0[0] : !llvm.struct<(ptr, ptr)>
-       %base_2 = llvm.insertvalue %lds_addr, %base_1[1] : !llvm.struct(ptr, ptr)>
-       // type(%base_2) = !llvm.struct<(ptr, ptr) roughly corresponds to amdgpu.tdm_base<i32>
-
-       // The base will be used when contructing dgroup0
-       // when lowering amdgpu.make_dma_descriptor
-       %dgroup0_0 = llvm.mlir.undef : !llvm.struct<(....)>
-       %dgroup0_1 = llvm.insertvalue %base2, %dgroup0_0 : ....
-
-       // When lowering amdgpu.tensor_load_to_lds
-       rocdl.tensor.load.to.lds %dgroup0, %dgroup1, %dgroup2, %dgroup3 cachepolicy 0 : vector<4xi32>, vector<8xi32>
+      // pseudo-code
+      %global_base = llvm.extractvalue %global_memref[1]
+      %global_address = llvm.get_element_ptr ...
+
+      %lds_base = llvm.extractvalue %lds_memref[1]
+      %lds_address = llvm.get_element_ptr ...
+
+      // Definition of %base
+      %undef = llvm.mlir.undef : vector<4xi32>
+      %v0 = llvm.insertelement %15, %undef[0] : vector<4xi32>
+      %v1 = llvm.insertelement %lds_address, %v0[1] : vector<4xi32>
+      %v2 = llvm.insertelement %global_address_low, %v1[2] : vector<4xi32>
+      %base = llvm.insertelement %global_address_high, %v2[3] : vector<4xi32>
+
+      rocdl.tensor.load.to.lds %base, %dgroup1, %dgroup2, %dgroup3 cachepolicy 0 : vector<4xi32>, vector<8xi32>
     ```
 
     These tensor DMA operations were introduced in gfx1250.
   }];
 
   let assemblyFormat = [{
-    $src `[` $src_indices `]` `,` $dst `[` $dst_indices `]` attr-dict `:` type($src) `,` type($dst) `->` type(results)
+    $global `[` $global_indices `]` `,` $lds `[` $lds_indices `]` attr-dict `:` type($global) `,` type($lds) `->` type(results)
   }];
+
+  let hasVerifier = 1;
 }
 
 def AMDGPU_MakeDmaDescriptorOp :
@@ -1323,12 +1325,12 @@ def AMDGPU_MakeDmaDescriptorOp :
 
      ```mlir
       // Example of moving a two-dimensional tensor to LDS.
-      %base = amdgpu.make_dma_base %src[0, 0], %dst[0, 0] : memref<64x64xi32>, memref<64x64xi32, #gpu.address_space<workgroup>> -> !amdgpu.tdm_base<i32>
+      %base = amdgpu.make_dma_base %global[0, 0], %lds[0, 0] : memref<64x64xi32>, memref<64x64xi32, #gpu.address_space<workgroup>> -> !amdgpu.tdm_base<i32>
       %descriptor = amdgpu.make_dma_descriptor %base globalSize [64, 64] globalStride [64, 1] sharedSize [64, 64] : !amdgpu.tdm_base<i32> -> !amdgpu.tdm_descriptor
       amdgpu.tensor_load_to_lds %descriptor : !amdgpu.tdm_descriptor
 
       // Example of moving a two dimension tensor to LDS where padding is applied after every integer.
-      %base = amdgpu.make_dma_base %src[0, 0], %dst[0, 0] : memref<32x32xi32>, memref<64x64xi32, #gpu.address_space<workgroup>> -> !amdgpu.tdm_base<i32>
+      %base = amdgpu.make_dma_base %global[0, 0], %lds[0, 0] : memref<32x32xi32>, memref<64x64xi32, #gpu.address_space<workgroup>> -> !amdgpu.tdm_base<i32>
       %descriptor = amdgpu.make_dma_descriptor %base globalSize [32, 32] globalStride [32, 1] sharedSize [64, 64] padding(%pad pad_every %pad_every) : !amdgpu.tdm_base<i32> -> !amdgpu.tdm_descriptor
       amdgpu.tensor_load_to_lds %descriptor : !amdgpu.tdm_descriptor
      ```
diff --git a/mlir/include/mlir/Dialect/OpenACC/OpenACCOps.td b/mlir/include/mlir/Dialect/OpenACC/OpenACCOps.td
index b8317b4a1d2ec..77d1a6f8d53b5 100644
--- a/mlir/include/mlir/Dialect/OpenACC/OpenACCOps.td
+++ b/mlir/include/mlir/Dialect/OpenACC/OpenACCOps.td
@@ -3232,6 +3232,18 @@ def OpenACC_RoutineOp : OpenACC_Op<"routine", [IsolatedFromAbove]> {
       OptionalAttr<DeviceTypeArrayAttr>:$gangDimDeviceType);
 
   let extraClassDeclaration = [{
+    // 'create' function to generate an 'empty' routine.
+    static RoutineOp create(::mlir::OpBuilder & builder,
+                            ::mlir::Location location,
+                            ::llvm::StringRef sym_name,
+                            mlir::SymbolRefAttr func_name, bool implicit) {
+      return create(builder, location, sym_name, func_name, /*bindIDName=*/{},
+                    /*bindStrName=*/{}, /*bindIdNameDeviceType=*/{},
+                    /*bindStrnameDeviceType=*/{}, /*worker=*/{}, /*vector=*/{},
+                    /*seq=*/{}, /*nohost=*/false, implicit, /*gang=*/{},
+                    /*gangDim=*/{}, /*gangDimDeviceType=*/{});
+    }
+
     static StringRef getGangDimKeyword() { return "dim"; }
 
     /// Return true if the op has the worker attribute for the
@@ -3267,6 +3279,13 @@ def OpenACC_RoutineOp : OpenACC_Op<"routine", [IsolatedFromAbove]> {
 
     std::optional<::std::variant<mlir::SymbolRefAttr, mlir::StringAttr>> getBindNameValue();
     std::optional<::std::variant<mlir::SymbolRefAttr, mlir::StringAttr>> getBindNameValue(mlir::acc::DeviceType deviceType);
+
+    // Add an entry to the 'seq' attribute for each additional device types.
+    void addSeq(MLIRContext *, llvm::ArrayRef<DeviceType>);
+    // Add an entry to the 'vector' attribute for each additional device types.
+    void addVector(MLIRContext *, llvm::ArrayRef<DeviceType>);
+    // Add an entry to the 'worker' attribute for each additional device types.
+    void addWorker(MLIRContext *, llvm::ArrayRef<DeviceType>);
   }];
 
   let assemblyFormat = [{
diff --git a/mlir/include/mlir/Dialect/OpenACC/OpenACCTypeInterfaces.td b/mlir/include/mlir/Dialect/OpenACC/OpenACCTypeInterfaces.td
index d1bbc7f206ce6..3f11bf6fbfce3 100644
--- a/mlir/include/mlir/Dialect/OpenACC/OpenACCTypeInterfaces.td
+++ b/mlir/include/mlir/Dialect/OpenACC/OpenACCTypeInterfaces.td
@@ -176,6 +176,50 @@ def OpenACC_PointerLikeTypeInterface : TypeInterface<"PointerLikeType"> {
         return false;
       }]
     >,
+    InterfaceMethod<
+      /*description=*/[{
+        Generates a load operation from the pointer-like type. This dereferences
+        the pointer and returns the loaded value.
+
+        The `srcPtr` parameter is the pointer to load from. If the current type is
+        represented in a way that it does not capture the pointee type, `valueType`
+        must be passed in to provide the necessary type information.
+
+        Returns the loaded value, or an empty Value if load generation failed.
+      }],
+      /*retTy=*/"::mlir::Value",
+      /*methodName=*/"genLoad",
+      /*args=*/(ins "::mlir::OpBuilder &":$builder,
+                    "::mlir::Location":$loc,
+                    "::mlir::TypedValue<::mlir::acc::PointerLikeType>":$srcPtr,
+                    "::mlir::Type":$valueType),
+      /*methodBody=*/"",
+      /*defaultImplementation=*/[{
+        return {};
+      }]
+    >,
+    InterfaceMethod<
+      /*description=*/[{
+        Generates a store operation to the pointer-like type. This stores a value
+        to the memory location pointed to by the pointer.
+
+        The `destPtr` parameter is the pointer to store to. The `valueToStore`
+        parameter is the value to be stored. The type information is derived from
+        the valueToStore parameter itself.
+
+        Returns true if store was successfully generated, false otherwise.
+      }],
+      /*retTy=*/"bool",
+      /*methodName=*/"genStore",
+      /*args=*/(ins "::mlir::OpBuilder &":$builder,
+                    "::mlir::Location":$loc,
+                    "::mlir::Value":$valueToStore,
+                    "::mlir::TypedValue<::mlir::acc::PointerLikeType>":$destPtr),
+      /*methodBody=*/"",
+      /*defaultImplementation=*/[{
+        return false;
+      }]
+    >,
   ];
 }
 
diff --git a/mlir/lib/Conversion/AMDGPUToROCDL/AMDGPUToROCDL.cpp b/mlir/lib/Conversion/AMDGPUToROCDL/AMDGPUToROCDL.cpp
index b9a5e7d7f6eac..2b6938712dad2 100644
--- a/mlir/lib/Conversion/AMDGPUToROCDL/AMDGPUToROCDL.cpp
+++ b/mlir/lib/Conversion/AMDGPUToROCDL/AMDGPUToROCDL.cpp
@@ -2264,6 +2264,77 @@ struct AMDGPUPermlaneLowering : public ConvertOpToLLVMPattern<PermlaneSwapOp> {
   }
 };
 
+struct AMDGPUMakeDmaBaseLowering
+    : public ConvertOpToLLVMPattern<MakeDmaBaseOp> {
+  using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern;
+
+  AMDGPUMakeDmaBaseLowering(const LLVMTypeConverter &converter, Chipset chipset)
+      : ConvertOpToLLVMPattern<MakeDmaBaseOp>(converter), chipset(chipset) {}
+  Chipset chipset;
+
+  LogicalResult
+  matchAndRewrite(MakeDmaBaseOp op, OpAdaptor adaptor,
+                  ConversionPatternRewriter &rewriter) const override {
+    if (chipset < kGfx1250)
+      return op->emitOpError("make_dma_base is only supported on gfx1250");
+
+    Location loc = op.getLoc();
+
+    ValueRange ldsIndices = adaptor.getLdsIndices();
+    Value lds = adaptor.getLds();
+    auto ldsMemRefType = cast<MemRefType>(op.getLds().getType());
+
+    Value ldsPtr =
+        getStridedElementPtr(rewriter, loc, ldsMemRefType, lds, ldsIndices);
+
+    ValueRange globalIndices = adaptor.getGlobalIndices();
+    Value global = adaptor.getGlobal();
+    auto globalMemRefType = cast<MemRefType>(op.getGlobal().getType());
+
+    Value globalPtr = getStridedElementPtr(rewriter, loc, globalMemRefType,
+                                           global, globalIndices);
+
+    Type i32 = rewriter.getI32Type();
+    Type i64 = rewriter.getI64Type();
+
+    Value castForLdsAddr = LLVM::PtrToIntOp::create(rewriter, loc, i32, ldsPtr);
+    Value castForGlobalAddr =
+        LLVM::PtrToIntOp::create(rewriter, loc, i64, globalPtr);
+
+    Value lowHalf =
+        LLVM::TruncOp::create(rewriter, loc, i32, castForGlobalAddr);
+
+    Value shift = LLVM::LShrOp::create(rewriter, loc, castForGlobalAddr,
+                                       createI64Constant(rewriter, loc, 32));
+
+    Value highHalf = LLVM::TruncOp::create(rewriter, loc, i32, shift);
+
+    Value mask = createI32Constant(rewriter, loc, (1ull << 25) - 1);
+    Value validHighHalf = LLVM::AndOp::create(rewriter, loc, highHalf, mask);
+
+    Value typeField = createI32Constant(rewriter, loc, 2 << 30);
+    Value highHalfPlusType =
+        LLVM::OrOp::create(rewriter, loc, validHighHalf, typeField);
+
+    Value c0 = createI32Constant(rewriter, loc, 0);
+    Value c1 = createI32Constant(rewriter, loc, 1);
+    Value c2 = createI32Constant(rewriter, loc, 2);
+    Value c3 = createI32Constant(rewriter, loc, 3);
+
+    Type v4i32 = this->typeConverter->convertType(VectorType::get(4, i32));
+    Value result = LLVM::PoisonOp::create(rewriter, loc, v4i32);
+    result = LLVM::InsertElementOp::create(rewriter, loc, result, c1, c0);
+    result = LLVM::InsertElementOp::create(rewriter, loc, result,
+                                           castForLdsAddr, c1);
+    result = LLVM::InsertElementOp::create(rewriter, loc, result, lowHalf, c2);
+    result = LLVM::InsertElementOp::create(rewriter, loc, result,
+                                           highHalfPlusType, c3);
+
+    rewriter.replaceOp(op, result);
+    return success();
+  }
+};
+
 struct ConvertAMDGPUToROCDLPass
     : public impl::ConvertAMDGPUToROCDLPassBase<ConvertAMDGPUToROCDLPass> {
   using Base::Base;
@@ -2278,6 +2349,10 @@ struct ConvertAMDGPUToROCDLPass
 
     RewritePatternSet patterns(ctx);
     LLVMTypeConverter converter(ctx);
+    converter.addConversion([&](TDMBaseType type) -> Type {
+      Type i32 = IntegerType::get(type.getContext(), 32);
+      return converter.convertType(VectorType::get(4, i32));
+    });
     populateAMDGPUToROCDLConversionPatterns(converter, patterns, *maybeChipset);
     LLVMConversionTarget target(getContext());
     target.addIllegalDialect<::mlir::amdgpu::AMDGPUDialect>();
@@ -2333,6 +2408,7 @@ void mlir::populateAMDGPUToROCDLConversionPatterns(LLVMTypeConverter &converter,
            ScaledExtPackedOpLowering, PackedScaledTruncOpLowering,
            PackedTrunc2xFp8OpLowering, PackedStochRoundFp8OpLowering,
            GatherToLDSOpLowering, TransposeLoadOpLowering,
-           AMDGPUPermlaneLowering>(converter, chipset);
+           AMDGPUPermlaneLowering, AMDGPUMakeDmaBaseLowering>(converter,
+                                                              chipset);
   patterns.add<AMDGPUSwizzleBitModeLowering>(converter);
 }
diff --git a/mlir/lib/Dialect/AMDGPU/IR/AMDGPUDialect.cpp b/mlir/lib/Dialect/AMDGPU/IR/AMDGPUDialect.cpp
index 93cb9b38a5ecf..8b58c3b1dd182 100644
--- a/mlir/lib/Dialect/AMDGPU/IR/AMDGPUDialect.cpp
+++ b/mlir/lib/Dialect/AMDGPU/IR/AMDGPUDialect.cpp
@@ -705,6 +705,24 @@ LogicalResult TransposeLoadOp::verify() {
   return success();
 }
 
+//===----------------------------------------------------------------------===//
+// MakeDmaBaseOp
+//===----------------------------------------------------------------------===//
+
+LogicalResult MakeDmaBaseOp::verify() {
+  MemRefType ldsType = cast<MemRefType>(getLds().getType());
+  MemRefType globalType = cast<MemRefType>(getGlobal().getType());
+  if (!hasWorkgroupMemorySpace(ldsType.getMemorySpace())) {
+    return emitOpError(
+        "lds memref must have workgroup address space attribute.");
+  }
+  if (!hasGlobalMemorySpace(globalType.getMemorySpace())) {
+    return emitOpError(
+        "global memref must have global address space attribute.");
+  }
+  return success();
+}
+
 //===----------------------------------------------------------------------===//
 // MakeDmaDescriptorOp
 //===----------------------------------------------------------------------===//
diff --git a/mlir/lib/Dialect/AMDGPU/Transforms/MaskedloadToLoad.cpp b/mlir/lib/Dialect/AMDGPU/Transforms/MaskedloadToLoad.cpp
index f15c63c166e0a..89ef51f922cad 100644
--- a/mlir/lib/Dialect/AMDGPU/Transforms/MaskedloadToLoad.cpp
+++ b/mlir/lib/Dialect/AMDGPU/Transforms/MaskedloadToLoad.cpp
@@ -33,19 +33,18 @@ using namespace mlir::amdgpu;
 
 /// This pattern supports lowering of: `vector.maskedload` to `vector.load`
 /// and `arith.select` if the memref is in buffer address space.
-static LogicalResult baseInBufferAddrSpace(PatternRewriter &rewriter,
-                                           vector::MaskedLoadOp maskedOp) {
-  auto memRefType = dyn_cast<MemRefType>(maskedOp.getBase().getType());
+static LogicalResult hasBufferAddressSpace(Type type) {
+  auto memRefType = dyn_cast<MemRefType>(type);
   if (!memRefType)
-    return rewriter.notifyMatchFailure(maskedOp, "not a memref source");
+    return failure();
 
   Attribute addrSpace = memRefType.getMemorySpace();
   if (!isa_and_nonnull<amdgpu::AddressSpaceAttr>(addrSpace))
-    return rewriter.notifyMatchFailure(maskedOp, "no address space");
+    return failure();
 
   if (dyn_cast<amdgpu::AddressSpaceAttr>(addrSpace).getValue() !=
       amdgpu::AddressSpace::FatRawBuffer)
-    return rewriter.notifyMatchFailure(maskedOp, "not in buffer address space");
+    return failure();
 
   return success();
 }
@@ -83,10 +82,11 @@ struct MaskedLoadLowering final : OpRewritePattern<vector::MaskedLoadOp> {
   LogicalResult matchAndRewrite(vector::MaskedLoadOp maskedOp,
                                 PatternRewriter &rewriter) const override {
     if (maskedOp->hasAttr(kMaskedloadNeedsMask))
-      return failure();
+      return rewriter.notifyMatchFailure(maskedOp, "already rewritten");
 
-    if (failed(baseInBufferAddrSpace(rewriter, maskedOp))) {
-      return failure();
+    if (failed(hasBufferAddressSpace(maskedOp.getBase().getType()))) {
+      return rewriter.notifyMatchFailure(
+          maskedOp, "isn't a load from a fat buffer resource");
     }
 
     // Check if this is either a full inbounds load or an empty, oob load. If
@@ -176,9 +176,14 @@ struct FullMaskedLoadToConditionalLoad
 
   LogicalResult matchAndRewrite(vector::MaskedLoadOp loadOp,
                                 PatternRewriter &rewriter) const override {
+    if (succeeded(hasBufferAddressSpace(loadOp.getBase().getType())))
+      return rewriter.notifyMatchFailure(
+          loadOp, "buffer loads are handled by a more specialized pattern");
+
     FailureOr<Value> maybeCond = matchFullMask(rewriter, loadOp.getMask());
     if (failed(maybeCond)) {
-      return failure();
+      return rewriter.notifyMatchFailure(loadOp,
+                                         "isn't loading a broadcasted scalar");
     }
 
     Value cond = maybeCond.value();
@@ -203,6 +208,15 @@ struct FullMaskedStoreToConditionalStore
 
   LogicalResult matchAndRewrite(vector::MaskedStoreOp storeOp,
                                 PatternRewriter &rewriter) const override {
+    // A condition-free implementation of fully masked stores requires
+    // 1) an accessor for the num_records field on buffer resources/fat pointers
+    // 2) knowledge that said field will always be set accurately - that is,
+    // that writes to x < num_records of offset wouldn't trap, which is
+    // something a pattern user would need to assert or we'd need to prove.
+    //
+    // Therefore, conditional stores to buffers still go down this path at
+    // present.
+
     FailureOr<Value> maybeCond = matchFullMask(rewriter, storeOp.getMask());
     if (failed(maybeCond)) {
       return failure();
diff --git a/mlir/lib/Dialect/OpenACC/IR/OpenACC.cpp b/mlir/lib/Dialect/OpenACC/IR/OpenACC.cpp
index 841d1d781f1a1..7039bbe1d11ec 100644
--- a/mlir/lib/Dialect/OpenACC/IR/OpenACC.cpp
+++ b/mlir/lib/Dialect/OpenACC/IR/OpenACC.cpp
@@ -203,12 +203,68 @@ struct MemRefPointerLikeModel
 
     return false;
   }
+
+  mlir::Value genLoad(Type pointer, OpBuilder &builder, Location loc,
+                      TypedValue<PointerLikeType> srcPtr,
+                      Type valueType) const {
+    // Load from a memref - only valid for scalar memrefs (rank 0).
+    // This is because the address computation for memrefs is part of the load
+    // (and not computed separately), but the API does not have arguments for
+    // indexing.
+    auto memrefValue = dyn_cast_if_present<TypedValue<MemRefType>>(srcPtr);
+    if (!memrefValue)
+      return {};
+
+    auto memrefTy = memrefValue.getType();
+
+    // Only load from scalar memrefs (rank 0)
+    if (memrefTy.getRank() != 0)
+      return {};
+
+    return memref::LoadOp::create(builder, loc, memrefValue);
+  }
+
+  bool genStore(Type pointer, OpBuilder &builder, Location loc,
+                Value valueToStore, TypedValue<PointerLikeType> destPtr) const {
+    // Store to a memref - only valid for scalar memrefs (rank 0)
+    // This is because the address computation for memrefs is part of the store
+    // (and not computed separately), but the API does not have arguments for
+    // indexing.
+    auto memrefValue = dyn_cast_if_present<TypedValue<MemRefType>>(destPtr);
+    if (!memrefValue)
+      return false;
+
+    auto memrefTy = memrefValue.getType();
+
+    // Only store to scalar memrefs (rank 0)
+    if (memrefTy.getRank() != 0)
+      return false;
+
+    memref::StoreOp::create(builder, loc, valueToStore, memrefValue);
+    return true;
+  }
 };
 
 struct LLVMPointerPointerLikeModel
     : public PointerLikeType::ExternalModel<LLVMPointerPointerLikeModel,
                                             LLVM::LLVMPointerType> {
   Type getElementType(Type pointer) const { return Type(); }
+
+  mlir::Value genLoad(Type pointer, OpBuilder &builder, Location loc,
+                      TypedValue<PointerLikeType> srcPtr,
+                      Type valueType) const {
+    // For LLVM pointers, we need the valueType to determine what to load
+    if (!valueType)
+      return {};
+
+    return LLVM::LoadOp::create(builder, loc, valueType, srcPtr);
+  }
+
+  bool genStore(Type pointer, OpBuilder &builder, Location loc,
+                Value valueToStore, TypedValue<PointerLikeType> destPtr) const {
+    LLVM::StoreOp::create(builder, loc, valueToStore, destPtr);
+    return true;
+  }
 };
 
 struct MemrefAddressOfGlobalModel
@@ -4293,6 +4349,24 @@ RoutineOp::getGangDimValue(mlir::acc::DeviceType deviceType) {
   return std::nullopt;
 }
 
+void RoutineOp::addSeq(MLIRContext *context,
+                       llvm::ArrayRef<DeviceType> effectiveDeviceTypes) {
+  setSeqAttr(addDeviceTypeAffectedOperandHelper(context, getSeqAttr(),
+                                                effectiveDeviceTypes));
+}
+
+void RoutineOp::addVector(MLIRContext *context,
+                          llvm::ArrayRef<DeviceType> effectiveDeviceTypes) {
+  setVectorAttr(addDeviceTypeAffectedOperandHelper(context, getVectorAttr(),
+                                                   effectiveDeviceTypes));
+}
+
+void RoutineOp::addWorker(MLIRContext *context,
+                          llvm::ArrayRef<DeviceType> effectiveDeviceTypes) {
+  setWorkerAttr(addDeviceTypeAffectedOperandHelper(context, getWorkerAttr(),
+                                                   effectiveDeviceTypes));
+}
+
 //===----------------------------------------------------------------------===//
 // InitOp
 //===----------------------------------------------------------------------===//
diff --git a/mlir/lib/Dialect/XeGPU/Transforms/XeGPUPropagateLayout.cpp b/mlir/lib/Dialect/XeGPU/Transforms/XeGPUPropagateLayout.cpp
index f2b0e71c9397f..59a1ad9dbe189 100644
--- a/mlir/lib/Dialect/XeGPU/Transforms/XeGPUPropagateLayout.cpp
+++ b/mlir/lib/Dialect/XeGPU/Transforms/XeGPUPropagateLayout.cpp
@@ -517,8 +517,7 @@ void LayoutInfoPropagation::visitPrefetchNdOp(
     auto [bWidth, bHeight, bCount] = blockWHC.value();
     SmallVector<int> instData;
     int instWidth = xegpu::getLargestDivisor(
-        static_cast<int>(tdescTy.getDimSize(tdescTy.getRank() - 1)), bWidth,
-        bCount);
+        static_cast<int>(tdescTy.getDimSize(tdescTy.getRank() - 1)), bWidth);
     if (instWidth == -1)
       prefetch.emitWarning(
           "No suitable instruction multiple found for the given shape.");
@@ -759,8 +758,7 @@ void LayoutInfoPropagation::visitStoreNdOp(
     auto [bWidth, bHeight, bCount] = blockWHC.value();
     SmallVector<int> instData;
     int instWidth = xegpu::getLargestDivisor(
-        static_cast<int>(dataTy.getDimSize(dataTy.getRank() - 1)), bWidth,
-        bCount);
+        static_cast<int>(dataTy.getDimSize(dataTy.getRank() - 1)), bWidth);
     if (instWidth == -1)
       store.emitWarning(
           "No suitable instruction multiple found for the given shape.");
diff --git a/mlir/test/Conversion/AMDGPUToROCDL/cvt_scale_pk-gfx1250.mlir b/mlir/test/Conversion/AMDGPUToROCDL/gfx1250.mlir
similarity index 81%
rename from mlir/test/Conversion/AMDGPUToROCDL/cvt_scale_pk-gfx1250.mlir
rename to mlir/test/Conversion/AMDGPUToROCDL/gfx1250.mlir
index d2391140ce056..27daea58f8f92 100644
--- a/mlir/test/Conversion/AMDGPUToROCDL/cvt_scale_pk-gfx1250.mlir
+++ b/mlir/test/Conversion/AMDGPUToROCDL/gfx1250.mlir
@@ -162,3 +162,51 @@ func.func @amdgpu.scaled_ext_packed816_invalid_dst_elem_type(%v: vector<16xf6E3M
   %ret0 = amdgpu.scaled_ext_packed816 %v scale(%scale) blockSize(32) firstScaleLane(0) firstScaleByte(0) : vector<16xf6E3M2FN>, vector<4xf8E8M0FNU> -> vector<16xf64>
   return %ret0: vector<16xf64>
 }
+
+// -----
+
+#gpu_global_addrspace = 1
+#gpu_lds_addrspace = 3
+#amdgpu_fat_buffer_addrspace = 7
+
+// CHECK-LABEL: func @make_dma_base
+// CHECK-SAME: (%[[IDX:.+]]: index, %[[MEM:.+]]: memref<8xi32, 1>, %[[SMEM:.+]]: memref<8xi32, 3>)
+func.func @make_dma_base(%idx: index, %mem: memref<8xi32, #gpu_global_addrspace>, %smem: memref<8xi32,#gpu_lds_addrspace>) -> (!amdgpu.tdm_base<i32>) {
+  // CHECK-DAG: %[[INT:.+]] = builtin.unrealized_conversion_cast %[[IDX]] : index to i64
+  // CHECK-DAG: %[[MEMREF_DESC_MEM:.+]] = builtin.unrealized_conversion_cast %[[MEM]] : memref<8xi32, 1>
+  // CHECK-DAG: %[[MEMREF_DESC_SMEM:.+]] = builtin.unrealized_conversion_cast %[[SMEM]] : memref<8xi32, 3>
+
+  // CHECK-DAG: %[[MEM_BASE_PTR:.+]] = llvm.extractvalue %[[MEMREF_DESC_MEM]][1] : !llvm.struct<(ptr<1>
+  // CHECK-DAG: %[[SMEM_BASE_PTR:.+]] = llvm.extractvalue %[[MEMREF_DESC_SMEM]][1] : !llvm.struct<(ptr<3>
+
+  // CHECK-DAG: %[[MEM_BASE_OFFSET:.+]] = llvm.getelementptr %[[MEM_BASE_PTR]][%[[INT]]]
+  // CHECK-DAG: %[[SMEM_BASE_OFFSET:.+]] = llvm.getelementptr %[[SMEM_BASE_PTR]][%[[INT]]]
+
+  // CHECK-DAG: %[[MEM_INT:.+]] = llvm.ptrtoint %[[MEM_BASE_OFFSET]] : !llvm.ptr<1> to i64
+  // CHECK-DAG: %[[SMEM_INT:.+]] = llvm.ptrtoint %[[SMEM_BASE_OFFSET]] : !llvm.ptr<3> to i32
+
+  // CHECK: %[[MEM_INT_LOW:.+]] = llvm.trunc %[[MEM_INT]] : i64 to i32
+  // CHECK-DAG: %[[SHIFT:.+]] = llvm.mlir.constant(32 : i64)
+  // CHECK: %[[SHIFTED_MEM_INT:.+]] = llvm.lshr %[[MEM_INT]], %[[SHIFT]]
+  // CHECK: %[[MEM_INT_HIGH:.+]] = llvm.trunc %[[SHIFTED_MEM_INT]] : i64 to i32
+  // CHECK-DAG: %[[MASK:.+]] = llvm.mlir.constant(33554431 : i32)
+  // CHECK: %[[VALID_MEM_INT_HIGH:.+]] = llvm.and %[[MEM_INT_HIGH]], %[[MASK]]
+
+  // CHECK-DAG: %[[TYPE_FIELD:.+]] = llvm.mlir.constant(-2147483648 : i32)
+  // CHECK: %[[MEM_INT_HIGH_TYPE:.+]] = llvm.or %[[VALID_MEM_INT_HIGH]], %[[TYPE_FIELD]]
+
+  // CHECK-DAG: %[[C0:.+]] = llvm.mlir.constant(0 : i32) : i32
+  // CHECK-DAG: %[[C1:.+]] = llvm.mlir.constant(1 : i32) : i32
+  // CHECK-DAG: %[[C2:.+]] = llvm.mlir.constant(2 : i32) : i32
+  // CHECK-DAG: %[[C3:.+]] = llvm.mlir.constant(3 : i32) : i32
+
+  // CHECK: %[[V4I32_0_0:.+]] = llvm.mlir.poison : vector<4xi32>
+  // CHECK: %[[V4I32_0_1:.+]] = llvm.insertelement %[[C1]], %[[V4I32_0_0]][%[[C0]] : i32]
+  // CHECK: %[[V4I32_0_2:.+]] = llvm.insertelement %[[SMEM_INT]], %[[V4I32_0_1]][%[[C1]] : i32]
+  // CHECK: %[[V4I32_0_3:.+]] = llvm.insertelement %[[MEM_INT_LOW]], %[[V4I32_0_2]][%[[C2]] : i32]
+  // CHECK: %[[V4I32_0_4:.+]] = llvm.insertelement %[[MEM_INT_HIGH_TYPE]], %[[V4I32_0_3]][%[[C3]] : i32]
+
+  %0 = amdgpu.make_dma_base %mem[%idx], %smem[%idx] : memref<8xi32, #gpu_global_addrspace>, memref<8xi32, #gpu_lds_addrspace> -> !amdgpu.tdm_base<i32>
+
+  func.return %0 : !amdgpu.tdm_base<i32>
+}
diff --git a/mlir/test/Dialect/AMDGPU/invalid.mlir b/mlir/test/Dialect/AMDGPU/invalid.mlir
index 5b3a79d14cb1a..b915bfa324c77 100644
--- a/mlir/test/Dialect/AMDGPU/invalid.mlir
+++ b/mlir/test/Dialect/AMDGPU/invalid.mlir
@@ -357,6 +357,20 @@ func.func @scaled_mfma_invalid_k(%arg0 : vector<4xf8E8M0FNU>, %arg1 : vector<32x
 
 // -----
 
+func.func @make_dma_base_invalid_addressspace(%idx: index, %mem: memref<8xi32>) {
+  // expected-error@+1 {{'amdgpu.make_dma_base' op lds memref must have workgroup address space attribute.}}
+  amdgpu.make_dma_base %mem[%idx], %mem[%idx] : memref<8xi32>, memref<8xi32> -> !amdgpu.tdm_base<i32>
+}
+
+// -----
+
+func.func @make_dma_base_invalid_addressspace(%idx: index, %smem : memref<8xi32, #gpu.address_space<workgroup>>) {
+  // expected-error@+1 {{'amdgpu.make_dma_base' op global memref must have global address space attribute.}}
+  amdgpu.make_dma_base %smem[%idx], %smem[%idx] : memref<8xi32, #gpu.address_space<workgroup>>, memref<8xi32, #gpu.address_space<workgroup>> -> !amdgpu.tdm_base<i32>
+}
+
+// -----
+
 func.func @make_dma_base_invalid_barrier(%base: !amdgpu.tdm_base<i32>, %barrier: memref<8xi32>, %idx: index) {
   // expected-error@+1 {{'amdgpu.make_dma_descriptor' op atomic barrier address must be in LDS.}}
   amdgpu.make_dma_descriptor %base globalSize [0] globalStride [1] sharedSize [0] atomicBarrier(%barrier[%idx] : memref<8xi32>) : !amdgpu.tdm_base<i32> -> !amdgpu.tdm_descriptor
diff --git a/mlir/test/Dialect/AMDGPU/ops.mlir b/mlir/test/Dialect/AMDGPU/ops.mlir
index 390ad8cb8c1a5..3260bd4a8df9a 100644
--- a/mlir/test/Dialect/AMDGPU/ops.mlir
+++ b/mlir/test/Dialect/AMDGPU/ops.mlir
@@ -691,9 +691,6 @@ func.func @memory_counter_wait() {
 func.func @make_dma_base(%idx: index, %mem: memref<8xi32>, %smem: memref<8xi32, #gpu.address_space<workgroup>>) {
   // CHECK: amdgpu.make_dma_base %[[MEM]][%[[IDX]]], %[[SMEM]][%[[IDX]]] : memref<8xi32>, memref<8xi32, #gpu.address_space<workgroup>> -> !amdgpu.tdm_base<i32>
   amdgpu.make_dma_base %mem[%idx], %smem[%idx] : memref<8xi32>, memref<8xi32, #gpu.address_space<workgroup>> -> !amdgpu.tdm_base<i32>
-
-  // CHECK: amdgpu.make_dma_base %[[SMEM]][%[[IDX]]], %[[MEM]][%[[IDX]]] : memref<8xi32, #gpu.address_space<workgroup>>, memref<8xi32> -> !amdgpu.tdm_base<i32>
-  amdgpu.make_dma_base %smem[%idx], %mem[%idx] : memref<8xi32, #gpu.address_space<workgroup>>, memref<8xi32> -> !amdgpu.tdm_base<i32>
   func.return
 }
 
@@ -748,3 +745,4 @@ func.func @make_dma_descriptor(%base: !amdgpu.tdm_base<i32>, %barrier: memref<8x
 
   func.return
 }
+
diff --git a/mlir/test/Dialect/OpenACC/pointer-like-interface-load.mlir b/mlir/test/Dialect/OpenACC/pointer-like-interface-load.mlir
new file mode 100644
index 0000000000000..36df6a1d1bbe3
--- /dev/null
+++ b/mlir/test/Dialect/OpenACC/pointer-like-interface-load.mlir
@@ -0,0 +1,29 @@
+// RUN: mlir-opt %s --split-input-file --pass-pipeline="builtin.module(func.func(test-acc-pointer-like-interface{test-mode=load}))" 2>&1 | FileCheck %s
+
+func.func @test_memref_load_scalar() {
+  %ptr = memref.alloca() {test.ptr} : memref<f32>
+  // CHECK: Successfully generated load for operation: %[[PTR:.*]] = memref.alloca() {test.ptr} : memref<f32>
+  // CHECK: Loaded value type: f32
+  // CHECK: Generated: %{{.*}} = memref.load %[[PTR]][] : memref<f32>
+  return
+}
+
+// -----
+
+func.func @test_memref_load_int() {
+  %ptr = memref.alloca() {test.ptr} : memref<i64>
+  // CHECK: Successfully generated load for operation: %[[PTR:.*]] = memref.alloca() {test.ptr} : memref<i64>
+  // CHECK: Loaded value type: i64
+  // CHECK: Generated: %{{.*}} = memref.load %[[PTR]][] : memref<i64>
+  return
+}
+
+// -----
+
+func.func @test_memref_load_dynamic() {
+  %c10 = arith.constant 10 : index
+  %ptr = memref.alloc(%c10) {test.ptr} : memref<?xf32>
+  // CHECK: Failed to generate load for operation: %[[PTR:.*]] = memref.alloc(%{{.*}}) {test.ptr} : memref<?xf32>
+  return
+}
+
diff --git a/mlir/test/Dialect/OpenACC/pointer-like-interface-store.mlir b/mlir/test/Dialect/OpenACC/pointer-like-interface-store.mlir
new file mode 100644
index 0000000000000..0fee43102d6d9
--- /dev/null
+++ b/mlir/test/Dialect/OpenACC/pointer-like-interface-store.mlir
@@ -0,0 +1,39 @@
+// RUN: mlir-opt %s --split-input-file --pass-pipeline="builtin.module(func.func(test-acc-pointer-like-interface{test-mode=store}))" 2>&1 | FileCheck %s
+
+func.func @test_memref_store_scalar() {
+  %ptr = memref.alloca() {test.ptr} : memref<f32>
+  // CHECK: Successfully generated store for operation: %[[PTR:.*]] = memref.alloca() {test.ptr} : memref<f32>
+  // CHECK: Generated: %[[VAL:.*]] = arith.constant 4.200000e+01 : f32
+  // CHECK: Generated: memref.store %[[VAL]], %[[PTR]][] : memref<f32>
+  return
+}
+
+// -----
+
+func.func @test_memref_store_int() {
+  %ptr = memref.alloca() {test.ptr} : memref<i32>
+  // CHECK: Successfully generated store for operation: %[[PTR:.*]] = memref.alloca() {test.ptr} : memref<i32>
+  // CHECK: Generated: %[[VAL:.*]] = arith.constant 42 : i32
+  // CHECK: Generated: memref.store %[[VAL]], %[[PTR]][] : memref<i32>
+  return
+}
+
+// -----
+
+func.func @test_memref_store_i64() {
+  %ptr = memref.alloca() {test.ptr} : memref<i64>
+  // CHECK: Successfully generated store for operation: %[[PTR:.*]] = memref.alloca() {test.ptr} : memref<i64>
+  // CHECK: Generated: %[[VAL:.*]] = arith.constant 42 : i64
+  // CHECK: Generated: memref.store %[[VAL]], %[[PTR]][] : memref<i64>
+  return
+}
+
+// -----
+
+func.func @test_memref_store_dynamic() {
+  %c10 = arith.constant 10 : index
+  %ptr = memref.alloc(%c10) {test.ptr} : memref<?xf32>
+  // CHECK: Failed to generate store for operation: %[[PTR:.*]] = memref.alloc(%{{.*}}) {test.ptr} : memref<?xf32>
+  return
+}
+
diff --git a/mlir/test/Dialect/Vector/vector-sink.mlir b/mlir/test/Dialect/Vector/vector-sink.mlir
index 577b06df42929..beaba52af1841 100644
--- a/mlir/test/Dialect/Vector/vector-sink.mlir
+++ b/mlir/test/Dialect/Vector/vector-sink.mlir
@@ -780,7 +780,7 @@ func.func @negative_extract_load_scalable(%arg0: memref<?xf32>, %arg1: index) ->
 }
 
 //-----------------------------------------------------------------------------
-// [Pattern: StoreOpFromSplatOrBroadcast]
+// [Pattern: StoreOpFromBroadcast]
 //-----------------------------------------------------------------------------
 
 // CHECK-LABEL: @store_splat
diff --git a/mlir/test/Dialect/XeGPU/propagate-layout-inst-data.mlir b/mlir/test/Dialect/XeGPU/propagate-layout-inst-data.mlir
index d911baa49acbb..32fb3178a8af2 100644
--- a/mlir/test/Dialect/XeGPU/propagate-layout-inst-data.mlir
+++ b/mlir/test/Dialect/XeGPU/propagate-layout-inst-data.mlir
@@ -6,6 +6,8 @@
 // CHECK: %[[CST:.*]] = arith.constant dense<0.000000e+00> : vector<8x16xf32>
 // CHECK: %[[TDESC_SRC:.*]] = xegpu.create_nd_tdesc %[[ARG0]] : memref<8x32xf32> -> !xegpu.tensor_desc<8x32xf32, #xegpu.layout<inst_data = [8, 16]>>
 // CHECK: %[[TDESC_DST:.*]] = xegpu.create_nd_tdesc %[[ARG1]] : memref<8x32xf32> -> !xegpu.tensor_desc<8x32xf32, #xegpu.layout<inst_data = [8, 16]>>
+// CHECK: xegpu.prefetch_nd %[[TDESC_SRC]] <{l1_hint = #xegpu.cache_hint<cached>, l2_hint = #xegpu.cache_hint<uncached>, layout = #xegpu.layout<inst_data = [8, 16]>}> :
+// CHECK-SAME: !xegpu.tensor_desc<8x32xf32, #xegpu.layout<inst_data = [8, 16]>>
 // CHECK: %[[LOADED:.*]] = xegpu.load_nd %0 <{layout = #xegpu.layout<inst_data = [8, 16]>}> {layout_result_0 = #xegpu.layout<inst_data = [8, 16]>} :
 // CHECK-SAME: !xegpu.tensor_desc<8x32xf32, #xegpu.layout<inst_data = [8, 16]>> -> vector<8x32xf32>
 // CHECK: xegpu.store_nd %[[LOADED]], %[[TDESC_DST]] <{layout = #xegpu.layout<inst_data = [8, 16]>}> : vector<8x32xf32>, !xegpu.tensor_desc<8x32xf32, #xegpu.layout<inst_data = [8, 16]>>
@@ -16,6 +18,7 @@ func.func @load_store_no_array_len(%arg0: memref<8x32xf32>, %arg1: memref<8x32xf
   %cst = arith.constant dense<0.000000e+00> : vector<8x16xf32>
   %0 = xegpu.create_nd_tdesc %arg0 : memref<8x32xf32> -> !xegpu.tensor_desc<8x32xf32>
   %1 = xegpu.create_nd_tdesc %arg1 : memref<8x32xf32> -> !xegpu.tensor_desc<8x32xf32>
+  xegpu.prefetch_nd %0 <{l1_hint = #xegpu.cache_hint<cached>, l2_hint = #xegpu.cache_hint<uncached>}>: !xegpu.tensor_desc<8x32xf32>
   %2 = xegpu.load_nd %0  : !xegpu.tensor_desc<8x32xf32> -> vector<8x32xf32>
   xegpu.store_nd %2, %1  : vector<8x32xf32>, !xegpu.tensor_desc<8x32xf32>
   return
diff --git a/mlir/test/lib/Dialect/OpenACC/TestPointerLikeTypeInterface.cpp b/mlir/test/lib/Dialect/OpenACC/TestPointerLikeTypeInterface.cpp
index 027b0a1a8b80b..3ff0dc85b2152 100644
--- a/mlir/test/lib/Dialect/OpenACC/TestPointerLikeTypeInterface.cpp
+++ b/mlir/test/lib/Dialect/OpenACC/TestPointerLikeTypeInterface.cpp
@@ -46,7 +46,7 @@ struct TestPointerLikeTypeInterfacePass
 
   Pass::Option<std::string> testMode{
       *this, "test-mode",
-      llvm::cl::desc("Test mode: walk, alloc, copy, or free"),
+      llvm::cl::desc("Test mode: walk, alloc, copy, free, load, or store"),
       llvm::cl::init("walk")};
 
   StringRef getArgument() const override {
@@ -75,6 +75,10 @@ struct TestPointerLikeTypeInterfacePass
   void testGenCopy(Operation *srcOp, Operation *destOp, Value srcResult,
                    Value destResult, PointerLikeType pointerType,
                    OpBuilder &builder);
+  void testGenLoad(Operation *op, Value result, PointerLikeType pointerType,
+                   OpBuilder &builder);
+  void testGenStore(Operation *op, Value result, PointerLikeType pointerType,
+                    OpBuilder &builder, Value providedValue = {});
 
   struct PointerCandidate {
     Operation *op;
@@ -92,9 +96,12 @@ void TestPointerLikeTypeInterfacePass::runOnOperation() {
   auto func = getOperation();
   OpBuilder builder(&getContext());
 
-  if (testMode == "alloc" || testMode == "free") {
+  if (testMode == "alloc" || testMode == "free" || testMode == "load" ||
+      testMode == "store") {
     // Collect all candidates first
     SmallVector<PointerCandidate> candidates;
+    // For store mode, also look for a test value to use
+    Value testValue;
     func.walk([&](Operation *op) {
       if (op->hasAttr("test.ptr")) {
         for (auto result : op->getResults()) {
@@ -105,6 +112,11 @@ void TestPointerLikeTypeInterfacePass::runOnOperation() {
           }
         }
       }
+      // Collect value marked with test.value for store tests
+      if (testMode == "store" && op->hasAttr("test.value")) {
+        if (op->getNumResults() > 0)
+          testValue = op->getResult(0);
+      }
     });
 
     // Now test all candidates
@@ -115,6 +127,12 @@ void TestPointerLikeTypeInterfacePass::runOnOperation() {
       else if (testMode == "free")
         testGenFree(candidate.op, candidate.result, candidate.pointerType,
                     builder);
+      else if (testMode == "load")
+        testGenLoad(candidate.op, candidate.result, candidate.pointerType,
+                    builder);
+      else if (testMode == "store")
+        testGenStore(candidate.op, candidate.result, candidate.pointerType,
+                     builder, testValue);
     }
   } else if (testMode == "copy") {
     // Collect all source and destination candidates
@@ -292,6 +310,105 @@ void TestPointerLikeTypeInterfacePass::testGenCopy(
   }
 }
 
+void TestPointerLikeTypeInterfacePass::testGenLoad(Operation *op, Value result,
+                                                   PointerLikeType pointerType,
+                                                   OpBuilder &builder) {
+  Location loc = op->getLoc();
+
+  // Create a new builder with the listener and set insertion point
+  OperationTracker tracker;
+  OpBuilder newBuilder(op->getContext());
+  newBuilder.setListener(&tracker);
+  newBuilder.setInsertionPointAfter(op);
+
+  // Call the genLoad API
+  auto typedResult = cast<TypedValue<PointerLikeType>>(result);
+  Value loadRes = pointerType.genLoad(newBuilder, loc, typedResult, Type());
+
+  if (loadRes) {
+    llvm::errs() << "Successfully generated load for operation: ";
+    op->print(llvm::errs());
+    llvm::errs() << "\n";
+    llvm::errs() << "\tLoaded value type: ";
+    loadRes.getType().print(llvm::errs());
+    llvm::errs() << "\n";
+
+    // Print all operations that were inserted
+    for (Operation *insertedOp : tracker.insertedOps) {
+      llvm::errs() << "\tGenerated: ";
+      insertedOp->print(llvm::errs());
+      llvm::errs() << "\n";
+    }
+  } else {
+    llvm::errs() << "Failed to generate load for operation: ";
+    op->print(llvm::errs());
+    llvm::errs() << "\n";
+  }
+}
+
+void TestPointerLikeTypeInterfacePass::testGenStore(Operation *op, Value result,
+                                                    PointerLikeType pointerType,
+                                                    OpBuilder &builder,
+                                                    Value providedValue) {
+  Location loc = op->getLoc();
+
+  // Create a new builder with the listener and set insertion point
+  OperationTracker tracker;
+  OpBuilder newBuilder(op->getContext());
+  newBuilder.setListener(&tracker);
+  newBuilder.setInsertionPointAfter(op);
+
+  // Use provided value if available, otherwise create a constant
+  Value valueToStore = providedValue;
+  if (!valueToStore) {
+    // Create a test value to store - use a constant matching the element type
+    Type elementType = pointerType.getElementType();
+    if (!elementType) {
+      llvm::errs() << "Failed to generate store for operation: ";
+      op->print(llvm::errs());
+      llvm::errs() << "\n";
+      return;
+    }
+
+    if (elementType.isIntOrIndex()) {
+      auto attr = newBuilder.getIntegerAttr(elementType, 42);
+      valueToStore =
+          arith::ConstantOp::create(newBuilder, loc, elementType, attr);
+    } else if (auto floatType = dyn_cast<FloatType>(elementType)) {
+      auto attr = newBuilder.getFloatAttr(floatType, 42.0);
+      valueToStore =
+          arith::ConstantOp::create(newBuilder, loc, floatType, attr);
+    } else {
+      llvm::errs() << "Failed to generate store for operation: ";
+      op->print(llvm::errs());
+      llvm::errs() << "\n";
+      return;
+    }
+  }
+
+  // Call the genStore API
+  auto typedResult = cast<TypedValue<PointerLikeType>>(result);
+  bool success =
+      pointerType.genStore(newBuilder, loc, valueToStore, typedResult);
+
+  if (success) {
+    llvm::errs() << "Successfully generated store for operation: ";
+    op->print(llvm::errs());
+    llvm::errs() << "\n";
+
+    // Print all operations that were inserted
+    for (Operation *insertedOp : tracker.insertedOps) {
+      llvm::errs() << "\tGenerated: ";
+      insertedOp->print(llvm::errs());
+      llvm::errs() << "\n";
+    }
+  } else {
+    llvm::errs() << "Failed to generate store for operation: ";
+    op->print(llvm::errs());
+    llvm::errs() << "\n";
+  }
+}
+
 } // namespace
 
 //===----------------------------------------------------------------------===//
diff --git a/offload/include/Shared/Debug.h b/offload/include/Shared/Debug.h
index 1f8633b0afa55..d38e85dfb966a 100644
--- a/offload/include/Shared/Debug.h
+++ b/offload/include/Shared/Debug.h
@@ -39,6 +39,7 @@
 #define OMPTARGET_SHARED_DEBUG_H
 
 #include <atomic>
+#include <cstdarg>
 #include <mutex>
 #include <string>
 #include <cstdint>
@@ -106,17 +107,6 @@ inline std::atomic<uint32_t> &getInfoLevelInternal() {
 
 inline uint32_t getInfoLevel() { return getInfoLevelInternal().load(); }
 
-inline uint32_t getDebugLevel() {
-  static uint32_t DebugLevel = 0;
-  static std::once_flag Flag{};
-  std::call_once(Flag, []() {
-    if (char *EnvStr = getenv("LIBOMPTARGET_DEBUG"))
-      DebugLevel = std::stoi(EnvStr);
-  });
-
-  return DebugLevel;
-}
-
 #undef USED
 #undef GCC_VERSION
 
@@ -175,46 +165,11 @@ inline uint32_t getDebugLevel() {
     fprintf(_stdDst, __VA_ARGS__);                                             \
   } while (0)
 
-// Debugging messages
-#ifdef OMPTARGET_DEBUG
-#include <stdio.h>
-
-#define DEBUGP(prefix, ...)                                                    \
-  {                                                                            \
-    fprintf(stderr, "%s --> ", prefix);                                        \
-    fprintf(stderr, __VA_ARGS__);                                              \
-  }
-
-/// Emit a message for debugging
-#define DP(...)                                                                \
-  do {                                                                         \
-    if (getDebugLevel() > 0) {                                                 \
-      DEBUGP(DEBUG_PREFIX, __VA_ARGS__);                                       \
-    }                                                                          \
-  } while (false)
-
-/// Emit a message for debugging or failure if debugging is disabled
-#define REPORT(...)                                                            \
-  do {                                                                         \
-    if (getDebugLevel() > 0) {                                                 \
-      DP(__VA_ARGS__);                                                         \
-    } else {                                                                   \
-      FAILURE_MESSAGE(__VA_ARGS__);                                            \
-    }                                                                          \
-  } while (false)
-#else
-#define DEBUGP(prefix, ...)                                                    \
-  {}
-#define DP(...)                                                                \
-  {}
-#define REPORT(...) FAILURE_MESSAGE(__VA_ARGS__);
-#endif // OMPTARGET_DEBUG
-
 /// Emit a message giving the user extra information about the runtime if
 #define INFO(_flags, _id, ...)                                                 \
   do {                                                                         \
-    if (getDebugLevel() > 0) {                                                 \
-      DEBUGP(DEBUG_PREFIX, __VA_ARGS__);                                       \
+    if (::llvm::offload::debug::isDebugEnabled()) {                            \
+      DP(__VA_ARGS__);                                                         \
     } else if (getInfoLevel() & _flags) {                                      \
       INFO_MESSAGE(_id, __VA_ARGS__);                                          \
     }                                                                          \
@@ -231,17 +186,92 @@ inline uint32_t getDebugLevel() {
 
 namespace llvm::offload::debug {
 
-#ifdef OMPTARGET_DEBUG
+/// A raw_ostream that tracks `\n` and print the prefix after each
+/// newline. Based on raw_ldbg_ostream from Support/DebugLog.h
+class LLVM_ABI odbg_ostream final : public raw_ostream {
+public:
+  enum IfLevel : uint32_t;
+  enum OnlyLevel : uint32_t;
 
-struct DebugFilter {
-  StringRef Type;
-  uint32_t Level;
-};
+private:
+  std::string Prefix;
+  raw_ostream &Os;
+  uint32_t BaseLevel;
+  bool ShouldPrefixNextString;
+  bool ShouldEmitNewLineOnDestruction;
+  bool NeedEndNewLine = false;
 
-struct DebugSettings {
-  bool Enabled = false;
-  uint32_t DefaultLevel = 1;
-  llvm::SmallVector<DebugFilter> Filters;
+  /// If the stream is muted, writes to it are ignored
+  bool Muted = false;
+
+  /// Split the line on newlines and insert the prefix before each
+  /// newline. Forward everything to the underlying stream.
+  void write_impl(const char *Ptr, size_t Size) final {
+    if (Muted)
+      return;
+
+    NeedEndNewLine = false;
+    auto Str = StringRef(Ptr, Size);
+    auto Eol = Str.find('\n');
+    // Handle `\n` occurring in the string, ensure to print the prefix at the
+    // beginning of each line.
+    while (Eol != StringRef::npos) {
+      // Take the line up to the newline (including the newline).
+      StringRef Line = Str.take_front(Eol + 1);
+      if (!Line.empty())
+        writeWithPrefix(Line);
+      // We printed a newline, record here to print a prefix.
+      ShouldPrefixNextString = true;
+      Str = Str.drop_front(Eol + 1);
+      Eol = Str.find('\n');
+    }
+    if (!Str.empty()) {
+      writeWithPrefix(Str);
+      NeedEndNewLine = true;
+    }
+  }
+  void emitPrefix() { Os.write(Prefix.c_str(), Prefix.size()); }
+  void writeWithPrefix(StringRef Str) {
+    if (ShouldPrefixNextString) {
+      emitPrefix();
+      ShouldPrefixNextString = false;
+    }
+    Os.write(Str.data(), Str.size());
+  }
+
+public:
+  explicit odbg_ostream(std::string Prefix, raw_ostream &Os, uint32_t BaseLevel,
+                        bool ShouldPrefixNextString = true,
+                        bool ShouldEmitNewLineOnDestruction = true)
+      : Prefix(std::move(Prefix)), Os(Os), BaseLevel(BaseLevel),
+        ShouldPrefixNextString(ShouldPrefixNextString),
+        ShouldEmitNewLineOnDestruction(ShouldEmitNewLineOnDestruction) {
+    SetUnbuffered();
+  }
+  ~odbg_ostream() final {
+    if (ShouldEmitNewLineOnDestruction && NeedEndNewLine)
+      Os << '\n';
+  }
+  odbg_ostream(const odbg_ostream &) = delete;
+  odbg_ostream &operator=(const odbg_ostream &) = delete;
+  odbg_ostream(odbg_ostream &&other) : Os(other.Os) {
+    Prefix = std::move(other.Prefix);
+    BaseLevel = other.BaseLevel;
+    ShouldPrefixNextString = other.ShouldPrefixNextString;
+    ShouldEmitNewLineOnDestruction = other.ShouldEmitNewLineOnDestruction;
+    NeedEndNewLine = other.NeedEndNewLine;
+    Muted = other.Muted;
+  }
+
+  /// Forward the current_pos method to the underlying stream.
+  uint64_t current_pos() const final { return Os.tell(); }
+
+  /// Some of the `<<` operators expect an lvalue, so we trick the type
+  /// system.
+  odbg_ostream &asLvalue() { return *this; }
+
+  void shouldMute(const IfLevel Filter) { Muted = Filter > BaseLevel; }
+  void shouldMute(const OnlyLevel Filter) { Muted = BaseLevel != Filter; }
 };
 
 /// dbgs - Return a circular-buffered debug stream.
@@ -256,6 +286,19 @@ struct DebugSettings {
   return thestrm.strm;
 }
 
+#ifdef OMPTARGET_DEBUG
+
+struct DebugFilter {
+  StringRef Type;
+  uint32_t Level;
+};
+
+struct DebugSettings {
+  bool Enabled = false;
+  uint32_t DefaultLevel = 1;
+  llvm::SmallVector<DebugFilter> Filters;
+};
+
 [[maybe_unused]] static DebugFilter parseDebugFilter(StringRef Filter) {
   size_t Pos = Filter.find(':');
   if (Pos == StringRef::npos)
@@ -337,80 +380,6 @@ shouldPrintDebug(const char *Component, const char *Type, uint32_t &Level) {
   return false;
 }
 
-/// A raw_ostream that tracks `\n` and print the prefix after each
-/// newline. Based on raw_ldbg_ostream from Support/DebugLog.h
-class LLVM_ABI odbg_ostream final : public raw_ostream {
-public:
-  enum IfLevel : uint32_t;
-  enum OnlyLevel : uint32_t;
-
-private:
-  std::string Prefix;
-  raw_ostream &Os;
-  uint32_t BaseLevel;
-  bool ShouldPrefixNextString;
-  bool ShouldEmitNewLineOnDestruction;
-
-  /// If the stream is muted, writes to it are ignored
-  bool Muted = false;
-
-  /// Split the line on newlines and insert the prefix before each
-  /// newline. Forward everything to the underlying stream.
-  void write_impl(const char *Ptr, size_t Size) final {
-    if (Muted)
-      return;
-
-    auto Str = StringRef(Ptr, Size);
-    auto Eol = Str.find('\n');
-    // Handle `\n` occurring in the string, ensure to print the prefix at the
-    // beginning of each line.
-    while (Eol != StringRef::npos) {
-      // Take the line up to the newline (including the newline).
-      StringRef Line = Str.take_front(Eol + 1);
-      if (!Line.empty())
-        writeWithPrefix(Line);
-      // We printed a newline, record here to print a prefix.
-      ShouldPrefixNextString = true;
-      Str = Str.drop_front(Eol + 1);
-      Eol = Str.find('\n');
-    }
-    if (!Str.empty())
-      writeWithPrefix(Str);
-  }
-  void emitPrefix() { Os.write(Prefix.c_str(), Prefix.size()); }
-  void writeWithPrefix(StringRef Str) {
-    if (ShouldPrefixNextString) {
-      emitPrefix();
-      ShouldPrefixNextString = false;
-    }
-    Os.write(Str.data(), Str.size());
-  }
-
-public:
-  explicit odbg_ostream(std::string Prefix, raw_ostream &Os, uint32_t BaseLevel,
-                        bool ShouldPrefixNextString = true,
-                        bool ShouldEmitNewLineOnDestruction = false)
-      : Prefix(std::move(Prefix)), Os(Os), BaseLevel(BaseLevel),
-        ShouldPrefixNextString(ShouldPrefixNextString),
-        ShouldEmitNewLineOnDestruction(ShouldEmitNewLineOnDestruction) {
-    SetUnbuffered();
-  }
-  ~odbg_ostream() final {
-    if (ShouldEmitNewLineOnDestruction)
-      Os << '\n';
-  }
-
-  /// Forward the current_pos method to the underlying stream.
-  uint64_t current_pos() const final { return Os.tell(); }
-
-  /// Some of the `<<` operators expect an lvalue, so we trick the type
-  /// system.
-  odbg_ostream &asLvalue() { return *this; }
-
-  void shouldMute(const IfLevel Filter) { Muted = Filter > BaseLevel; }
-  void shouldMute(const OnlyLevel Filter) { Muted = BaseLevel != Filter; }
-};
-
 /// Compute the prefix for the debug log in the form of:
 /// "Component --> "
 [[maybe_unused]] static std::string computePrefix(StringRef Component,
@@ -491,6 +460,8 @@ static inline raw_ostream &operator<<(raw_ostream &Os,
 
 #else
 
+inline bool isDebugEnabled() { return false; }
+
 #define ODBG_NULL                                                              \
   for (bool _c = false; _c; _c = false)                                        \
   ::llvm::nulls()
@@ -507,4 +478,94 @@ static inline raw_ostream &operator<<(raw_ostream &Os,
 
 } // namespace llvm::offload::debug
 
+namespace llvm::omp::target::debug {
+using namespace llvm::offload::debug;
+
+enum OmpDebugLevel : uint32_t {
+  ODL_Default = 1,
+  ODL_Error = ODL_Default,
+  ODL_Detailed = 2,
+  ODL_Verbose = 3,
+  ODL_VeryVerbose = 4,
+  ODL_Dumping = 5
+};
+
+/* Debug types to use in libomptarget */
+constexpr const char *ODT_Init = "Init";
+constexpr const char *ODT_Mapping = "Mapping";
+constexpr const char *ODT_Kernel = "Kernel";
+constexpr const char *ODT_DataTransfer = "DataTransfer";
+constexpr const char *ODT_Sync = "Sync";
+constexpr const char *ODT_Deinit = "Deinit";
+constexpr const char *ODT_Error = "Error";
+constexpr const char *ODT_KernelArgs = "KernelArgs";
+constexpr const char *ODT_MappingExists = "MappingExists";
+constexpr const char *ODT_DumpTable = "DumpTable";
+constexpr const char *ODT_MappingChanged = "MappingChanged";
+constexpr const char *ODT_PluginKernel = "PluginKernel";
+constexpr const char *ODT_EmptyMapping = "EmptyMapping";
+
+static inline odbg_ostream reportErrorStream() {
+#ifdef OMPTARGET_DEBUG
+  if (::llvm::offload::debug::isDebugEnabled()) {
+    uint32_t RealLevel = ODL_Error;
+    if (::llvm::offload::debug::shouldPrintDebug(GETNAME(TARGET_NAME),
+                                                 (ODT_Error), RealLevel))
+      return odbg_ostream{
+          ::llvm::offload::debug::computePrefix("DEBUG_PREFIX_FIXME:", ODT_Error),
+          ::llvm::offload::debug::dbgs(), RealLevel};
+    else
+      return odbg_ostream{"", ::llvm::nulls(), 1};
+  }
+#endif
+  return odbg_ostream{GETNAME(TARGET_NAME) " error: ",
+                      ::llvm::offload::debug::dbgs(), ODL_Error};
+}
+
+#ifdef OMPTARGET_DEBUG
+// Deprecated debug print macros
+[[maybe_unused]] static std::string formatToStr(const char *format, ...) {
+  va_list args;
+  va_start(args, format);
+  size_t len = std::vsnprintf(NULL, 0, format, args);
+  va_end(args);
+  llvm::SmallVector<char, 128> vec(len + 1);
+  va_start(args, format);
+  std::vsnprintf(&vec[0], len + 1, format, args);
+  va_end(args);
+  return &vec[0];
+}
+
+// helper macro to support old DP and REPORT macros with printf syntax
+#define FORMAT_TO_STR(Format, ...)                                             \
+  ::llvm::omp::target::debug::formatToStr(Format __VA_OPT__(, ) __VA_ARGS__)
+
+#define DP(...) ODBG() << FORMAT_TO_STR(__VA_ARGS__);
+
+#define REPORT_INT_OLD(...)                                                    \
+  do {                                                                         \
+    if (::llvm::offload::debug::isDebugEnabled()) {                            \
+      ODBG(::llvm::omp::target::debug::ODT_Error,                              \
+           ::llvm::omp::target::debug::ODL_Error)                              \
+          << FORMAT_TO_STR(__VA_ARGS__);                                       \
+    } else {                                                                   \
+      FAILURE_MESSAGE(__VA_ARGS__);                                            \
+    }                                                                          \
+  } while (false)
+
+#else
+#define DP(...)                                                                \
+  {                                                                            \
+  }
+#define REPORT_INT_OLD(...) FAILURE_MESSAGE(__VA_ARGS__);
+#endif // OMPTARGET_DEBUG
+
+// This is used for the new style REPORT macro
+#define REPORT_INT() ::llvm::omp::target::debug::reportErrorStream()
+
+// Make REPORT compatible with old and new syntax
+#define REPORT(...) REPORT_INT##__VA_OPT__(_OLD)(__VA_ARGS__)
+
+} // namespace llvm::omp::target::debug
+
 #endif // OMPTARGET_SHARED_DEBUG_H
diff --git a/offload/libomptarget/OffloadRTL.cpp b/offload/libomptarget/OffloadRTL.cpp
index 0ae325bf496d9..3a18d76aaae15 100644
--- a/offload/libomptarget/OffloadRTL.cpp
+++ b/offload/libomptarget/OffloadRTL.cpp
@@ -19,6 +19,7 @@
 #ifdef OMPT_SUPPORT
 extern void llvm::omp::target::ompt::connectLibrary();
 #endif
+using namespace llvm::omp::target::debug;
 
 static std::mutex PluginMtx;
 static uint32_t RefCount = 0;
@@ -35,7 +36,7 @@ void initRuntime() {
 
   RefCount++;
   if (RefCount == 1) {
-    ODBG() << "Init offload library!";
+    ODBG(ODT_Init) << "Init offload library!";
 #ifdef OMPT_SUPPORT
     // Initialize OMPT first
     llvm::omp::target::ompt::connectLibrary();
@@ -54,12 +55,12 @@ void deinitRuntime() {
   assert(PM && "Runtime not initialized");
 
   if (RefCount == 1) {
-    DP("Deinit offload library!\n");
+    ODBG(ODT_Deinit) << "Deinit offload library!";
     // RTL deinitialization has started
     RTLAlive = false;
     while (RTLOngoingSyncs > 0) {
-      DP("Waiting for ongoing syncs to finish, count: %d\n",
-         RTLOngoingSyncs.load());
+      ODBG(ODT_Sync) << "Waiting for ongoing syncs to finish, count:"
+                     << RTLOngoingSyncs.load();
       std::this_thread::sleep_for(std::chrono::milliseconds(100));
     }
     PM->deinit();
diff --git a/offload/libomptarget/device.cpp b/offload/libomptarget/device.cpp
index 18c6a8ef239de..f46442c9d0f18 100644
--- a/offload/libomptarget/device.cpp
+++ b/offload/libomptarget/device.cpp
@@ -41,6 +41,7 @@ using namespace ompt;
 #endif
 
 using namespace llvm::omp::target::plugin;
+using namespace llvm::omp::target::debug;
 
 int HostDataToTargetTy::addEventIfNecessary(DeviceTy &Device,
                                             AsyncInfoTy &AsyncInfo) const {
@@ -51,7 +52,7 @@ int HostDataToTargetTy::addEventIfNecessary(DeviceTy &Device,
   void *Event = getEvent();
   bool NeedNewEvent = Event == nullptr;
   if (NeedNewEvent && Device.createEvent(&Event) != OFFLOAD_SUCCESS) {
-    REPORT("Failed to create event\n");
+    REPORT() << "Failed to create event";
     return OFFLOAD_FAIL;
   }
 
@@ -59,7 +60,7 @@ int HostDataToTargetTy::addEventIfNecessary(DeviceTy &Device,
   // know if the target support event. But if a target doesn't,
   // recordEvent should always return success.
   if (Device.recordEvent(Event, AsyncInfo) != OFFLOAD_SUCCESS) {
-    REPORT("Failed to set dependence on event " DPxMOD "\n", DPxPTR(Event));
+    REPORT() << "Failed to set dependence on event " << Event;
     return OFFLOAD_FAIL;
   }
 
@@ -361,21 +362,21 @@ int32_t DeviceTy::dataFence(AsyncInfoTy &AsyncInfo) {
 }
 
 int32_t DeviceTy::notifyDataMapped(void *HstPtr, int64_t Size) {
-  DP("Notifying about new mapping: HstPtr=" DPxMOD ", Size=%" PRId64 "\n",
-     DPxPTR(HstPtr), Size);
+  ODBG(ODT_Mapping) << "Notifying about new mapping: HstPtr=" << HstPtr
+                    << ", Size=" << Size;
 
   if (RTL->data_notify_mapped(RTLDeviceID, HstPtr, Size)) {
-    REPORT("Notifying about data mapping failed.\n");
+    REPORT() << "Notifying about data mapping failed.";
     return OFFLOAD_FAIL;
   }
   return OFFLOAD_SUCCESS;
 }
 
 int32_t DeviceTy::notifyDataUnmapped(void *HstPtr) {
-  DP("Notifying about an unmapping: HstPtr=" DPxMOD "\n", DPxPTR(HstPtr));
+  ODBG(ODT_Mapping) << "Notifying about an unmapping: HstPtr=" << HstPtr;
 
   if (RTL->data_notify_unmapped(RTLDeviceID, HstPtr)) {
-    REPORT("Notifying about data unmapping failed.\n");
+    REPORT() << "Notifying about data unmapping failed.";
     return OFFLOAD_FAIL;
   }
   return OFFLOAD_SUCCESS;
diff --git a/revert_patches.txt b/revert_patches.txt
index f4ec0a3444c46..9e465ba90ae6a 100644
--- a/revert_patches.txt
+++ b/revert_patches.txt
@@ -5,6 +5,3 @@ d57230c7 [AMDGPU][MC] Disallow op_sel in some VOP3P dot instructions (#100485)
 breaks build of ROCmValidationSuite
 [C2y] Support WG14 N3457, the __COUNTER__ macro (#162662)
 ---
-breaks rocRAND
-[CUDA][HIP] Fix CTAD for host/device constructors (#168711) 
----
diff --git a/utils/bazel/llvm-project-overlay/lldb/source/Plugins/BUILD.bazel b/utils/bazel/llvm-project-overlay/lldb/source/Plugins/BUILD.bazel
index da39e58ac70ed..7dc16674b979f 100644
--- a/utils/bazel/llvm-project-overlay/lldb/source/Plugins/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/lldb/source/Plugins/BUILD.bazel
@@ -2091,6 +2091,7 @@ cc_library(
         "//lldb:Target",
         "//lldb:TargetHeaders",
         "//lldb:Utility",
+        "//llvm:Support",
     ],
 )
 
@@ -2142,11 +2143,14 @@ cc_library(
         ":PluginObjectFilePlaceholder",
         ":PluginProcessUtility",
         "//lldb:Core",
+        "//lldb:CoreHeaders",
         "//lldb:Host",
         "//lldb:InterpreterHeaders",
+        "//lldb:SymbolHeaders",
         "//lldb:Target",
         "//lldb:TargetHeaders",
         "//lldb:Utility",
+        "//llvm:Support",
     ],
 )
 
diff --git a/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel b/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel
index 1428299076fb3..8e9b51b58f4f5 100644
--- a/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel
@@ -1243,42 +1243,12 @@ cc_library(
     ],
 )
 
-AnalysisFpExcSrcs = [
-    "lib/Analysis/ConstantFolding.cpp",
-]
-
-cc_library(
-    name = "AnalysisFpExc",
-    srcs = AnalysisFpExcSrcs,
-    hdrs = glob(
-        [
-            "include/llvm/Analysis/*.h",
-            "include/llvm/Analysis/Utils/*.h",
-        ],
-    ),
-    copts = llvm_copts + ["-ftrapping-math"],
-    textual_hdrs = glob([
-        "include/llvm/Analysis/*.def",
-    ]),
-    deps = [
-        ":BinaryFormat",
-        ":Core",
-        ":Object",
-        ":ProfileData",
-        ":Support",
-        ":TargetParser",
-        ":config",
-        ":target_library_info_gen",
-    ],
-)
-
 cc_library(
     name = "Analysis",
     srcs = glob(
         [
             "lib/Analysis/*.cpp",
         ],
-        exclude = AnalysisFpExcSrcs,
     ),
     hdrs = glob(
         [
@@ -1288,12 +1258,11 @@ cc_library(
     ) + [
         "include/llvm-c/Analysis.h",
     ],
-    copts = llvm_copts,
+    copts = llvm_copts + ["-ftrapping-math"],
     textual_hdrs = glob([
         "include/llvm/Analysis/*.def",
     ]),
     deps = [
-        ":AnalysisFpExc",
         ":BinaryFormat",
         ":Core",
         ":FrontendHLSL",