diff --git a/src/coreclr/jit/hwintrinsiclistarm64sve.h b/src/coreclr/jit/hwintrinsiclistarm64sve.h index 75e9603c8946ea..696da224f33fb5 100644 --- a/src/coreclr/jit/hwintrinsiclistarm64sve.h +++ b/src/coreclr/jit/hwintrinsiclistarm64sve.h @@ -344,6 +344,8 @@ HARDWARE_INTRINSIC(Sve2, ConvertToDoubleOdd, HARDWARE_INTRINSIC(Sve2, ConvertToSingleEvenRoundToOdd, -1, -1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_fcvtx, INS_sve_fcvtx}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) HARDWARE_INTRINSIC(Sve2, ConvertToSingleOdd, -1, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_fcvtnt, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasRMWSemantics) HARDWARE_INTRINSIC(Sve2, ConvertToSingleOddRoundToOdd, -1, -1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_fcvtxnt, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve2, CountMatchingElements, -1, 3, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_histcnt, INS_sve_histcnt, INS_sve_histcnt, INS_sve_histcnt, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation) +HARDWARE_INTRINSIC(Sve2, CountMatchingElementsIn128BitSegments, -1, 2, {INS_sve_histseg, INS_sve_histseg, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg) HARDWARE_INTRINSIC(Sve2, DotProductRotateComplex, -1, 4, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_cdot, INS_invalid, INS_sve_cdot, INS_invalid, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasRMWSemantics|HW_Flag_SpecialCodeGen|HW_Flag_HasImmediateOperand) HARDWARE_INTRINSIC(Sve2, DotProductRotateComplexBySelectedIndex, -1, 5, {INS_sve_cdot, INS_invalid, INS_sve_cdot, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasRMWSemantics|HW_Flag_SpecialCodeGen|HW_Flag_HasImmediateOperand|HW_Flag_LowVectorOperation|HW_Flag_SpecialImport|HW_Flag_BaseTypeFromSecondArg) HARDWARE_INTRINSIC(Sve2, FusedAddHalving, -1, -1, {INS_sve_shadd, INS_sve_uhadd, INS_sve_shadd, INS_sve_uhadd, INS_sve_shadd, INS_sve_uhadd, INS_sve_shadd, INS_sve_uhadd, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation) diff --git a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve2.PlatformNotSupported.cs b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve2.PlatformNotSupported.cs index 9722d809c321c7..c087f5d870ef91 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve2.PlatformNotSupported.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve2.PlatformNotSupported.cs @@ -1216,6 +1216,48 @@ internal Arm64() { } /// public static Vector ConvertToSingleOddRoundToOdd(Vector even, Vector value) { throw new PlatformNotSupportedException(); } + + // Count matching elements in vector + + /// + /// svuint32_t svhistcnt[_s32]_z(svbool_t pg, svint32_t op1, svint32_t op2) + /// HISTCNT Zresult.S, Pg/Z, Zop1.S, Zop2.S + /// + public static Vector CountMatchingElements(Vector mask, Vector left, Vector right) { throw new PlatformNotSupportedException(); } + + /// + /// svuint64_t svhistcnt[_s64]_z(svbool_t pg, svint64_t op1, svint64_t op2) + /// HISTCNT Zresult.D, Pg/Z, Zop1.D, Zop2.D + /// + public static Vector CountMatchingElements(Vector mask, Vector left, Vector right) { throw new PlatformNotSupportedException(); } + + /// + /// svuint32_t svhistcnt[_u32]_z(svbool_t pg, svuint32_t op1, svuint32_t op2) + /// HISTCNT Zresult.S, Pg/Z, Zop1.S, Zop2.S + /// + public static Vector CountMatchingElements(Vector mask, Vector left, Vector right) { throw new PlatformNotSupportedException(); } + + /// + /// svuint64_t svhistcnt[_u64]_z(svbool_t pg, svuint64_t op1, svuint64_t op2) + /// HISTCNT Zresult.D, Pg/Z, Zop1.D, Zop2.D + /// + public static Vector CountMatchingElements(Vector mask, Vector left, Vector right) { throw new PlatformNotSupportedException(); } + + // Count matching elements in vector segments + + /// + /// svuint8_t svhistseg[_s8](svint8_t op1, svint8_t op2) + /// HISTSEG Zresult.B, Zop1.B, Zop2.B + /// + public static Vector CountMatchingElementsIn128BitSegments(Vector left, Vector right) { throw new PlatformNotSupportedException(); } + + /// + /// svuint8_t svhistseg[_u8](svuint8_t op1, svuint8_t op2) + /// HISTSEG Zresult.B, Zop1.B, Zop2.B + /// + public static Vector CountMatchingElementsIn128BitSegments(Vector left, Vector right) { throw new PlatformNotSupportedException(); } + + // Complex dot product /// diff --git a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve2.cs b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve2.cs index 904ce01689efbb..89dffec2c241d1 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve2.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve2.cs @@ -1216,6 +1216,48 @@ internal Arm64() { } /// public static Vector ConvertToSingleOddRoundToOdd(Vector even, Vector value) => ConvertToSingleOddRoundToOdd(even, value); + + // Count matching elements in vector + + /// + /// svuint32_t svhistcnt[_s32]_z(svbool_t pg, svint32_t op1, svint32_t op2) + /// HISTCNT Zresult.S, Pg/Z, Zop1.S, Zop2.S + /// + public static Vector CountMatchingElements(Vector mask, Vector left, Vector right) => CountMatchingElements(mask, left, right); + + /// + /// svuint64_t svhistcnt[_s64]_z(svbool_t pg, svint64_t op1, svint64_t op2) + /// HISTCNT Zresult.D, Pg/Z, Zop1.D, Zop2.D + /// + public static Vector CountMatchingElements(Vector mask, Vector left, Vector right) => CountMatchingElements(mask, left, right); + + /// + /// svuint32_t svhistcnt[_u32]_z(svbool_t pg, svuint32_t op1, svuint32_t op2) + /// HISTCNT Zresult.S, Pg/Z, Zop1.S, Zop2.S + /// + public static Vector CountMatchingElements(Vector mask, Vector left, Vector right) => CountMatchingElements(mask, left, right); + + /// + /// svuint64_t svhistcnt[_u64]_z(svbool_t pg, svuint64_t op1, svuint64_t op2) + /// HISTCNT Zresult.D, Pg/Z, Zop1.D, Zop2.D + /// + public static Vector CountMatchingElements(Vector mask, Vector left, Vector right) => CountMatchingElements(mask, left, right); + + // Count matching elements in vector segments + + /// + /// svuint8_t svhistseg[_s8](svint8_t op1, svint8_t op2) + /// HISTSEG Zresult.B, Zop1.B, Zop2.B + /// + public static Vector CountMatchingElementsIn128BitSegments(Vector left, Vector right) => CountMatchingElementsIn128BitSegments(left, right); + + /// + /// svuint8_t svhistseg[_u8](svuint8_t op1, svuint8_t op2) + /// HISTSEG Zresult.B, Zop1.B, Zop2.B + /// + public static Vector CountMatchingElementsIn128BitSegments(Vector left, Vector right) => CountMatchingElementsIn128BitSegments(left, right); + + // Complex dot product /// diff --git a/src/libraries/System.Runtime.Intrinsics/ref/System.Runtime.Intrinsics.cs b/src/libraries/System.Runtime.Intrinsics/ref/System.Runtime.Intrinsics.cs index 479501776f2a19..8905d76d65c6d6 100644 --- a/src/libraries/System.Runtime.Intrinsics/ref/System.Runtime.Intrinsics.cs +++ b/src/libraries/System.Runtime.Intrinsics/ref/System.Runtime.Intrinsics.cs @@ -6356,6 +6356,12 @@ internal Arm64() { } public static System.Numerics.Vector ConvertToSingleEvenRoundToOdd(System.Numerics.Vector value) { throw null; } public static System.Numerics.Vector ConvertToSingleOdd(System.Numerics.Vector even, System.Numerics.Vector value) { throw null; } public static System.Numerics.Vector ConvertToSingleOddRoundToOdd(System.Numerics.Vector even, System.Numerics.Vector value) { throw null; } + public static System.Numerics.Vector CountMatchingElements(System.Numerics.Vector mask, System.Numerics.Vector left, System.Numerics.Vector right) { throw null; } + public static System.Numerics.Vector CountMatchingElements(System.Numerics.Vector mask, System.Numerics.Vector left, System.Numerics.Vector right) { throw null; } + public static System.Numerics.Vector CountMatchingElements(System.Numerics.Vector mask, System.Numerics.Vector left, System.Numerics.Vector right) { throw null; } + public static System.Numerics.Vector CountMatchingElements(System.Numerics.Vector mask, System.Numerics.Vector left, System.Numerics.Vector right) { throw null; } + public static System.Numerics.Vector CountMatchingElementsIn128BitSegments(System.Numerics.Vector left, System.Numerics.Vector right) { throw null; } + public static System.Numerics.Vector CountMatchingElementsIn128BitSegments(System.Numerics.Vector left, System.Numerics.Vector right) { throw null; } public static System.Numerics.Vector DotProductRotateComplex(System.Numerics.Vector addend, System.Numerics.Vector left, System.Numerics.Vector right, [ConstantExpected(Min = 0, Max = (byte)(3))] byte rotation) { throw null; } public static System.Numerics.Vector DotProductRotateComplex(System.Numerics.Vector addend, System.Numerics.Vector left, System.Numerics.Vector right, [ConstantExpected(Min = 0, Max = (byte)(3))] byte rotation) { throw null; } public static System.Numerics.Vector DotProductRotateComplexBySelectedIndex(System.Numerics.Vector addend, System.Numerics.Vector left, System.Numerics.Vector right, [ConstantExpected(Min = 0, Max = (byte)(3))] byte rightIndex, [ConstantExpected(Min = 0, Max = (byte)(3))] byte rotation) { throw null; } diff --git a/src/tests/Common/GenerateHWIntrinsicTests/Arm/Sve2Tests.cs b/src/tests/Common/GenerateHWIntrinsicTests/Arm/Sve2Tests.cs index ff34ba5f7229e2..82c5ba278cd1a2 100644 --- a/src/tests/Common/GenerateHWIntrinsicTests/Arm/Sve2Tests.cs +++ b/src/tests/Common/GenerateHWIntrinsicTests/Arm/Sve2Tests.cs @@ -228,6 +228,14 @@ public static (string templateFileName, Dictionary templateData) ("SveVecBinOpDiffRetTypeFloats.template", new Dictionary { ["TestName"] = "Sve2_ConvertToSingleOdd_float_double", ["Isa"] = "Sve2", ["LoadIsa"] = "Sve2", ["Method"] = "ConvertToSingleOdd", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Single", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Single", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "Double", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetSingle()", ["NextValueOp2"] = "TestLibrary.Generator.GetDouble()", ["ConvertFunc"] = "BitConverter.SingleToInt32Bits", ["ValidateIterResult"] = "Helpers.ConvertToSingleOdd(left, right, i) != result[i]", ["GetIterResult"] = "Helpers.ConvertToSingleOdd(left, right, i)"}), ("SveVecBinOpDiffRetTypeFloats.template", new Dictionary { ["TestName"] = "Sve2_ConvertToSingleOddRoundToOdd_float_double", ["Isa"] = "Sve2", ["LoadIsa"] = "Sve2", ["Method"] = "ConvertToSingleOddRoundToOdd", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Single", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Single", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "Double", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetSingle()", ["NextValueOp2"] = "TestLibrary.Generator.GetDouble()", ["ConvertFunc"] = "BitConverter.SingleToInt32Bits", ["ValidateIterResult"] = "Helpers.ConvertToSingleOddRoundToOdd(left, right, i) != result[i]", ["GetIterResult"] = "Helpers.ConvertToSingleOddRoundToOdd(left, right, i)"}), + ("SveVecTernOpMaskedTest.template", new Dictionary { ["TestName"] = "Sve2_CountMatchingElements_int", ["Isa"] = "Sve2", ["LoadIsa"] = "Sve2", ["Method"] = "CountMatchingElements", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt32", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int32", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "Int32", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetInt32()", ["NextValueOp2"] = "TestLibrary.Generator.GetInt32()", ["ConvertFunc"] = "", ["ValidateIterResult"] = "Helpers.CountMatchingElements(maskArray, first, second, i) != result[i]", ["GetIterResult"] = "Helpers.CountMatchingElements(mask, left, right, i)"}), + ("SveVecTernOpMaskedTest.template", new Dictionary { ["TestName"] = "Sve2_CountMatchingElements_long", ["Isa"] = "Sve2", ["LoadIsa"] = "Sve2", ["Method"] = "CountMatchingElements", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt64", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int64", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "Int64", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetInt64()", ["NextValueOp2"] = "TestLibrary.Generator.GetInt64()", ["ConvertFunc"] = "", ["ValidateIterResult"] = "Helpers.CountMatchingElements(maskArray, first, second, i) != result[i]", ["GetIterResult"] = "Helpers.CountMatchingElements(mask, left, right, i)"}), + ("SveVecTernOpMaskedTest.template", new Dictionary { ["TestName"] = "Sve2_CountMatchingElements_uint", ["Isa"] = "Sve2", ["LoadIsa"] = "Sve2", ["Method"] = "CountMatchingElements", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt32", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt32", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "UInt32", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetUInt32()", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt32()", ["ConvertFunc"] = "", ["ValidateIterResult"] = "Helpers.CountMatchingElements(maskArray, first, second, i) != result[i]", ["GetIterResult"] = "Helpers.CountMatchingElements(mask, left, right, i)"}), + ("SveVecTernOpMaskedTest.template", new Dictionary { ["TestName"] = "Sve2_CountMatchingElements_ulong", ["Isa"] = "Sve2", ["LoadIsa"] = "Sve2", ["Method"] = "CountMatchingElements", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt64", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt64", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "UInt64", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetUInt64()", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt64()", ["ConvertFunc"] = "", ["ValidateIterResult"] = "Helpers.CountMatchingElements(maskArray, first, second, i) != result[i]", ["GetIterResult"] = "Helpers.CountMatchingElements(mask, left, right, i)"}), + + ("SveVecBinOpDifferentRetType.template", new Dictionary { ["TestName"] = "Sve2_CountMatchingElementsIn128BitSegments_sbyte", ["Isa"] = "Sve2", ["LoadIsa"] = "Sve2", ["Method"] = "CountMatchingElementsIn128BitSegments", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Byte", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "SByte", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "SByte", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetSByte()", ["NextValueOp2"] = "TestLibrary.Generator.GetSByte()", ["ConvertFunc"] = "", ["ValidateIterResult"] = "Helpers.CountMatchingElementsIn128BitSegments(left, right, i) != result[i]", ["GetIterResult"] = "Helpers.CountMatchingElementsIn128BitSegments(left, right, i)"}), + ("SveVecBinOpDifferentRetType.template", new Dictionary { ["TestName"] = "Sve2_CountMatchingElementsIn128BitSegments_byte", ["Isa"] = "Sve2", ["LoadIsa"] = "Sve2", ["Method"] = "CountMatchingElementsIn128BitSegments", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Byte", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Byte", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "Byte", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetByte()", ["NextValueOp2"] = "TestLibrary.Generator.GetByte()", ["ConvertFunc"] = "", ["ValidateIterResult"] = "Helpers.CountMatchingElementsIn128BitSegments(left, right, i) != result[i]", ["GetIterResult"] = "Helpers.CountMatchingElementsIn128BitSegments(left, right, i)"}), + ("SveVecImmTernOpFirstArgTest.template", new Dictionary { ["TestName"] = "Sve2_DotProductRotateComplex_int_sbyte_0", ["Isa"] = "Sve2", ["LoadIsa"] = "Sve2", ["Method"] = "DotProductRotateComplex", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int32", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int32", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "SByte", ["Op3VectorType"] = "Vector", ["Op3BaseType"] = "SByte", ["Op4BaseType"] = "Byte", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetInt32()", ["NextValueOp2"] = "TestLibrary.Generator.GetSByte()", ["NextValueOp3"] = "TestLibrary.Generator.GetSByte()", ["Imm"] = "0", ["InvalidImm"] = "4", ["ConvertFunc"] = "", ["ValidateIterResult"] = "Helpers.DotProductRotateComplex(first[i], second, 4 * i, third, Imm) != result[i]", ["GetIterResult"] = "Helpers.DotProductRotateComplex(first[i], second, 4 * i, third, Imm)"}), ("SveVecImmTernOpFirstArgTest.template", new Dictionary { ["TestName"] = "Sve2_DotProductRotateComplex_int_sbyte_1", ["Isa"] = "Sve2", ["LoadIsa"] = "Sve2", ["Method"] = "DotProductRotateComplex", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int32", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int32", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "SByte", ["Op3VectorType"] = "Vector", ["Op3BaseType"] = "SByte", ["Op4BaseType"] = "Byte", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetInt32()", ["NextValueOp2"] = "TestLibrary.Generator.GetSByte()", ["NextValueOp3"] = "TestLibrary.Generator.GetSByte()", ["Imm"] = "1", ["InvalidImm"] = "4", ["ConvertFunc"] = "", ["ValidateIterResult"] = "Helpers.DotProductRotateComplex(first[i], second, 4 * i, third, Imm) != result[i]", ["GetIterResult"] = "Helpers.DotProductRotateComplex(first[i], second, 4 * i, third, Imm)"}), ("SveVecImmTernOpFirstArgTest.template", new Dictionary { ["TestName"] = "Sve2_DotProductRotateComplex_int_sbyte_2", ["Isa"] = "Sve2", ["LoadIsa"] = "Sve2", ["Method"] = "DotProductRotateComplex", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int32", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int32", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "SByte", ["Op3VectorType"] = "Vector", ["Op3BaseType"] = "SByte", ["Op4BaseType"] = "Byte", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetInt32()", ["NextValueOp2"] = "TestLibrary.Generator.GetSByte()", ["NextValueOp3"] = "TestLibrary.Generator.GetSByte()", ["Imm"] = "2", ["InvalidImm"] = "4", ["ConvertFunc"] = "", ["ValidateIterResult"] = "Helpers.DotProductRotateComplex(first[i], second, 4 * i, third, Imm) != result[i]", ["GetIterResult"] = "Helpers.DotProductRotateComplex(first[i], second, 4 * i, third, Imm)"}), diff --git a/src/tests/JIT/HardwareIntrinsics/Arm/Shared/Helpers.cs b/src/tests/JIT/HardwareIntrinsics/Arm/Shared/Helpers.cs index d8f9646f54e44b..c8c44047b9fa82 100644 --- a/src/tests/JIT/HardwareIntrinsics/Arm/Shared/Helpers.cs +++ b/src/tests/JIT/HardwareIntrinsics/Arm/Shared/Helpers.cs @@ -108,6 +108,42 @@ public static unsafe T CountLeadingZeroBits(T value) return T.CreateChecked(bitSize - (highest + 1)); } + public static U CountMatchingElements(T[] mask, T[] left, T[] right, int i) + where U : unmanaged, INumber + where T : unmanaged, INumber + { + int result = 0; + + if (mask[i] != T.Zero) + { + for (int j = 0; j <= i; j++) + { + if (mask[j] != T.Zero && left[i] == right[j]) + { + result++; + } + } + } + + return U.CreateChecked(result); + } + + public static byte CountMatchingElementsIn128BitSegments(T[] left, T[] right, int i) + where T : unmanaged, INumber + { + int result = 0; + + for (int j = 0; j < right.Length; j++) + { + if (left[i] == right[j]) + { + result++; + } + } + + return byte.CreateChecked(result); + } + public static unsafe int HighestSetBit(T value) where T : unmanaged, INumber, IBitwiseOperators { diff --git a/src/tests/JIT/HardwareIntrinsics/Arm/Shared/_SveTernOpMaskedOpTestTemplate.template b/src/tests/JIT/HardwareIntrinsics/Arm/Shared/_SveTernOpMaskedOpTestTemplate.template index 06cce1bdfef0d6..e4f7fbe06ee1d6 100644 --- a/src/tests/JIT/HardwareIntrinsics/Arm/Shared/_SveTernOpMaskedOpTestTemplate.template +++ b/src/tests/JIT/HardwareIntrinsics/Arm/Shared/_SveTernOpMaskedOpTestTemplate.template @@ -116,7 +116,7 @@ namespace JIT.HardwareIntrinsics.Arm._{Isa} Unsafe.CopyBlockUnaligned(ref Unsafe.AsRef(inArray1Ptr), ref Unsafe.As<{Op1BaseType}, byte>(ref inArray1[0]), (uint)sizeOfinArray1); Unsafe.CopyBlockUnaligned(ref Unsafe.AsRef(inArray2Ptr), ref Unsafe.As<{Op2BaseType}, byte>(ref inArray2[0]), (uint)sizeOfinArray2); - Unsafe.CopyBlockUnaligned(ref Unsafe.AsRef(maskArrayPtr), ref Unsafe.As<{RetBaseType}, byte>(ref maskArray[0]), (uint)sizeOfmaskArray); + Unsafe.CopyBlockUnaligned(ref Unsafe.AsRef(maskArrayPtr), ref Unsafe.As<{Op1BaseType}, byte>(ref maskArray[0]), (uint)sizeOfmaskArray); } public void* inArray1Ptr => Align((byte*)(inHandle1.AddrOfPinnedObject().ToPointer()), alignment); @@ -182,22 +182,22 @@ namespace JIT.HardwareIntrinsics.Arm._{Isa} public void ConditionalSelect_Op1() { TestLibrary.TestFramework.BeginScenario("ConditionalSelect_Op1_mask - operation in TrueValue"); - ConditionalSelectScenario_TrueValue(_mask, _fld1, _fld2, _fld2); + ConditionalSelectScenario_TrueValue(_mask, _fld1, _fld2, _falseFld); TestLibrary.TestFramework.BeginScenario("ConditionalSelect_Op1_zero - operation in TrueValue"); - ConditionalSelectScenario_TrueValue(Vector<{Op1BaseType}>.Zero, _fld1, _fld2, _fld2); + ConditionalSelectScenario_TrueValue(Vector<{Op1BaseType}>.Zero, _fld1, _fld2, _falseFld); TestLibrary.TestFramework.BeginScenario("ConditionalSelect_Op1_all - operation in TrueValue"); - ConditionalSelectScenario_TrueValue(Vector<{Op1BaseType}>.AllBitsSet, _fld1, _fld2, _fld2); + ConditionalSelectScenario_TrueValue(Vector<{Op1BaseType}>.AllBitsSet, _fld1, _fld2, _falseFld); TestLibrary.TestFramework.BeginScenario("ConditionalSelect_Op1_mask - operation in FalseValue"); - ConditionalSelectScenario_FalseValue(_mask, _fld1, _fld2, _fld2); + ConditionalSelectScenario_FalseValue(_mask, _fld1, _fld2, _falseFld); TestLibrary.TestFramework.BeginScenario("ConditionalSelect_Op1_zero - operation in FalseValue"); - ConditionalSelectScenario_FalseValue(Vector<{Op1BaseType}>.Zero, _fld1, _fld2, _fld2); + ConditionalSelectScenario_FalseValue(Vector<{Op1BaseType}>.Zero, _fld1, _fld2, _falseFld); TestLibrary.TestFramework.BeginScenario("ConditionalSelect_Op1_all - operation in FalseValue"); - ConditionalSelectScenario_FalseValue(Vector<{Op1BaseType}>.AllBitsSet, _fld1, _fld2, _fld2); + ConditionalSelectScenario_FalseValue(Vector<{Op1BaseType}>.AllBitsSet, _fld1, _fld2, _falseFld); } public void ConditionalSelect_FalseOp() @@ -224,37 +224,37 @@ namespace JIT.HardwareIntrinsics.Arm._{Isa} public void ConditionalSelect_ZeroOp() { TestLibrary.TestFramework.BeginScenario("ConditionalSelect_ZeroOp_mask - operation in TrueValue"); - ConditionalSelectScenario_TrueValue(_mask, _fld1, _fld2, Vector<{Op1BaseType}>.Zero); + ConditionalSelectScenario_TrueValue(_mask, _fld1, _fld2, Vector<{RetBaseType}>.Zero); TestLibrary.TestFramework.BeginScenario("ConditionalSelect_ZeroOp_zero - operation in TrueValue"); - ConditionalSelectScenario_TrueValue(Vector<{Op1BaseType}>.Zero, _fld1, _fld2, Vector<{Op1BaseType}>.Zero); + ConditionalSelectScenario_TrueValue(Vector<{Op1BaseType}>.Zero, _fld1, _fld2, Vector<{RetBaseType}>.Zero); TestLibrary.TestFramework.BeginScenario("ConditionalSelect_ZeroOp_all - operation in TrueValue"); - ConditionalSelectScenario_TrueValue(Vector<{Op1BaseType}>.AllBitsSet, _fld1, _fld2, Vector<{Op1BaseType}>.Zero); + ConditionalSelectScenario_TrueValue(Vector<{Op1BaseType}>.AllBitsSet, _fld1, _fld2, Vector<{RetBaseType}>.Zero); TestLibrary.TestFramework.BeginScenario("ConditionalSelect_ZeroOp_mask - operation in FalseValue"); - ConditionalSelectScenario_FalseValue(_mask, _fld1, _fld2, Vector<{Op1BaseType}>.Zero); + ConditionalSelectScenario_FalseValue(_mask, _fld1, _fld2, Vector<{RetBaseType}>.Zero); TestLibrary.TestFramework.BeginScenario("ConditionalSelect_ZeroOp_zero - operation in FalseValue"); - ConditionalSelectScenario_FalseValue(Vector<{Op1BaseType}>.Zero, _fld1, _fld2, Vector<{Op1BaseType}>.Zero); + ConditionalSelectScenario_FalseValue(Vector<{Op1BaseType}>.Zero, _fld1, _fld2, Vector<{RetBaseType}>.Zero); TestLibrary.TestFramework.BeginScenario("ConditionalSelect_ZeroOp_all - operation in FalseValue"); - ConditionalSelectScenario_FalseValue(Vector<{Op1BaseType}>.AllBitsSet, _fld1, _fld2, Vector<{Op1BaseType}>.Zero); + ConditionalSelectScenario_FalseValue(Vector<{Op1BaseType}>.AllBitsSet, _fld1, _fld2, Vector<{RetBaseType}>.Zero); } [method: MethodImpl(MethodImplOptions.AggressiveInlining)] - private void ConditionalSelectScenario_TrueValue(Vector<{Op1BaseType}> mask, Vector<{Op1BaseType}> op1, Vector<{Op2BaseType}> op2, Vector<{Op1BaseType}> falseOp) + private void ConditionalSelectScenario_TrueValue(Vector<{Op1BaseType}> mask, Vector<{Op1BaseType}> op1, Vector<{Op2BaseType}> op2, Vector<{RetBaseType}> falseOp) { - var result = Sve.ConditionalSelect(mask, {Isa}.{Method}(mask, op1, op2), falseOp); + var result = Sve.ConditionalSelect((Vector<{RetBaseType}>)mask, {Isa}.{Method}(mask, op1, op2), falseOp); Unsafe.Write(_dataTable.outArrayPtr, result); ValidateConditionalSelectResult_TrueValue(mask, op1, op2, falseOp, _dataTable.outArrayPtr); } [method: MethodImpl(MethodImplOptions.AggressiveInlining)] - private void ConditionalSelectScenario_FalseValue(Vector<{Op1BaseType}> mask, Vector<{Op1BaseType}> op1, Vector<{Op2BaseType}> op2, Vector<{Op1BaseType}> trueOp) + private void ConditionalSelectScenario_FalseValue(Vector<{Op1BaseType}> mask, Vector<{Op1BaseType}> op1, Vector<{Op2BaseType}> op2, Vector<{RetBaseType}> trueOp) { - var result = Sve.ConditionalSelect(mask, trueOp, {Isa}.{Method}(mask, op1, op2)); + var result = Sve.ConditionalSelect((Vector<{RetBaseType}>)mask, trueOp, {Isa}.{Method}(mask, op1, op2)); Unsafe.Write(_dataTable.outArrayPtr, result); ValidateConditionalSelectResult_FalseValue(mask, op1, op2, trueOp, _dataTable.outArrayPtr); @@ -274,7 +274,7 @@ namespace JIT.HardwareIntrinsics.Arm._{Isa} private Vector<{Op1BaseType}> _fld1; private Vector<{Op2BaseType}> _fld2; private Vector<{Op1BaseType}> _mask; - private Vector<{Op1BaseType}> _falseFld; + private Vector<{RetBaseType}> _falseFld; private DataTable _dataTable; @@ -288,7 +288,7 @@ namespace JIT.HardwareIntrinsics.Arm._{Isa} Unsafe.CopyBlockUnaligned(ref Unsafe.As, byte>(ref _fld1), ref Unsafe.As<{Op1BaseType}, byte>(ref _data1[0]), (uint)Unsafe.SizeOf>()); Unsafe.CopyBlockUnaligned(ref Unsafe.As, byte>(ref _fld2), ref Unsafe.As<{Op2BaseType}, byte>(ref _data2[0]), (uint)Unsafe.SizeOf>()); Unsafe.CopyBlockUnaligned(ref Unsafe.As, byte>(ref _mask), ref Unsafe.As<{Op1BaseType}, byte>(ref _maskData[0]), (uint)Unsafe.SizeOf>()); - Unsafe.CopyBlockUnaligned(ref Unsafe.As, byte>(ref _falseFld), ref Unsafe.As<{Op1BaseType}, byte>(ref _data1[0]), (uint)Unsafe.SizeOf>()); + Unsafe.CopyBlockUnaligned(ref Unsafe.As, byte>(ref _falseFld), ref Unsafe.As<{Op1BaseType}, byte>(ref _data1[0]), (uint)Unsafe.SizeOf>()); _dataTable = new DataTable(_data1, _data2, new {RetBaseType}[RetElementCount], _maskData, LargestVectorSize); } @@ -339,7 +339,7 @@ namespace JIT.HardwareIntrinsics.Arm._{Isa} Vector<{Op1BaseType}> op1 = Unsafe.Read>(_dataTable.inArray1Ptr); Vector<{Op2BaseType}> op2 = Unsafe.Read>(_dataTable.inArray2Ptr); - Vector<{RetBaseType}> storeMask = Unsafe.Read>(_dataTable.maskArrayPtr); + Vector<{Op1BaseType}> storeMask = Unsafe.Read>(_dataTable.maskArrayPtr); var result = {Isa}.{Method}(storeMask, op1, op2); @@ -352,7 +352,7 @@ namespace JIT.HardwareIntrinsics.Arm._{Isa} Vector<{Op1BaseType}> op1 = Unsafe.Read>(_dataTable.inArray1Ptr); Vector<{Op2BaseType}> op2 = Unsafe.Read>(_dataTable.inArray2Ptr); - Vector<{RetBaseType}> storeMask = Unsafe.Read>(_dataTable.maskArrayPtr); + Vector<{Op1BaseType}> storeMask = Unsafe.Read>(_dataTable.maskArrayPtr); var result = {Isa}.{Method}(storeMask, op1, op2); @@ -398,19 +398,19 @@ namespace JIT.HardwareIntrinsics.Arm._{Isa} } } - private void ValidateConditionalSelectResult_TrueValue(Vector<{Op1BaseType}> maskOp, Vector<{Op1BaseType}> leftOp, Vector<{Op2BaseType}> rightOp, Vector<{Op1BaseType}> falseOp, void* output, [CallerMemberName] string method = "") + private void ValidateConditionalSelectResult_TrueValue(Vector<{Op1BaseType}> maskOp, Vector<{Op1BaseType}> leftOp, Vector<{Op2BaseType}> rightOp, Vector<{RetBaseType}> falseOp, void* output, [CallerMemberName] string method = "") { {Op1BaseType}[] mask = new {Op1BaseType}[Op1ElementCount]; {Op1BaseType}[] left = new {Op1BaseType}[Op1ElementCount]; {Op2BaseType}[] right = new {Op2BaseType}[Op2ElementCount]; - {Op1BaseType}[] falseVal = new {Op1BaseType}[Op1ElementCount]; + {RetBaseType}[] falseVal = new {RetBaseType}[RetElementCount]; {RetBaseType}[] result = new {RetBaseType}[RetElementCount]; Unsafe.WriteUnaligned(ref Unsafe.As<{Op1BaseType}, byte>(ref mask[0]), maskOp); Unsafe.WriteUnaligned(ref Unsafe.As<{Op1BaseType}, byte>(ref left[0]), leftOp); Unsafe.WriteUnaligned(ref Unsafe.As<{Op2BaseType}, byte>(ref right[0]), rightOp); - Unsafe.WriteUnaligned(ref Unsafe.As<{Op1BaseType}, byte>(ref falseVal[0]), falseOp); - Unsafe.CopyBlockUnaligned(ref Unsafe.As<{Op1BaseType}, byte>(ref result[0]), ref Unsafe.AsRef(output), (uint)Unsafe.SizeOf>()); + Unsafe.WriteUnaligned(ref Unsafe.As<{RetBaseType}, byte>(ref falseVal[0]), falseOp); + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{RetBaseType}, byte>(ref result[0]), ref Unsafe.AsRef(output), (uint)Unsafe.SizeOf>()); bool succeeded = true; @@ -430,19 +430,19 @@ namespace JIT.HardwareIntrinsics.Arm._{Isa} } } - private void ValidateConditionalSelectResult_FalseValue(Vector<{Op1BaseType}> maskOp, Vector<{Op1BaseType}> leftOp, Vector<{Op2BaseType}> rightOp, Vector<{Op1BaseType}> trueOp, void* output, [CallerMemberName] string method = "") + private void ValidateConditionalSelectResult_FalseValue(Vector<{Op1BaseType}> maskOp, Vector<{Op1BaseType}> leftOp, Vector<{Op2BaseType}> rightOp, Vector<{RetBaseType}> trueOp, void* output, [CallerMemberName] string method = "") { {Op1BaseType}[] mask = new {Op1BaseType}[Op1ElementCount]; {Op1BaseType}[] left = new {Op1BaseType}[Op1ElementCount]; {Op2BaseType}[] right = new {Op2BaseType}[Op2ElementCount]; - {Op1BaseType}[] trueVal = new {Op1BaseType}[Op1ElementCount]; + {RetBaseType}[] trueVal = new {RetBaseType}[RetElementCount]; {RetBaseType}[] result = new {RetBaseType}[RetElementCount]; Unsafe.WriteUnaligned(ref Unsafe.As<{Op1BaseType}, byte>(ref mask[0]), maskOp); Unsafe.WriteUnaligned(ref Unsafe.As<{Op1BaseType}, byte>(ref left[0]), leftOp); Unsafe.WriteUnaligned(ref Unsafe.As<{Op2BaseType}, byte>(ref right[0]), rightOp); - Unsafe.WriteUnaligned(ref Unsafe.As<{Op1BaseType}, byte>(ref trueVal[0]), trueOp); - Unsafe.CopyBlockUnaligned(ref Unsafe.As<{Op1BaseType}, byte>(ref result[0]), ref Unsafe.AsRef(output), (uint)Unsafe.SizeOf>()); + Unsafe.WriteUnaligned(ref Unsafe.As<{RetBaseType}, byte>(ref trueVal[0]), trueOp); + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{RetBaseType}, byte>(ref result[0]), ref Unsafe.AsRef(output), (uint)Unsafe.SizeOf>()); bool succeeded = true;