diff --git a/src/libraries/System.Private.CoreLib/src/System/Collections/BitArray.cs b/src/libraries/System.Private.CoreLib/src/System/Collections/BitArray.cs index 66ffc9fee7bcd7..c065b685fc09b4 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Collections/BitArray.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Collections/BitArray.cs @@ -165,6 +165,7 @@ public BitArray(ReadOnlySpan bytes) _array = CreateArray(bytes, out _bitLength); } + [MethodImpl(MethodImplOptions.AggressiveInlining)] private static byte[] CreateArray(ReadOnlySpan bytes, out int bitLength) { if (bytes.Length > int.MaxValue / BitsPerByte) @@ -215,64 +216,95 @@ private static byte[] CreateArray(ReadOnlySpan values, out int bitLength) uint i = 0; - if (!BitConverter.IsLittleEndian || values.Length < Vector256.Count) + if (BitConverter.IsLittleEndian && values.Length >= sizeof(ulong)) { - goto Remainder; - } - - // Comparing with 1s would get rid of the final negation, however this would not work for some CLR bools - // (true for any non-zero values, false for 0) - any values between 2-255 will be interpreted as false. - // Instead, we compare with zeroes (== false) then negate the result to ensure compatibility. + // Comparing with 1s would get rid of the final negation, however this would not work for some CLR bools + // (true for any non-zero values, false for 0) - any values between 2-255 will be interpreted as false. + // Instead, we compare with zeroes (== false) then negate the result to ensure compatibility. + // + // Every step below consumes a fixed number of bools and emits the matching bit mask into the + // destination. Both spans are tested against a constant length and then sliced by that same + // constant so that the JIT can elide all of the bounds checks. The destination test can never fail + // before the source one does: 'array' always holds at least one bit per bool in 'values'. + ReadOnlySpan source = MemoryMarshal.AsBytes(values); + Span destination = array; + + if (Vector512.IsHardwareAccelerated) + { + while (source.Length >= Vector512.Count && destination.Length >= sizeof(ulong)) + { + ulong isFalse = Vector512.Equals(Vector512.Create(source), Vector512.Zero).ExtractMostSignificantBits(); + BinaryPrimitives.WriteUInt64LittleEndian(destination, ~isFalse); - ref byte arrayRef = ref MemoryMarshal.GetArrayDataReference(array); - ReadOnlySpan valuesAsBytes = MemoryMarshal.AsBytes(values); - if (Vector512.IsHardwareAccelerated) - { - while (valuesAsBytes.Length >= Vector512.Count) + source = source.Slice(Vector512.Count); + destination = destination.Slice(sizeof(ulong)); + } + } + else if (Vector256.IsHardwareAccelerated) { - Vector512 vector = Vector512.Create(valuesAsBytes); - Vector512 isFalse = Vector512.Equals(vector, Vector512.Zero); + while (source.Length >= Vector256.Count && destination.Length >= sizeof(uint)) + { + uint isFalse = Vector256.Equals(Vector256.Create(source), Vector256.Zero).ExtractMostSignificantBits(); + BinaryPrimitives.WriteUInt32LittleEndian(destination, ~isFalse); - ulong result = isFalse.ExtractMostSignificantBits(); - Unsafe.WriteUnaligned(ref Unsafe.Add(ref arrayRef, sizeof(ulong) * (i / 64u)), ~result); - i += (uint)Vector512.Count; - valuesAsBytes = valuesAsBytes.Slice(Vector512.Count); + source = source.Slice(Vector256.Count); + destination = destination.Slice(sizeof(uint)); + } } - } - else if (Vector256.IsHardwareAccelerated) - { - while (valuesAsBytes.Length >= Vector256.Count) + else if (Vector128.IsHardwareAccelerated) { - Vector256 vector = Vector256.Create(valuesAsBytes); - Vector256 isFalse = Vector256.Equals(vector, Vector256.Zero); + while (source.Length >= Vector128.Count * 2 && destination.Length >= sizeof(uint)) + { + uint lowerIsFalse = Vector128.Equals(Vector128.Create(source), Vector128.Zero).ExtractMostSignificantBits(); + uint upperIsFalse = Vector128.Equals(Vector128.Create(source.Slice(Vector128.Count)), Vector128.Zero).ExtractMostSignificantBits(); + BinaryPrimitives.WriteUInt32LittleEndian(destination, ~((upperIsFalse << 16) | lowerIsFalse)); - uint result = isFalse.ExtractMostSignificantBits(); - Unsafe.WriteUnaligned(ref Unsafe.Add(ref arrayRef, sizeof(uint) * (i / 32u)), ~result); - i += (uint)Vector256.Count; - valuesAsBytes = valuesAsBytes.Slice(Vector256.Count); + source = source.Slice(Vector128.Count * 2); + destination = destination.Slice(sizeof(uint)); + } } - } - else if (Vector128.IsHardwareAccelerated) - { - while (valuesAsBytes.Length >= Vector128.Count * 2) + + // Too few bools are left for another iteration of the loop above. Drain them with progressively + // narrower steps, each of which runs at most once, so that the scalar loop below handles at most + // seven bools. The whole block is skipped when the loop above consumed everything. + if (source.Length >= sizeof(ulong)) { - Vector128 lowerVector = Vector128.Create(valuesAsBytes); - Vector128 lowerIsFalse = Vector128.Equals(lowerVector, Vector128.Zero); - uint lowerResult = lowerIsFalse.ExtractMostSignificantBits(); - - Vector128 upperVector = Vector128.Create(valuesAsBytes.Slice(Vector128.Count)); - Vector128 upperIsFalse = Vector128.Equals(upperVector, Vector128.Zero); - uint upperResult = upperIsFalse.ExtractMostSignificantBits(); - - Unsafe.WriteUnaligned( - ref Unsafe.Add(ref arrayRef, sizeof(uint) * (i / 32u)), - ~((upperResult << 16) | lowerResult)); - i += (uint)Vector128.Count * 2u; - valuesAsBytes = valuesAsBytes.Slice(Vector128.Count * 2); + if (Vector512.IsHardwareAccelerated && + source.Length >= Vector256.Count && destination.Length >= sizeof(uint)) + { + uint isFalse = Vector256.Equals(Vector256.Create(source), Vector256.Zero).ExtractMostSignificantBits(); + BinaryPrimitives.WriteUInt32LittleEndian(destination, ~isFalse); + + source = source.Slice(Vector256.Count); + destination = destination.Slice(sizeof(uint)); + } + + if (Vector128.IsHardwareAccelerated) + { + if (source.Length >= Vector128.Count && destination.Length >= sizeof(ushort)) + { + uint isFalse = Vector128.Equals(Vector128.Create(source), Vector128.Zero).ExtractMostSignificantBits(); + BinaryPrimitives.WriteUInt16LittleEndian(destination, (ushort)~isFalse); + + source = source.Slice(Vector128.Count); + destination = destination.Slice(sizeof(ushort)); + } + + // A ulong holds exactly eight bools, so the low half of a Vector128 produces the last byte. + if (source.Length >= sizeof(ulong) && destination.Length >= sizeof(byte)) + { + ulong eightBools = BinaryPrimitives.ReadUInt64LittleEndian(source); + uint isFalse = Vector128.Equals(Vector128.CreateScalar(eightBools).AsByte(), Vector128.Zero).ExtractMostSignificantBits(); + destination[0] = (byte)~isFalse; + + source = source.Slice(sizeof(ulong)); + } + } } + + i = (uint)(values.Length - source.Length); } - Remainder: for (; i < (uint)values.Length; i++) { if (values[(int)i])