Skip to content

Cleanup SimdUtils #2654

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
JimBobSquarePants merged 8 commits into from
Feb 14, 2024
Merged

Cleanup SimdUtils #2654

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
980347e
ENhance NormalizedFloatToByteSaturate
JimBobSquarePants Feb 2, 2024
89cd849
Remove unused methods
JimBobSquarePants Feb 2, 2024
68b7dbc
Update src/ImageSharp/Common/Helpers/SimdUtils.HwIntrinsics.cs
JimBobSquarePants Feb 2, 2024
9d737b7
Avoid downclocking
JimBobSquarePants Feb 2, 2024
805c7c4
Merge branch 'js/xplat-intrinsics' of https://github.com/SixLabors/Im…
JimBobSquarePants Feb 2, 2024
c6758df
Optimize and cleanup ByteToNormalizedFloatReduce
JimBobSquarePants Feb 6, 2024
0ec839f
Update SimdUtils.HwIntrinsics.cs
JimBobSquarePants Feb 6, 2024
4e494d4
Simplify checks
JimBobSquarePants Feb 14, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 20 additions & 0 deletions src/ImageSharp/Common/Helpers/Numerics.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1010,6 +1010,26 @@ public static nuint Vector256Count<TVector>(this ReadOnlySpan<byte> span)
where TVector : struct
=> (uint)span.Length / (uint)Vector256<TVector>.Count;

/// <summary>
/// Gets the count of vectors that safely fit into the given span.
/// </summary>
/// <typeparam name="TVector">The type of the vector.</typeparam>
/// <param name="span">The given span.</param>
/// <returns>Count of vectors that safely fit into the span.</returns>
public static nuint Vector512Count<TVector>(this Span<byte> span)
where TVector : struct
=> (uint)span.Length / (uint)Vector512<TVector>.Count;

/// <summary>
/// Gets the count of vectors that safely fit into the given span.
/// </summary>
/// <typeparam name="TVector">The type of the vector.</typeparam>
/// <param name="span">The given span.</param>
/// <returns>Count of vectors that safely fit into the span.</returns>
public static nuint Vector512Count<TVector>(this ReadOnlySpan<byte> span)
where TVector : struct
=> (uint)span.Length / (uint)Vector512<TVector>.Count;

/// <summary>
/// Gets the count of vectors that safely fit into the given span.
/// </summary>
Expand Down
4 changes: 2 additions & 2 deletions src/ImageSharp/Common/Helpers/SimdUtils.ExtendedIntrinsics.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -95,7 +95,7 @@ internal static void NormalizedFloatToByteSaturateReduce(
/// </summary>
internal static void ByteToNormalizedFloat(ReadOnlySpan<byte> source, Span<float> dest)
{
VerifySpanInput(source, dest, Vector<byte>.Count);
DebugVerifySpanInput(source, dest, Vector<byte>.Count);

nuint n = dest.VectorCount<byte>();

Expand Down Expand Up @@ -130,7 +130,7 @@ internal static void NormalizedFloatToByteSaturate(
ReadOnlySpan<float> source,
Span<byte> dest)
{
VerifySpanInput(source, dest, Vector<byte>.Count);
DebugVerifySpanInput(source, dest, Vector<byte>.Count);

nuint n = dest.VectorCount<byte>();

Expand Down
4 changes: 2 additions & 2 deletions src/ImageSharp/Common/Helpers/SimdUtils.FallbackIntrinsics128.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,7 @@ internal static void NormalizedFloatToByteSaturateReduce(
[MethodImpl(InliningOptions.ColdPath)]
internal static void ByteToNormalizedFloat(ReadOnlySpan<byte> source, Span<float> dest)
{
VerifySpanInput(source, dest, 4);
DebugVerifySpanInput(source, dest, 4);

uint count = (uint)dest.Length / 4;
if (count == 0)
Expand Down Expand Up @@ -103,7 +103,7 @@ internal static void NormalizedFloatToByteSaturate(
ReadOnlySpan<float> source,
Span<byte> dest)
{
VerifySpanInput(source, dest, 4);
DebugVerifySpanInput(source, dest, 4);

uint count = (uint)source.Length / 4;
if (count == 0)
Expand Down
150 changes: 97 additions & 53 deletions src/ImageSharp/Common/Helpers/SimdUtils.HwIntrinsics.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,13 @@ internal static partial class SimdUtils
{
public static class HwIntrinsics
{
#pragma warning disable SA1117 // Parameters should be on same line or separate lines
#pragma warning disable SA1137 // Elements should have the same indentation
[MethodImpl(MethodImplOptions.AggressiveInlining)] // too much IL for JIT to inline, so give a hint
public static Vector256<int> PermuteMaskDeinterleave8x32() => Vector256.Create(0, 0, 0, 0, 4, 0, 0, 0, 1, 0, 0, 0, 5, 0, 0, 0, 2, 0, 0, 0, 6, 0, 0, 0, 3, 0, 0, 0, 7, 0, 0, 0).AsInt32();
public static Vector256<int> PermuteMaskDeinterleave8x32() => Vector256.Create(0, 4, 1, 5, 2, 6, 3, 7);

[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static Vector512<int> PermuteMaskDeinterleave16x32() => Vector512.Create(0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15);

[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static Vector256<uint> PermuteMaskEvenOdd8x32() => Vector256.Create(0, 0, 0, 0, 2, 0, 0, 0, 4, 0, 0, 0, 6, 0, 0, 0, 1, 0, 0, 0, 3, 0, 0, 0, 5, 0, 0, 0, 7, 0, 0, 0).AsUInt32();
Expand All @@ -38,17 +43,18 @@ public static class HwIntrinsics
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static Vector128<byte> ShuffleMaskSlice4Nx16() => Vector128.Create(0, 1, 2, 4, 5, 6, 8, 9, 10, 12, 13, 14, 0x80, 0x80, 0x80, 0x80);

#pragma warning disable SA1003, SA1116, SA1117 // Parameters should be on same line or separate lines
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static Vector256<byte> ShuffleMaskShiftAlpha() => Vector256.Create((byte)
0, 1, 2, 4, 5, 6, 8, 9, 10, 12, 13, 14, 3, 7, 11, 15,
0, 1, 2, 4, 5, 6, 8, 9, 10, 12, 13, 14, 3, 7, 11, 15);
private static Vector256<byte> ShuffleMaskShiftAlpha() => Vector256.Create(
(byte)0, 1, 2, 4, 5, 6, 8, 9, 10, 12, 13, 14, 3, 7, 11, 15,
0, 1, 2, 4, 5, 6, 8, 9, 10, 12, 13, 14, 3, 7, 11, 15);

[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static Vector256<uint> PermuteMaskShiftAlpha8x32() => Vector256.Create(
0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 4, 0, 0, 0,
5, 0, 0, 0, 6, 0, 0, 0, 3, 0, 0, 0, 7, 0, 0, 0).AsUInt32();
#pragma warning restore SA1003, SA1116, SA1117 // Parameters should be on same line or separate lines
public static Vector256<uint> PermuteMaskShiftAlpha8x32()
=> Vector256.Create(
0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 4, 0, 0, 0,
5, 0, 0, 0, 6, 0, 0, 0, 3, 0, 0, 0, 7, 0, 0, 0).AsUInt32();
Comment thread
JimBobSquarePants marked this conversation as resolved.
Outdated
#pragma warning restore SA1137 // Elements should have the same indentation
#pragma warning restore SA1117 // Parameters should be on same line or separate lines

/// <summary>
/// Shuffle single-precision (32-bit) floating-point elements in <paramref name="source"/>
Expand Down Expand Up @@ -795,7 +801,7 @@ internal static unsafe void ByteToNormalizedFloat(
{
if (Avx2.IsSupported)
{
VerifySpanInput(source, dest, Vector256<byte>.Count);
DebugVerifySpanInput(source, dest, Vector256<byte>.Count);

nuint n = dest.Vector256Count<byte>();

Expand Down Expand Up @@ -828,7 +834,7 @@ internal static unsafe void ByteToNormalizedFloat(
else
{
// Sse
VerifySpanInput(source, dest, Vector128<byte>.Count);
DebugVerifySpanInput(source, dest, Vector128<byte>.Count);

nuint n = dest.Vector128Count<byte>();

Expand Down Expand Up @@ -881,17 +887,24 @@ internal static unsafe void ByteToNormalizedFloat(
/// <summary>
/// <see cref="NormalizedFloatToByteSaturate"/> as many elements as possible, slicing them down (keeping the remainder).
/// </summary>
/// <param name="source">The source buffer.</param>
/// <param name="destination">The destination buffer.</param>
[MethodImpl(InliningOptions.ShortMethod)]
internal static void NormalizedFloatToByteSaturateReduce(
ref ReadOnlySpan<float> source,
ref Span<byte> dest)
ref Span<byte> destination)
{
DebugGuard.IsTrue(source.Length == dest.Length, nameof(source), "Input spans must be of same length!");
DebugGuard.IsTrue(source.Length == destination.Length, nameof(source), "Input spans must be of same length!");

if (Avx2.IsSupported || Sse2.IsSupported)
if (Avx512BW.IsSupported || Avx2.IsSupported || Sse2.IsSupported || AdvSimd.IsSupported)

@saucecontrol saucecontrol Feb 2, 2024

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
if (Avx512BW.IsSupported || Avx2.IsSupported || Sse2.IsSupported || AdvSimd.IsSupported)
if (Sse2.IsSupported || AdvSimd.IsSupported)

Avx512BW.IsSupported and Avx2.IsSupported strictly imply Sse2.IsSupported, so this is redundant.

{
int remainder;
if (Avx2.IsSupported)

if (Avx512BW.IsSupported)

@saucecontrol saucecontrol Feb 2, 2024

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This one's probably ok, but you'll want to consider bypassing the AVX-512 path on older hardware like Skylake-X, because of the well-known downclocking issues.

Vector512.IsHardwareAccelerated will return false on these processors, even though they support the AVX-512 instructions, so you can check both conditions to make sure the code only runs where AVX-512 is supported and fast.

@JimBobSquarePants JimBobSquarePants Feb 2, 2024

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Foot meet gun!
I'll add the additional checks.

@saucecontrol saucecontrol Feb 8, 2024

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
if (Avx512BW.IsSupported)
if (Vector512.IsHardwareAccelerated && Avx512BW.IsSupported)

{
remainder = Numerics.ModuloP2(source.Length, Vector512<byte>.Count);
}
else if (Avx2.IsSupported)
{
remainder = Numerics.ModuloP2(source.Length, Vector256<byte>.Count);
}
Expand All @@ -906,36 +919,70 @@ internal static void NormalizedFloatToByteSaturateReduce(
{
NormalizedFloatToByteSaturate(
source[..adjustedCount],
dest[..adjustedCount]);
destination[..adjustedCount]);

source = source[adjustedCount..];
dest = dest[adjustedCount..];
destination = destination[adjustedCount..];
}
}
}

/// <summary>
/// Implementation of <see cref="SimdUtils.NormalizedFloatToByteSaturate"/>, which is faster on new .NET runtime.
/// </summary>
/// <param name="source">The source buffer.</param>
/// <param name="destination">The destination buffer.</param>
/// <remarks>
/// Implementation is based on MagicScaler code:
/// https://github.com/saucecontrol/PhotoSauce/blob/b5811908041200488aa18fdfd17df5fc457415dc/src/MagicScaler/Magic/Processors/ConvertersFloat.cs#L541-L622
/// </remarks>
internal static void NormalizedFloatToByteSaturate(
ReadOnlySpan<float> source,
Span<byte> dest)
Span<byte> destination)
{
if (Avx2.IsSupported)
if (Avx512BW.IsSupported)

@saucecontrol saucecontrol Feb 8, 2024

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
if (Avx512BW.IsSupported)
if (Vector512.IsHardwareAccelerated && Avx512BW.IsSupported)

{
VerifySpanInput(source, dest, Vector256<byte>.Count);
DebugVerifySpanInput(source, destination, Vector512<byte>.Count);

nuint n = destination.Vector512Count<byte>();

nuint n = dest.Vector256Count<byte>();
ref Vector512<float> sourceBase = ref Unsafe.As<float, Vector512<float>>(ref MemoryMarshal.GetReference(source));
ref Vector512<byte> destinationBase = ref Unsafe.As<byte, Vector512<byte>>(ref MemoryMarshal.GetReference(destination));

ref Vector256<float> sourceBase =
ref Unsafe.As<float, Vector256<float>>(ref MemoryMarshal.GetReference(source));
Vector512<float> scale = Vector512.Create((float)byte.MaxValue);
Vector512<int> mask = PermuteMaskDeinterleave16x32();

ref Vector256<byte> destBase =
ref Unsafe.As<byte, Vector256<byte>>(ref MemoryMarshal.GetReference(dest));
for (nuint i = 0; i < n; i++)
{
ref Vector512<float> s = ref Unsafe.Add(ref sourceBase, i * 4);

Vector512<float> f0 = scale * s;
Vector512<float> f1 = scale * Unsafe.Add(ref s, 1);
Vector512<float> f2 = scale * Unsafe.Add(ref s, 2);
Vector512<float> f3 = scale * Unsafe.Add(ref s, 3);

Vector512<int> w0 = Vector512Utilities.ConvertToInt32RoundToEven(f0);
Vector512<int> w1 = Vector512Utilities.ConvertToInt32RoundToEven(f1);
Vector512<int> w2 = Vector512Utilities.ConvertToInt32RoundToEven(f2);
Vector512<int> w3 = Vector512Utilities.ConvertToInt32RoundToEven(f3);

Vector512<short> u0 = Avx512BW.PackSignedSaturate(w0, w1);
Vector512<short> u1 = Avx512BW.PackSignedSaturate(w2, w3);
Vector512<byte> b = Avx512BW.PackUnsignedSaturate(u0, u1);
b = Avx512F.PermuteVar16x32(b.AsInt32(), mask).AsByte();

Unsafe.Add(ref destinationBase, i) = b;
}
}
else
if (Avx2.IsSupported)
{
DebugVerifySpanInput(source, destination, Vector256<byte>.Count);

nuint n = destination.Vector256Count<byte>();

ref Vector256<float> sourceBase = ref Unsafe.As<float, Vector256<float>>(ref MemoryMarshal.GetReference(source));
ref Vector256<byte> destinationBase = ref Unsafe.As<byte, Vector256<byte>>(ref MemoryMarshal.GetReference(destination));

Vector256<float> scale = Vector256.Create((float)byte.MaxValue);
Vector256<int> mask = PermuteMaskDeinterleave8x32();
Expand All @@ -944,57 +991,54 @@ internal static void NormalizedFloatToByteSaturate(
{
ref Vector256<float> s = ref Unsafe.Add(ref sourceBase, i * 4);

Vector256<float> f0 = Avx.Multiply(scale, s);
Vector256<float> f1 = Avx.Multiply(scale, Unsafe.Add(ref s, 1));
Vector256<float> f2 = Avx.Multiply(scale, Unsafe.Add(ref s, 2));
Vector256<float> f3 = Avx.Multiply(scale, Unsafe.Add(ref s, 3));
Vector256<float> f0 = scale * s;
Vector256<float> f1 = scale * Unsafe.Add(ref s, 1);
Vector256<float> f2 = scale * Unsafe.Add(ref s, 2);
Vector256<float> f3 = scale * Unsafe.Add(ref s, 3);

Vector256<int> w0 = Avx.ConvertToVector256Int32(f0);
Vector256<int> w1 = Avx.ConvertToVector256Int32(f1);
Vector256<int> w2 = Avx.ConvertToVector256Int32(f2);
Vector256<int> w3 = Avx.ConvertToVector256Int32(f3);
Vector256<int> w0 = Vector256Utilities.ConvertToInt32RoundToEven(f0);
Vector256<int> w1 = Vector256Utilities.ConvertToInt32RoundToEven(f1);
Vector256<int> w2 = Vector256Utilities.ConvertToInt32RoundToEven(f2);
Vector256<int> w3 = Vector256Utilities.ConvertToInt32RoundToEven(f3);

Vector256<short> u0 = Avx2.PackSignedSaturate(w0, w1);
Vector256<short> u1 = Avx2.PackSignedSaturate(w2, w3);
Vector256<byte> b = Avx2.PackUnsignedSaturate(u0, u1);
b = Avx2.PermuteVar8x32(b.AsInt32(), mask).AsByte();

Unsafe.Add(ref destBase, i) = b;
Unsafe.Add(ref destinationBase, i) = b;
}
}
else
{
// Sse
VerifySpanInput(source, dest, Vector128<byte>.Count);

nuint n = dest.Vector128Count<byte>();
// Sse, AdvSimd
DebugVerifySpanInput(source, destination, Vector128<byte>.Count);

ref Vector128<float> sourceBase =
ref Unsafe.As<float, Vector128<float>>(ref MemoryMarshal.GetReference(source));
nuint n = destination.Vector128Count<byte>();

ref Vector128<byte> destBase =
ref Unsafe.As<byte, Vector128<byte>>(ref MemoryMarshal.GetReference(dest));
ref Vector128<float> sourceBase = ref Unsafe.As<float, Vector128<float>>(ref MemoryMarshal.GetReference(source));
ref Vector128<byte> destinationBase = ref Unsafe.As<byte, Vector128<byte>>(ref MemoryMarshal.GetReference(destination));

Vector128<float> scale = Vector128.Create((float)byte.MaxValue);

for (nuint i = 0; i < n; i++)
{
ref Vector128<float> s = ref Unsafe.Add(ref sourceBase, i * 4);

Vector128<float> f0 = Sse.Multiply(scale, s);
Vector128<float> f1 = Sse.Multiply(scale, Unsafe.Add(ref s, 1));
Vector128<float> f2 = Sse.Multiply(scale, Unsafe.Add(ref s, 2));
Vector128<float> f3 = Sse.Multiply(scale, Unsafe.Add(ref s, 3));
Vector128<float> f0 = scale * s;
Vector128<float> f1 = scale * Unsafe.Add(ref s, 1);
Vector128<float> f2 = scale * Unsafe.Add(ref s, 2);
Vector128<float> f3 = scale * Unsafe.Add(ref s, 3);

Vector128<int> w0 = Sse2.ConvertToVector128Int32(f0);
Vector128<int> w1 = Sse2.ConvertToVector128Int32(f1);
Vector128<int> w2 = Sse2.ConvertToVector128Int32(f2);
Vector128<int> w3 = Sse2.ConvertToVector128Int32(f3);
Vector128<int> w0 = Vector128Utilities.ConvertToInt32RoundToEven(f0);
Vector128<int> w1 = Vector128Utilities.ConvertToInt32RoundToEven(f1);
Vector128<int> w2 = Vector128Utilities.ConvertToInt32RoundToEven(f2);
Vector128<int> w3 = Vector128Utilities.ConvertToInt32RoundToEven(f3);

Vector128<short> u0 = Sse2.PackSignedSaturate(w0, w1);
Vector128<short> u1 = Sse2.PackSignedSaturate(w2, w3);
Vector128<short> u0 = Vector128Utilities.PackSignedSaturate(w0, w1);
Vector128<short> u1 = Vector128Utilities.PackSignedSaturate(w2, w3);

Unsafe.Add(ref destBase, i) = Sse2.PackUnsignedSaturate(u0, u1);
Unsafe.Add(ref destinationBase, i) = Vector128Utilities.PackUnsignedSaturate(u0, u1);
}
}
}
Expand Down
Loading