Skip to content

Commit a3c4c5d

Browse files
tannergoodingtmds
tannergooding
authored and
tmds
committed
Adding vectorized implementations of Exp to Vector64/128/256/512 (dotnet#97114)
* Adding vectorized implementations of Exp to Vector64/128/256/512 * Accelerate TensorPrimitives.Exp for double * Ensure the right allowedVariance is used for the vectorized exp tests * Ensure V128/256/512 defers to the next smaller vector size by operating on the lower/upper halves * Ensure the right allowedVariance amounts are used for the vectorized Exp(float) tests * Ensure we call Exp and that the methods are properly inlined * Skip the Exp test for Vector128/256/512 on Mono due to dotnet#97176
1 parent 6c04b48 commit a3c4c5d

12 files changed

Lines changed: 971 additions & 48 deletions

File tree

‎src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.netcore.cs‎

Lines changed: 335 additions & 40 deletions
Large diffs are not rendered by default.

‎src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Vector128.cs‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1426,6 +1426,40 @@ public static bool EqualsAny<T>(Vector128<T> left, Vector128<T> right)
14261426
|| Vector64.EqualsAny(left._upper, right._upper);
14271427
}
14281428

1429+
/// <inheritdoc cref="Vector64.Exp(Vector64{double})" />
1430+
[MethodImpl(MethodImplOptions.AggressiveInlining)]
1431+
public static Vector128<double> Exp(Vector128<double> vector)
1432+
{
1433+
if (IsHardwareAccelerated)
1434+
{
1435+
return VectorMath.ExpDouble<Vector128<double>, Vector128<long>, Vector128<ulong>>(vector);
1436+
}
1437+
else
1438+
{
1439+
return Create(
1440+
Vector64.Exp(vector._lower),
1441+
Vector64.Exp(vector._upper)
1442+
);
1443+
}
1444+
}
1445+
1446+
/// <inheritdoc cref="Vector64.Exp(Vector64{float})" />
1447+
[MethodImpl(MethodImplOptions.AggressiveInlining)]
1448+
public static Vector128<float> Exp(Vector128<float> vector)
1449+
{
1450+
if (IsHardwareAccelerated)
1451+
{
1452+
return VectorMath.ExpSingle<Vector128<float>, Vector128<uint>, Vector128<double>, Vector128<ulong>>(vector);
1453+
}
1454+
else
1455+
{
1456+
return Create(
1457+
Vector64.Exp(vector._lower),
1458+
Vector64.Exp(vector._upper)
1459+
);
1460+
}
1461+
}
1462+
14291463
/// <summary>Extracts the most significant bit from each element in a vector.</summary>
14301464
/// <typeparam name="T">The type of the elements in the vector.</typeparam>
14311465
/// <param name="vector">The vector whose elements should have their most significant bit extracted.</param>
@@ -1782,6 +1816,7 @@ internal static Vector128<ushort> LoadUnsafe(ref char source, nuint elementOffse
17821816
LoadUnsafe(ref Unsafe.As<char, ushort>(ref source), elementOffset);
17831817

17841818
/// <inheritdoc cref="Vector64.Log(Vector64{double})" />
1819+
[MethodImpl(MethodImplOptions.AggressiveInlining)]
17851820
public static Vector128<double> Log(Vector128<double> vector)
17861821
{
17871822
if (IsHardwareAccelerated)
@@ -1798,6 +1833,7 @@ public static Vector128<double> Log(Vector128<double> vector)
17981833
}
17991834

18001835
/// <inheritdoc cref="Vector64.Log(Vector64{float})" />
1836+
[MethodImpl(MethodImplOptions.AggressiveInlining)]
18011837
public static Vector128<float> Log(Vector128<float> vector)
18021838
{
18031839
if (IsHardwareAccelerated)
@@ -1814,6 +1850,7 @@ public static Vector128<float> Log(Vector128<float> vector)
18141850
}
18151851

18161852
/// <inheritdoc cref="Vector64.Log2(Vector64{double})" />
1853+
[MethodImpl(MethodImplOptions.AggressiveInlining)]
18171854
public static Vector128<double> Log2(Vector128<double> vector)
18181855
{
18191856
if (IsHardwareAccelerated)
@@ -1830,6 +1867,7 @@ public static Vector128<double> Log2(Vector128<double> vector)
18301867
}
18311868

18321869
/// <inheritdoc cref="Vector64.Log2(Vector64{float})" />
1870+
[MethodImpl(MethodImplOptions.AggressiveInlining)]
18331871
public static Vector128<float> Log2(Vector128<float> vector)
18341872
{
18351873
if (IsHardwareAccelerated)

‎src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Vector256.cs‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1402,6 +1402,40 @@ public static bool EqualsAny<T>(Vector256<T> left, Vector256<T> right)
14021402
|| Vector128.EqualsAny(left._upper, right._upper);
14031403
}
14041404

1405+
/// <inheritdoc cref="Vector128.Exp(Vector128{double})" />
1406+
[MethodImpl(MethodImplOptions.AggressiveInlining)]
1407+
public static Vector256<double> Exp(Vector256<double> vector)
1408+
{
1409+
if (IsHardwareAccelerated)
1410+
{
1411+
return VectorMath.ExpDouble<Vector256<double>, Vector256<long>, Vector256<ulong>>(vector);
1412+
}
1413+
else
1414+
{
1415+
return Create(
1416+
Vector128.Exp(vector._lower),
1417+
Vector128.Exp(vector._upper)
1418+
);
1419+
}
1420+
}
1421+
1422+
/// <inheritdoc cref="Vector128.Exp(Vector128{float})" />
1423+
[MethodImpl(MethodImplOptions.AggressiveInlining)]
1424+
public static Vector256<float> Exp(Vector256<float> vector)
1425+
{
1426+
if (IsHardwareAccelerated)
1427+
{
1428+
return VectorMath.ExpSingle<Vector256<float>, Vector256<uint>, Vector256<double>, Vector256<ulong>>(vector);
1429+
}
1430+
else
1431+
{
1432+
return Create(
1433+
Vector128.Exp(vector._lower),
1434+
Vector128.Exp(vector._upper)
1435+
);
1436+
}
1437+
}
1438+
14051439
/// <summary>Extracts the most significant bit from each element in a vector.</summary>
14061440
/// <param name="vector">The vector whose elements should have their most significant bit extracted.</param>
14071441
/// <typeparam name="T">The type of the elements in the vector.</typeparam>
@@ -1756,6 +1790,7 @@ internal static Vector256<ushort> LoadUnsafe(ref char source, nuint elementOffse
17561790
LoadUnsafe(ref Unsafe.As<char, ushort>(ref source), elementOffset);
17571791

17581792
/// <inheritdoc cref="Vector128.Log(Vector128{double})" />
1793+
[MethodImpl(MethodImplOptions.AggressiveInlining)]
17591794
public static Vector256<double> Log(Vector256<double> vector)
17601795
{
17611796
if (IsHardwareAccelerated)
@@ -1772,6 +1807,7 @@ public static Vector256<double> Log(Vector256<double> vector)
17721807
}
17731808

17741809
/// <inheritdoc cref="Vector128.Log(Vector128{float})" />
1810+
[MethodImpl(MethodImplOptions.AggressiveInlining)]
17751811
public static Vector256<float> Log(Vector256<float> vector)
17761812
{
17771813
if (IsHardwareAccelerated)
@@ -1788,6 +1824,7 @@ public static Vector256<float> Log(Vector256<float> vector)
17881824
}
17891825

17901826
/// <inheritdoc cref="Vector128.Log2(Vector128{double})" />
1827+
[MethodImpl(MethodImplOptions.AggressiveInlining)]
17911828
public static Vector256<double> Log2(Vector256<double> vector)
17921829
{
17931830
if (IsHardwareAccelerated)
@@ -1804,6 +1841,7 @@ public static Vector256<double> Log2(Vector256<double> vector)
18041841
}
18051842

18061843
/// <inheritdoc cref="Vector128.Log2(Vector128{float})" />
1844+
[MethodImpl(MethodImplOptions.AggressiveInlining)]
18071845
public static Vector256<float> Log2(Vector256<float> vector)
18081846
{
18091847
if (IsHardwareAccelerated)

‎src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Vector512.cs‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1453,6 +1453,40 @@ public static bool EqualsAny<T>(Vector512<T> left, Vector512<T> right)
14531453
|| Vector256.EqualsAny(left._upper, right._upper);
14541454
}
14551455

1456+
/// <inheritdoc cref="Vector256.Exp(Vector256{double})" />
1457+
[MethodImpl(MethodImplOptions.AggressiveInlining)]
1458+
public static Vector512<double> Exp(Vector512<double> vector)
1459+
{
1460+
if (IsHardwareAccelerated)
1461+
{
1462+
return VectorMath.ExpDouble<Vector512<double>, Vector512<long>, Vector512<ulong>>(vector);
1463+
}
1464+
else
1465+
{
1466+
return Create(
1467+
Vector256.Exp(vector._lower),
1468+
Vector256.Exp(vector._upper)
1469+
);
1470+
}
1471+
}
1472+
1473+
/// <inheritdoc cref="Vector256.Exp(Vector256{float})" />
1474+
[MethodImpl(MethodImplOptions.AggressiveInlining)]
1475+
public static Vector512<float> Exp(Vector512<float> vector)
1476+
{
1477+
if (IsHardwareAccelerated)
1478+
{
1479+
return VectorMath.ExpSingle<Vector512<float>, Vector512<uint>, Vector512<double>, Vector512<ulong>>(vector);
1480+
}
1481+
else
1482+
{
1483+
return Create(
1484+
Vector256.Exp(vector._lower),
1485+
Vector256.Exp(vector._upper)
1486+
);
1487+
}
1488+
}
1489+
14561490
/// <summary>Extracts the most significant bit from each element in a vector.</summary>
14571491
/// <param name="vector">The vector whose elements should have their most significant bit extracted.</param>
14581492
/// <typeparam name="T">The type of the elements in the vector.</typeparam>
@@ -1807,6 +1841,7 @@ internal static Vector512<ushort> LoadUnsafe(ref char source, nuint elementOffse
18071841
LoadUnsafe(ref Unsafe.As<char, ushort>(ref source), elementOffset);
18081842

18091843
/// <inheritdoc cref="Vector256.Log(Vector256{double})" />
1844+
[MethodImpl(MethodImplOptions.AggressiveInlining)]
18101845
public static Vector512<double> Log(Vector512<double> vector)
18111846
{
18121847
if (IsHardwareAccelerated)
@@ -1823,6 +1858,7 @@ public static Vector512<double> Log(Vector512<double> vector)
18231858
}
18241859

18251860
/// <inheritdoc cref="Vector256.Log(Vector256{float})" />
1861+
[MethodImpl(MethodImplOptions.AggressiveInlining)]
18261862
public static Vector512<float> Log(Vector512<float> vector)
18271863
{
18281864
if (IsHardwareAccelerated)
@@ -1839,6 +1875,7 @@ public static Vector512<float> Log(Vector512<float> vector)
18391875
}
18401876

18411877
/// <inheritdoc cref="Vector256.Log2(Vector256{double})" />
1878+
[MethodImpl(MethodImplOptions.AggressiveInlining)]
18421879
public static Vector512<double> Log2(Vector512<double> vector)
18431880
{
18441881
if (IsHardwareAccelerated)
@@ -1855,6 +1892,7 @@ public static Vector512<double> Log2(Vector512<double> vector)
18551892
}
18561893

18571894
/// <inheritdoc cref="Vector256.Log2(Vector256{float})" />
1895+
[MethodImpl(MethodImplOptions.AggressiveInlining)]
18581896
public static Vector512<float> Log2(Vector512<float> vector)
18591897
{
18601898
if (IsHardwareAccelerated)

‎src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Vector64.cs‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1156,6 +1156,52 @@ public static bool EqualsAny<T>(Vector64<T> left, Vector64<T> right)
11561156
return false;
11571157
}
11581158

1159+
internal static Vector64<T> Exp<T>(Vector64<T> vector)
1160+
where T : IExponentialFunctions<T>
1161+
{
1162+
Unsafe.SkipInit(out Vector64<T> result);
1163+
1164+
for (int index = 0; index < Vector64<T>.Count; index++)
1165+
{
1166+
T value = T.Exp(vector.GetElement(index));
1167+
result.SetElementUnsafe(index, value);
1168+
}
1169+
1170+
return result;
1171+
}
1172+
1173+
/// <summary>Computes the exp of each element in a vector.</summary>
1174+
/// <param name="vector">The vector that will have its Exp computed.</param>
1175+
/// <returns>A vector whose elements are the exp of the elements in <paramref name="vector" />.</returns>
1176+
[MethodImpl(MethodImplOptions.AggressiveInlining)]
1177+
public static Vector64<double> Exp(Vector64<double> vector)
1178+
{
1179+
if (IsHardwareAccelerated)
1180+
{
1181+
return VectorMath.ExpDouble<Vector64<double>, Vector64<long>, Vector64<ulong>>(vector);
1182+
}
1183+
else
1184+
{
1185+
return Exp<double>(vector);
1186+
}
1187+
}
1188+
1189+
/// <summary>Computes the exp of each element in a vector.</summary>
1190+
/// <param name="vector">The vector that will have its exp computed.</param>
1191+
/// <returns>A vector whose elements are the exp of the elements in <paramref name="vector" />.</returns>
1192+
[MethodImpl(MethodImplOptions.AggressiveInlining)]
1193+
public static Vector64<float> Exp(Vector64<float> vector)
1194+
{
1195+
if (IsHardwareAccelerated)
1196+
{
1197+
return VectorMath.ExpSingle<Vector64<float>, Vector64<uint>, Vector64<double>, Vector64<ulong>>(vector);
1198+
}
1199+
else
1200+
{
1201+
return Exp<float>(vector);
1202+
}
1203+
}
1204+
11591205
/// <summary>Extracts the most significant bit from each element in a vector.</summary>
11601206
/// <typeparam name="T">The type of the elements in the vector.</typeparam>
11611207
/// <param name="vector">The vector whose elements should have their most significant bit extracted.</param>
@@ -1588,6 +1634,7 @@ internal static Vector64<T> Log<T>(Vector64<T> vector)
15881634
/// <summary>Computes the log of each element in a vector.</summary>
15891635
/// <param name="vector">The vector that will have its log computed.</param>
15901636
/// <returns>A vector whose elements are the log of the elements in <paramref name="vector" />.</returns>
1637+
[MethodImpl(MethodImplOptions.AggressiveInlining)]
15911638
public static Vector64<double> Log(Vector64<double> vector)
15921639
{
15931640
if (IsHardwareAccelerated)
@@ -1603,6 +1650,7 @@ public static Vector64<double> Log(Vector64<double> vector)
16031650
/// <summary>Computes the log of each element in a vector.</summary>
16041651
/// <param name="vector">The vector that will have its log computed.</param>
16051652
/// <returns>A vector whose elements are the log of the elements in <paramref name="vector" />.</returns>
1653+
[MethodImpl(MethodImplOptions.AggressiveInlining)]
16061654
public static Vector64<float> Log(Vector64<float> vector)
16071655
{
16081656
if (IsHardwareAccelerated)
@@ -1632,6 +1680,7 @@ internal static Vector64<T> Log2<T>(Vector64<T> vector)
16321680
/// <summary>Computes the log2 of each element in a vector.</summary>
16331681
/// <param name="vector">The vector that will have its log2 computed.</param>
16341682
/// <returns>A vector whose elements are the log2 of the elements in <paramref name="vector" />.</returns>
1683+
[MethodImpl(MethodImplOptions.AggressiveInlining)]
16351684
public static Vector64<double> Log2(Vector64<double> vector)
16361685
{
16371686
if (IsHardwareAccelerated)
@@ -1647,6 +1696,7 @@ public static Vector64<double> Log2(Vector64<double> vector)
16471696
/// <summary>Computes the log2 of each element in a vector.</summary>
16481697
/// <param name="vector">The vector that will have its log2 computed.</param>
16491698
/// <returns>A vector whose elements are the log2 of the elements in <paramref name="vector" />.</returns>
1699+
[MethodImpl(MethodImplOptions.AggressiveInlining)]
16501700
public static Vector64<float> Log2(Vector64<float> vector)
16511701
{
16521702
if (IsHardwareAccelerated)

0 commit comments

Comments
 (0)