From ed9606c6326e1bd05e99eec9990f37a5c5ab4a6c Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Thu, 17 Jul 2025 09:55:48 +0100 Subject: [PATCH 1/4] [UnitTest] Add test for fmax reductions without fast-math. Add unit tests for vectorizing FMax reductions without fast-math flags, covering a wide range of inputs, including various combinations of NaNs and signed-zeros. --- SingleSource/UnitTests/Vectorizer/common.h | 8 + .../UnitTests/Vectorizer/fmax-reduction.cpp | 218 ++++++++++++++++++ 2 files changed, 226 insertions(+) create mode 100644 SingleSource/UnitTests/Vectorizer/fmax-reduction.cpp diff --git a/SingleSource/UnitTests/Vectorizer/common.h b/SingleSource/UnitTests/Vectorizer/common.h index 35107b15b1..ac779d5447 100644 --- a/SingleSource/UnitTests/Vectorizer/common.h +++ b/SingleSource/UnitTests/Vectorizer/common.h @@ -1,6 +1,14 @@ #include #include +#define DEFINE_SCALAR_AND_VECTOR_FN1_TYPE(Init, Loop, Type) \ + auto ScalarFn = [](auto *A, Type TC) -> Type { \ + Init _Pragma("clang loop vectorize(disable) interleave_count(1)") Loop \ + }; \ + auto VectorFn = [](auto *A, Type TC) -> Type { \ + Init _Pragma("clang loop vectorize(enable)") Loop \ + }; + #define DEFINE_SCALAR_AND_VECTOR_FN2(Init, Loop) \ auto ScalarFn = [](auto *A, auto *B, unsigned TC) { \ Init _Pragma("clang loop vectorize(disable) interleave_count(1)") Loop \ diff --git a/SingleSource/UnitTests/Vectorizer/fmax-reduction.cpp b/SingleSource/UnitTests/Vectorizer/fmax-reduction.cpp new file mode 100644 index 0000000000..8ddcb23851 --- /dev/null +++ b/SingleSource/UnitTests/Vectorizer/fmax-reduction.cpp @@ -0,0 +1,218 @@ +#include +#include +#include +#include +#include +#include + +#include "common.h" + +static bool isEqual(float A, float B) { + if (std::isnan(A)) + return std::isnan(B); + + if (A == 0.0f) + return std::signbit(A) == std::signbit(B); + + return A == B; +} + +template using Fn1Ty = std::function; + +template +static void check(Fn1Ty ScalarFn, Fn1Ty VectorFn, float *Src, + unsigned N, const char *Type) { + auto Reference = ScalarFn(Src, N); + auto ToCheck = VectorFn(Src, N); + if (!isEqual(Reference, ToCheck)) { + std::cerr << "Miscompare " << Type << ": " << Reference << " != " << ToCheck + << "\n"; + exit(1); + } +} + +template +static void checkVectorFunction(Fn1Ty ScalarFn, Fn1Ty VectorFn, + const char *Name) { + std::cout << "Checking " << Name << "\n"; + + unsigned N = 1024; + std::unique_ptr Src1(new Ty[N]); + init_data(Src1, N); + + // Check with random inputs. + + // Check with sorted inputs. + std::sort(&Src1[0], &Src1[N]); + check(ScalarFn, VectorFn, &Src1[0], N, "sorted"); + + // Check with reverse sorted inputs. + std::reverse(&Src1[0], &Src1[N]); + check(ScalarFn, VectorFn, &Src1[0], N, "reverse-sorted"); + + // Check with all max values. + for (unsigned I = 0; I != N; ++I) + Src1[I] = std::numeric_limits::max(); + check(ScalarFn, VectorFn, &Src1[0], N, "all-max"); + + // Check with all min values. + for (unsigned I = 0; I != N; ++I) + Src1[I] = std::numeric_limits::min(); + check(ScalarFn, VectorFn, &Src1[0], N, "all-min"); + + // Check with inputs all zero. + for (unsigned I = 0; I != N; ++I) + Src1[I] = 0.0; + check(ScalarFn, VectorFn, &Src1[0], N, "all-zeros"); + + // Check with NaN at different indices. + for (unsigned NaNIdx = 3; NaNIdx != 32; NaNIdx++) { + for (unsigned I = 0; I != N; ++I) + Src1[I] = 100; + Src1[NaNIdx] = std::numeric_limits::quiet_NaN(); + + check(ScalarFn, VectorFn, &Src1[0], N, "NaN"); + } + + // Check with multiple signed-zeros at different positions. + for (unsigned Idx = 0; Idx != 64; ++Idx) { + for (unsigned I = 0; I != N; ++I) + Src1[I] = -1.0; + + for (unsigned Offset = 1; Offset != 16; ++Offset) { + Src1[Idx] = -0.0; + Src1[Idx + Offset] = +0.0; + + check(ScalarFn, VectorFn, &Src1[0], N, "signed-zeros"); + } + } + + // Check with max value at all possible indices. + for (unsigned Idx = 0; Idx != N; ++Idx) { + for (unsigned I = 0; I != N; ++I) + Src1[I] = I; + + Src1[Idx] = N + 1; + + check(ScalarFn, VectorFn, &Src1[0], N, "full"); + + for (unsigned Offset = 1; Offset != 16; ++Offset) { + if (Idx + Offset < N) + Src1[Idx + Offset] = N + 1; + + check(ScalarFn, VectorFn, &Src1[0], N, "full"); + } + } + + // Check with NaN value at all possible indices. + for (unsigned Idx = 0; Idx != N; ++Idx) { + for (unsigned I = 0; I != N; ++I) + Src1[I] = I; + + Src1[Idx] = std::numeric_limits::quiet_NaN(); + check(ScalarFn, VectorFn, &Src1[0], N, "full-with-nan"); + + // Check with multiple NaNs at different offsets. + for (unsigned Offset = 1; Offset != 16; ++Offset) { + if (Idx + Offset < N) + Src1[Idx + Offset] = std::numeric_limits::quiet_NaN(); + + check(ScalarFn, VectorFn, &Src1[0], N, "full-with-multiple-nan"); + } + } +} + +int main(void) { + rng = std::mt19937(15); + + { + DEFINE_SCALAR_AND_VECTOR_FN1_TYPE( + float Max = -2.0;, for (unsigned I = 0; I < 1024; + I++) { Max = std::fmax(Max, A[I]); } return Max; + , float); + checkVectorFunction(ScalarFn, VectorFn, "fmaxnum_start_neg_2"); + } + { + DEFINE_SCALAR_AND_VECTOR_FN1_TYPE( + float Max = std::numeric_limits::min(); + , for (unsigned I = 0; I < 1024; + I++) { Max = std::fmax(Max, A[I]); } return Max; + , float); + checkVectorFunction(ScalarFn, VectorFn, "fmaxnum_start_min"); + } + { + DEFINE_SCALAR_AND_VECTOR_FN1_TYPE( + float Max = std::numeric_limits::quiet_NaN(); + , for (unsigned I = 0; I < 1024; + I++) { Max = std::fmax(Max, A[I]); } return Max; + , float); + checkVectorFunction(ScalarFn, VectorFn, "fmaxnum_start_is_nan"); + } + + { + DEFINE_SCALAR_AND_VECTOR_FN1_TYPE( + float Max = -2.0; + , for (unsigned I = 0; I < 1024; + I++) { Max = A[I] > Max ? A[I] : Max; } return Max; + , float); + checkVectorFunction(ScalarFn, VectorFn, "fmax_strict_start_neg_2"); + } + { + DEFINE_SCALAR_AND_VECTOR_FN1_TYPE( + float Max = std::numeric_limits::min(); + , for (unsigned I = 0; I < 1024; + I++) { Max = A[I] > Max ? A[I] : Max; } return Max; + , float); + checkVectorFunction(ScalarFn, VectorFn, "fmax_strict_start_min"); + } + { + DEFINE_SCALAR_AND_VECTOR_FN1_TYPE( + float Max = std::numeric_limits::quiet_NaN(); + , for (unsigned I = 0; I < 1025; + I++) { Max = A[I] > Max ? A[I] : Max; } return Max; + , float); + checkVectorFunction(ScalarFn, VectorFn, "fmax_strict_start_nan"); + } + + { + DEFINE_SCALAR_AND_VECTOR_FN1_TYPE( + float Max = -2.0; + , for (unsigned I = 0; I < 1024; + I++) { Max = Max >= A[I] ? Max : A[I]; } return Max; + , float); + checkVectorFunction(ScalarFn, VectorFn, + "fmax_non_strict_start_neg_2"); + } + + { + DEFINE_SCALAR_AND_VECTOR_FN1_TYPE( + float Max = -2.0; + , for (unsigned I = 0; I < 1024; + I++) { Max = Max > A[I] ? Max : A[I]; } return Max; + , float); + checkVectorFunction(ScalarFn, VectorFn, + "fmax_cmp_max_gt_start_neg_2"); + } + + { + DEFINE_SCALAR_AND_VECTOR_FN1_TYPE( + float Max = -2.0; + , for (unsigned I = 0; I < 1024; + I++) { Max = Max < A[I] ? A[I] : Max; } return Max; + , float); + checkVectorFunction(ScalarFn, VectorFn, + "fmax_cmp_max_lt_start_neg_2"); + } + + { + DEFINE_SCALAR_AND_VECTOR_FN1_TYPE( + float Max = std::numeric_limits::quiet_NaN(); + , for (unsigned I = 0; I < 1024; + I++) { Max = Max < A[I] ? A[I] : Max; } return Max; + , float); + checkVectorFunction(ScalarFn, VectorFn, + "fmax_cmp_max_lt_start_neg_nan"); + } + + return 0; +} From 2113a9e4b0bc7881f6b004aa4c1ff116e61ab8ca Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Thu, 17 Jul 2025 11:25:12 +0100 Subject: [PATCH 2/4] !fixup also check with denormals and infinity. --- .../UnitTests/Vectorizer/fmax-reduction.cpp | 50 +++++++++++++++++-- .../fmax-reduction.cpp.reference_output | 14 ++++++ 2 files changed, 61 insertions(+), 3 deletions(-) create mode 100644 SingleSource/UnitTests/Vectorizer/fmax-reduction.cpp.reference_output diff --git a/SingleSource/UnitTests/Vectorizer/fmax-reduction.cpp b/SingleSource/UnitTests/Vectorizer/fmax-reduction.cpp index 8ddcb23851..e802ffbc7c 100644 --- a/SingleSource/UnitTests/Vectorizer/fmax-reduction.cpp +++ b/SingleSource/UnitTests/Vectorizer/fmax-reduction.cpp @@ -8,8 +8,8 @@ #include "common.h" static bool isEqual(float A, float B) { - if (std::isnan(A)) - return std::isnan(B); + if (std::isnan(A) || std::isnan(B)) + return std::isnan(A) && std::isnan(B); if (A == 0.0f) return std::signbit(A) == std::signbit(B); @@ -60,6 +60,12 @@ static void checkVectorFunction(Fn1Ty ScalarFn, Fn1Ty VectorFn, Src1[I] = std::numeric_limits::min(); check(ScalarFn, VectorFn, &Src1[0], N, "all-min"); + // Check with various denormals. + Src1[0] = std::numeric_limits::denorm_min(); + for (unsigned I = 1; I != N; ++I) + Src1[I] = std::numeric_limits::denorm_min() / I; + check(ScalarFn, VectorFn, &Src1[0], N, "denormals"); + // Check with inputs all zero. for (unsigned I = 0; I != N; ++I) Src1[I] = 0.0; @@ -120,6 +126,19 @@ static void checkVectorFunction(Fn1Ty ScalarFn, Fn1Ty VectorFn, check(ScalarFn, VectorFn, &Src1[0], N, "full-with-multiple-nan"); } } + + // Check with multiple infinity values at different positions. + for (unsigned Idx = 0; Idx != 64; ++Idx) { + for (unsigned I = 0; I != N; ++I) + Src1[I] = -1.0; + + for (unsigned Offset = 1; Offset != 16; ++Offset) { + Src1[Idx] = -std::numeric_limits::infinity(); + Src1[Idx + Offset] = std::numeric_limits::infinity(); + + check(ScalarFn, VectorFn, &Src1[0], N, "infinity"); + } + } } int main(void) { @@ -140,6 +159,14 @@ int main(void) { , float); checkVectorFunction(ScalarFn, VectorFn, "fmaxnum_start_min"); } + { + DEFINE_SCALAR_AND_VECTOR_FN1_TYPE( + float Max = std::numeric_limits::denorm_min(); + , for (unsigned I = 0; I < 1024; + I++) { Max = std::fmax(Max, A[I]); } return Max; + , float); + checkVectorFunction(ScalarFn, VectorFn, "fmaxnum_start_denorm_min"); + } { DEFINE_SCALAR_AND_VECTOR_FN1_TYPE( float Max = std::numeric_limits::quiet_NaN(); @@ -165,6 +192,15 @@ int main(void) { , float); checkVectorFunction(ScalarFn, VectorFn, "fmax_strict_start_min"); } + { + DEFINE_SCALAR_AND_VECTOR_FN1_TYPE( + float Max = std::numeric_limits::denorm_min(); + , for (unsigned I = 0; I < 1025; + I++) { Max = A[I] > Max ? A[I] : Max; } return Max; + , float); + checkVectorFunction(ScalarFn, VectorFn, + "fmax_strict_start_denorm_min"); + } { DEFINE_SCALAR_AND_VECTOR_FN1_TYPE( float Max = std::numeric_limits::quiet_NaN(); @@ -203,7 +239,15 @@ int main(void) { checkVectorFunction(ScalarFn, VectorFn, "fmax_cmp_max_lt_start_neg_2"); } - + { + DEFINE_SCALAR_AND_VECTOR_FN1_TYPE( + float Max = std::numeric_limits::denorm_min(); + , for (unsigned I = 0; I < 1024; + I++) { Max = Max < A[I] ? A[I] : Max; } return Max; + , float); + checkVectorFunction(ScalarFn, VectorFn, + "fmax_cmp_max_lt_start_denorm_min"); + } { DEFINE_SCALAR_AND_VECTOR_FN1_TYPE( float Max = std::numeric_limits::quiet_NaN(); diff --git a/SingleSource/UnitTests/Vectorizer/fmax-reduction.cpp.reference_output b/SingleSource/UnitTests/Vectorizer/fmax-reduction.cpp.reference_output new file mode 100644 index 0000000000..e58266f902 --- /dev/null +++ b/SingleSource/UnitTests/Vectorizer/fmax-reduction.cpp.reference_output @@ -0,0 +1,14 @@ +Checking fmaxnum_start_neg_2 +Checking fmaxnum_start_min +Checking fmaxnum_start_denorm_min +Checking fmaxnum_start_is_nan +Checking fmax_strict_start_neg_2 +Checking fmax_strict_start_min +Checking fmax_strict_start_denorm_min +Checking fmax_strict_start_nan +Checking fmax_non_strict_start_neg_2 +Checking fmax_cmp_max_gt_start_neg_2 +Checking fmax_cmp_max_lt_start_neg_2 +Checking fmax_cmp_max_lt_start_denorm_min +Checking fmax_cmp_max_lt_start_neg_nan +exit 0 From e24fbfd56b07025a02f60bdd402570dfb988dbba Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Thu, 17 Jul 2025 11:43:30 +0100 Subject: [PATCH 3/4] !fixup check if B is also zero. --- SingleSource/UnitTests/Vectorizer/fmax-reduction.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SingleSource/UnitTests/Vectorizer/fmax-reduction.cpp b/SingleSource/UnitTests/Vectorizer/fmax-reduction.cpp index e802ffbc7c..b8f0fe840a 100644 --- a/SingleSource/UnitTests/Vectorizer/fmax-reduction.cpp +++ b/SingleSource/UnitTests/Vectorizer/fmax-reduction.cpp @@ -12,7 +12,7 @@ static bool isEqual(float A, float B) { return std::isnan(A) && std::isnan(B); if (A == 0.0f) - return std::signbit(A) == std::signbit(B); + return B == 0.0f && std::signbit(A) == std::signbit(B); return A == B; } From e9cb3f8dda338336180b735e7a6607028a99d880 Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Fri, 18 Jul 2025 10:19:13 +0100 Subject: [PATCH 4/4] !fixup also check different order of signed zeros. --- .../UnitTests/Vectorizer/fmax-reduction.cpp | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/SingleSource/UnitTests/Vectorizer/fmax-reduction.cpp b/SingleSource/UnitTests/Vectorizer/fmax-reduction.cpp index b8f0fe840a..dcae59295d 100644 --- a/SingleSource/UnitTests/Vectorizer/fmax-reduction.cpp +++ b/SingleSource/UnitTests/Vectorizer/fmax-reduction.cpp @@ -85,7 +85,7 @@ static void checkVectorFunction(Fn1Ty ScalarFn, Fn1Ty VectorFn, for (unsigned I = 0; I != N; ++I) Src1[I] = -1.0; - for (unsigned Offset = 1; Offset != 16; ++Offset) { + for (unsigned Offset = 1; Offset != 32; ++Offset) { Src1[Idx] = -0.0; Src1[Idx + Offset] = +0.0; @@ -93,6 +93,18 @@ static void checkVectorFunction(Fn1Ty ScalarFn, Fn1Ty VectorFn, } } + for (unsigned Idx = 0; Idx != 64; ++Idx) { + for (unsigned I = 0; I != N; ++I) + Src1[I] = -1.0; + + for (unsigned Offset = 1; Offset != 32; ++Offset) { + Src1[Idx] = +0.0; + Src1[Idx + Offset] = -0.0; + + check(ScalarFn, VectorFn, &Src1[0], N, "signed-zeros"); + } + } + // Check with max value at all possible indices. for (unsigned Idx = 0; Idx != N; ++Idx) { for (unsigned I = 0; I != N; ++I)