Fix Frac() returning 1.0.

Frac() should always produce results in the range [0.0, 1.0), and thus
never produce a 1.0. However, the current implementation uses
x - floor(x) and this returns 1.0 for very small negative values due to
catastrophic cancellation.

Bug swiftshader:74

Change-Id: I942dd7cfb1f7ee3a260070e748704f005eed0b13
Reviewed-on: https://swiftshader-review.googlesource.com/10648
Tested-by: Nicolas Capens <nicolascapens@google.com>
Reviewed-by: Alexis Hétu <sugoi@google.com>

diff --git a/src/Reactor/LLVMReactor.cpp b/src/Reactor/LLVMReactor.cpp
index dda5c04..d8bda70 100644 (file)
--- a/src/Reactor/LLVMReactor.cpp
+++ b/src/Reactor/LLVMReactor.cpp
@@ -6255,16 +6255,22 @@ namespace sw
 
        RValue<Float4> Frac(RValue<Float4> x)
        {
+               Float4 frc;
+
                if(CPUID::supportsSSE4_1())
                {
-                       return x - x86::floorps(x);
+                       frc = x - x86::floorps(x);
                }
                else
                {
-                       Float4 frc = x - Float4(Int4(x));   // Signed fractional part
+                       frc = x - Float4(Int4(x));   // Signed fractional part.
 
-                       return frc + As<Float4>(As<Int4>(CmpNLE(Float4(0.0f), frc)) & As<Int4>(Float4(1, 1, 1, 1)));
+                       frc += As<Float4>(As<Int4>(CmpNLE(Float4(0.0f), frc)) & As<Int4>(Float4(1.0f)));   // Add 1.0 if negative.
                }
+
+               // x - floor(x) can be 1.0 for very small negative x.
+               // Clamp against the value just below 1.0.
+               return Min(frc, As<Float4>(Int4(0x3F7FFFFF)));
        }
 
        RValue<Float4> Floor(RValue<Float4> x)

diff --git a/src/Reactor/SubzeroReactor.cpp b/src/Reactor/SubzeroReactor.cpp
index fc70ac2..fb116a3 100644 (file)
--- a/src/Reactor/SubzeroReactor.cpp
+++ b/src/Reactor/SubzeroReactor.cpp
@@ -6325,16 +6325,22 @@ namespace sw
 
        RValue<Float4> Frac(RValue<Float4> x)
        {
+               Float4 frc;
+
                if(CPUID::SSE4_1)
                {
-                       return x - Floor(x);
+                       frc = x - Floor(x);
                }
                else
                {
-                       Float4 frc = x - Float4(Int4(x));   // Signed fractional part
+                       frc = x - Float4(Int4(x));   // Signed fractional part.
 
-                       return frc + As<Float4>(As<Int4>(CmpNLE(Float4(0.0f), frc)) & As<Int4>(Float4(1, 1, 1, 1)));
+                       frc += As<Float4>(As<Int4>(CmpNLE(Float4(0.0f), frc)) & As<Int4>(Float4(1, 1, 1, 1)));   // Add 1.0 if negative.
                }
+
+               // x - floor(x) can be 1.0 for very small negative x.
+               // Clamp against the value just below 1.0.
+               return Min(frc, As<Float4>(Int4(0x3F7FFFFF)));
        }
 
        RValue<Float4> Floor(RValue<Float4> x)