SpirvShader: Add PhysicalPointers

And add load / store paths to deal with data layouts that are not interleaved by lane.

Bug: b/126330097

Change-Id: Id7730a606d623ffe74ae57668216f05159972f10
Reviewed-on: https://swiftshader-review.googlesource.com/c/SwiftShader/+/25710
Reviewed-by: Nicolas Capens <nicolascapens@google.com>
Tested-by: Ben Clayton <bclayton@google.com>

diff --git a/src/Pipeline/SpirvShader.cpp b/src/Pipeline/SpirvShader.cpp
index d5ff62d..ff5d7d6 100644 (file)
--- a/src/Pipeline/SpirvShader.cpp
+++ b/src/Pipeline/SpirvShader.cpp
@@ -542,6 +542,18 @@ namespace sw
                }
        }
 
+       bool SpirvShader::IsStorageInterleavedByLane(spv::StorageClass storageClass)
+       {
+               switch (storageClass)
+               {
+               case spv::StorageClassUniform:
+               case spv::StorageClassStorageBuffer:
+                       return false;
+               default:
+                       return true;
+               }
+       }
+
        template<typename F>
        int SpirvShader::VisitInterfaceInner(TypeID id, Decorations d, F f) const
        {
@@ -1056,12 +1068,24 @@ namespace sw
                        UNIMPLEMENTED("Descriptor-backed load not yet implemented");
                }
 
-               auto &ptrBase = routine->getValue(pointer.pointerBase);
+               Pointer<Float> ptrBase;
+               if (pointerBase.kind == Object::Kind::PhysicalPointer)
+               {
+                       ptrBase = routine->getPhysicalPointer(pointer.pointerBase);
+               }
+               else
+               {
+                       ptrBase = &routine->getValue(pointer.pointerBase)[0];
+               }
+
+               bool interleavedByLane = IsStorageInterleavedByLane(pointerBaseTy.storageClass);
+
                auto &dst = routine->createIntermediate(objectId, objectTy.sizeInComponents);
 
                if (pointer.kind == Object::Kind::Value)
                {
-                       auto offsets = As<SIMD::Int>(routine->getIntermediate(insn.word(3))[0]);
+                       // Divergent offsets.
+                       auto offsets = As<SIMD::Int>(routine->getIntermediate(pointerId)[0]);
                        for (auto i = 0u; i < objectTy.sizeInComponents; i++)
                        {
                                // i wish i had a Float,Float,Float,Float constructor here..
@@ -1069,17 +1093,27 @@ namespace sw
                                for (int j = 0; j < SIMD::Width; j++)
                                {
                                        Int offset = Int(i) + Extract(offsets, j);
-                                       v = Insert(v, Extract(ptrBase[offset], j), j);
+                                       if (interleavedByLane) { offset = offset * SIMD::Width + j; }
+                                       v = Insert(v, ptrBase[offset], j);
                                }
                                dst.emplace(i, v);
                        }
                }
+               else if (interleavedByLane)
+               {
+                       // Lane-interleaved data. No divergent offsets.
+                       Pointer<SIMD::Float> src = ptrBase;
+                       for (auto i = 0u; i < objectTy.sizeInComponents; i++)
+                       {
+                               dst.emplace(i, src[i]);
+                       }
+               }
                else
                {
-                       // no divergent offsets to worry about
+                       // Non-interleaved data. No divergent offsets.
                        for (auto i = 0u; i < objectTy.sizeInComponents; i++)
                        {
-                               dst.emplace(i, ptrBase[i]);
+                               dst.emplace(i, RValue<SIMD::Float>(ptrBase[i]));
                        }
                }
        }
@@ -1124,7 +1158,17 @@ namespace sw
                        UNIMPLEMENTED("Descriptor-backed store not yet implemented");
                }
 
-               auto &ptrBase = routine->getValue(pointer.pointerBase);
+               Pointer<Float> ptrBase;
+               if (pointerBase.kind == Object::Kind::PhysicalPointer)
+               {
+                       ptrBase = routine->getPhysicalPointer(pointer.pointerBase);
+               }
+               else
+               {
+                       ptrBase = &routine->getValue(pointer.pointerBase)[0];
+               }
+
+               bool interleavedByLane = IsStorageInterleavedByLane(pointerBaseTy.storageClass);
 
                if (object.kind == Object::Kind::Constant)
                {
@@ -1132,23 +1176,25 @@ namespace sw
 
                        if (pointer.kind == Object::Kind::Value)
                        {
+                               // Constant source data. Divergent offsets.
                                auto offsets = As<SIMD::Int>(routine->getIntermediate(pointerId)[0]);
                                for (auto i = 0u; i < elementTy.sizeInComponents; i++)
                                {
-                                       // Scattered store
                                        for (int j = 0; j < SIMD::Width; j++)
                                        {
-                                               auto dst = ptrBase[Int(i) + Extract(offsets, j)];
-                                               dst = Insert(dst, Float(src[i]), j);
+                                               Int offset = Int(i) + Extract(offsets, j);
+                                               if (interleavedByLane) { offset = offset * SIMD::Width + j; }
+                                               ptrBase[offset] = RValue<Float>(src[i]);
                                        }
                                }
                        }
                        else
                        {
-                               // no divergent offsets
+                               // Constant source data. No divergent offsets.
+                               Pointer<SIMD::Float> dst = ptrBase;
                                for (auto i = 0u; i < elementTy.sizeInComponents; i++)
                                {
-                                       ptrBase[i] = RValue<SIMD::Float>(src[i]);
+                                       dst[i] = RValue<SIMD::Float>(src[i]);
                                }
                        }
                }
@@ -1158,23 +1204,34 @@ namespace sw
 
                        if (pointer.kind == Object::Kind::Value)
                        {
+                               // Intermediate source data. Divergent offsets.
                                auto offsets = As<SIMD::Int>(routine->getIntermediate(pointerId)[0]);
                                for (auto i = 0u; i < elementTy.sizeInComponents; i++)
                                {
-                                       // Scattered store
                                        for (int j = 0; j < SIMD::Width; j++)
                                        {
-                                               auto dst = ptrBase[Int(i) + Extract(offsets, j)];
-                                               dst = Insert(dst, Extract(src[i], j), j);
+                                               Int offset = Int(i) + Extract(offsets, j);
+                                               if (interleavedByLane) { offset = offset * SIMD::Width + j; }
+                                               ptrBase[offset] = Extract(src[i], j);
                                        }
                                }
                        }
+                       else if (interleavedByLane)
+                       {
+                               // Intermediate source data. Lane-interleaved data. No divergent offsets.
+                               Pointer<SIMD::Float> dst = ptrBase;
+                               for (auto i = 0u; i < elementTy.sizeInComponents; i++)
+                               {
+                                       dst[i] = src[i];
+                               }
+                       }
                        else
                        {
-                               // no divergent offsets
+                               // Intermediate source data. Non-interleaved data. No divergent offsets.
+                               Pointer<SIMD::Float> dst = ptrBase;
                                for (auto i = 0u; i < elementTy.sizeInComponents; i++)
                                {
-                                       ptrBase[i] = src[i];
+                                       dst[i] = SIMD::Float(src[i]);
                                }
                        }
                }

diff --git a/src/Pipeline/SpirvShader.hpp b/src/Pipeline/SpirvShader.hpp
index 3b59dfd..4bfdc4a 100644 (file)
--- a/src/Pipeline/SpirvShader.hpp
+++ b/src/Pipeline/SpirvShader.hpp
@@ -220,10 +220,11 @@ namespace sw
                        enum class Kind
                        {
                                Unknown,        /* for paranoia -- if we get left with an object in this state, the module was broken */
-                               Variable,
-                               InterfaceVariable,
-                               Constant,
-                               Value,
+                               Variable,          // TODO: Document
+                               InterfaceVariable, // TODO: Document
+                               Constant,          // Values held by Object::constantValue
+                               Value,             // Values held by SpirvRoutine::intermediates
+                               PhysicalPointer,   // Pointer held by SpirvRoutine::physicalPointers
                        } kind = Kind::Unknown;
                };
 
@@ -384,6 +385,34 @@ namespace sw
                void ApplyDecorationsForId(Decorations *d, TypeOrObjectID id) const;
                void ApplyDecorationsForIdMember(Decorations *d, TypeID id, uint32_t member) const;
 
+               // Returns true if data in the given storage class is word-interleaved
+               // by each SIMD vector lane, otherwise data is linerally stored.
+               //
+               // A 'lane' is a component of a SIMD vector register.
+               // Given 4 consecutive loads/stores of 4 SIMD vector registers:
+               //
+               // "StorageInterleavedByLane":
+               //
+               //  Ptr+0:Reg0.x | Ptr+1:Reg0.y | Ptr+2:Reg0.z | Ptr+3:Reg0.w
+               // --------------+--------------+--------------+--------------
+               //  Ptr+4:Reg1.x | Ptr+5:Reg1.y | Ptr+6:Reg1.z | Ptr+7:Reg1.w
+               // --------------+--------------+--------------+--------------
+               //  Ptr+8:Reg2.x | Ptr+9:Reg2.y | Ptr+a:Reg2.z | Ptr+b:Reg2.w
+               // --------------+--------------+--------------+--------------
+               //  Ptr+c:Reg3.x | Ptr+d:Reg3.y | Ptr+e:Reg3.z | Ptr+f:Reg3.w
+               //
+               // Not "StorageInterleavedByLane":
+               //
+               //  Ptr+0:Reg0.x | Ptr+0:Reg0.y | Ptr+0:Reg0.z | Ptr+0:Reg0.w
+               // --------------+--------------+--------------+--------------
+               //  Ptr+1:Reg1.x | Ptr+1:Reg1.y | Ptr+1:Reg1.z | Ptr+1:Reg1.w
+               // --------------+--------------+--------------+--------------
+               //  Ptr+2:Reg2.x | Ptr+2:Reg2.y | Ptr+2:Reg2.z | Ptr+2:Reg2.w
+               // --------------+--------------+--------------+--------------
+               //  Ptr+3:Reg3.x | Ptr+3:Reg3.y | Ptr+3:Reg3.z | Ptr+3:Reg3.w
+               //
+               static bool IsStorageInterleavedByLane(spv::StorageClass storageClass);
+
                template<typename F>
                int VisitInterfaceInner(TypeID id, Decorations d, F f) const;
 
@@ -430,6 +459,8 @@ namespace sw
 
                std::unordered_map<SpirvShader::ObjectID, Intermediate> intermediates;
 
+               std::unordered_map<SpirvShader::ObjectID, Pointer<Byte> > physicalPointers;
+
                Value inputs = Value{MAX_INTERFACE_COMPONENTS};
                Value outputs = Value{MAX_INTERFACE_COMPONENTS};
 
@@ -461,6 +492,13 @@ namespace sw
                        ASSERT(it != intermediates.end());
                        return it->second;
                }
+
+               Pointer<Byte>& getPhysicalPointer(SpirvShader::ObjectID id)
+               {
+                       auto it = physicalPointers.find(id);
+                       assert(it != physicalPointers.end());
+                       return it->second;
+               }
        };
 
        class GenericValue