Fixed recursion analysis

Recursion analysis was broken, because assigning
the error value UINT_MAX was then automatically
increased by 1 at the caller site, resulting in
a 0, and recursions would go undetected.

Change-Id: I8ab9990c12d827d8eac2d6084f9170096ad2aef2
Reviewed-on: https://swiftshader-review.googlesource.com/3552
Tested-by: Alexis Hétu <sugoi@google.com>
Reviewed-by: Nicolas Capens <capn@google.com>

diff --git a/src/OpenGL/compiler/AnalyzeCallDepth.cpp b/src/OpenGL/compiler/AnalyzeCallDepth.cpp
index 94c3540..d4f30fa 100644 (file)
--- a/src/OpenGL/compiler/AnalyzeCallDepth.cpp
+++ b/src/OpenGL/compiler/AnalyzeCallDepth.cpp
@@ -40,21 +40,24 @@ unsigned int AnalyzeCallDepth::FunctionNode::analyzeCallDepth(AnalyzeCallDepth *
 \r
     for(size_t i = 0; i < callees.size(); i++)\r
        {\r
+               unsigned int calleeDepth = 0;\r
         switch(callees[i]->visit)\r
                {\r
         case InVisit:\r
             // Cycle detected (recursion)\r
             return UINT_MAX;\r
         case PostVisit:\r
-                       callDepth = std::max(callDepth, 1 + callees[i]->getLastDepth());\r
+                       calleeDepth = callees[i]->getLastDepth();\r
             break;\r
         case PreVisit:\r
-                       callDepth = std::max(callDepth, 1 + callees[i]->analyzeCallDepth(analyzeCallDepth));\r
+                       calleeDepth = callees[i]->analyzeCallDepth(analyzeCallDepth);\r
                        break;\r
         default:\r
             UNREACHABLE(callees[i]->visit);\r
             break;\r
         }\r
+               if(calleeDepth != UINT_MAX) ++calleeDepth;\r
+               callDepth = std::max(callDepth, calleeDepth);\r
     }\r
 \r
     visit = PostVisit;\r
@@ -156,11 +159,13 @@ unsigned int AnalyzeCallDepth::analyzeCallDepth()
                return 0;\r
        }\r
 \r
-    unsigned int depth = 1 + main->analyzeCallDepth(this);\r
+    unsigned int depth = main->analyzeCallDepth(this);\r
+       if(depth != UINT_MAX) ++depth;\r
 \r
        for(FunctionSet::iterator globalCall = globalFunctionCalls.begin(); globalCall != globalFunctionCalls.end(); globalCall++)\r
        {\r
-               unsigned int globalDepth = 1 + (*globalCall)->analyzeCallDepth(this);\r
+               unsigned int globalDepth = (*globalCall)->analyzeCallDepth(this);\r
+               if(globalDepth != UINT_MAX) ++globalDepth;\r
 \r
                if(globalDepth > depth)\r
                {\r