tgsi: fix mixed data type comparison in tgsi_point_sprite.c

Cast the unsigned semantic index to integer datatype before comparing
to max_generic, otherwise, max_generic which is initialized to -1
will be converted to unsigned int before the comparison, causing a wrong
semantic index to be assigned to a shader output.

Fixes the assert running TurboCAD_gl.trace. (VMware bug 1667265)

Also tested with glretrace, mesa demos pointblast, spriteblast and pointcoord.

v2: use the original max_generic variable but add the (int) cast
    to the semantic index, as suggested by Brian.

Reviewed-by: Brian Paul <brianp@vmware.com>

diff --git a/src/gallium/auxiliary/tgsi/tgsi_point_sprite.c b/src/gallium/auxiliary/tgsi/tgsi_point_sprite.c
index cb8dbcb..713bd60 100644 (file)
--- a/src/gallium/auxiliary/tgsi/tgsi_point_sprite.c
+++ b/src/gallium/auxiliary/tgsi/tgsi_point_sprite.c
@@ -96,7 +96,7 @@ struct psprite_transform_context
    unsigned stream_out_point_pos:1; // set if to stream out original point pos
    unsigned aa_point:1;             // set if doing aa point
    unsigned out_tmp_index[PIPE_MAX_SHADER_OUTPUTS];
-   int max_generic;
+   int max_generic;                 // max generic semantic index
 };
 
 static inline struct psprite_transform_context *
@@ -133,7 +133,7 @@ psprite_decl(struct tgsi_transform_context *ctx,
       else if (decl->Semantic.Name == TGSI_SEMANTIC_GENERIC &&
                decl->Semantic.Index < 32) {
          ts->point_coord_decl |= 1 << decl->Semantic.Index;
-         ts->max_generic = MAX2(ts->max_generic, decl->Semantic.Index);
+         ts->max_generic = MAX2(ts->max_generic, (int)decl->Semantic.Index);
       }
       ts->num_out = MAX2(ts->num_out, decl->Range.Last + 1);
    }
@@ -216,7 +216,7 @@ psprite_prolog(struct tgsi_transform_context *ctx)
          if (en & 0x1) {
             tgsi_transform_output_decl(ctx, ts->num_out++,
                                        TGSI_SEMANTIC_GENERIC, i, 0);
-            ts->max_generic = MAX2(ts->max_generic, i);
+            ts->max_generic = MAX2(ts->max_generic, (int)i);
          }
       }
    }