src/mesa/drivers/dri/r300/compiler/radeon_dataflow.c

   1 /*
   2  * Copyright (C) 2009 Nicolai Haehnle.
   3  * Copyright 2010 Tom Stellard <tstellar@gmail.com>
   4  *
   5  * All Rights Reserved.
   6  *
   7  * Permission is hereby granted, free of charge, to any person obtaining
   8  * a copy of this software and associated documentation files (the
   9  * "Software"), to deal in the Software without restriction, including
  10  * without limitation the rights to use, copy, modify, merge, publish,
  11  * distribute, sublicense, and/or sell copies of the Software, and to
  12  * permit persons to whom the Software is furnished to do so, subject to
  13  * the following conditions:
  14  *
  15  * The above copyright notice and this permission notice (including the
  16  * next paragraph) shall be included in all copies or substantial
  17  * portions of the Software.
  18  *
  19  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  20  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  21  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
  22  * IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
  23  * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
  24  * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
  25  * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  26  *
  27  */
  28
  29 #include "radeon_dataflow.h"
  30
  31 #include "radeon_compiler.h"
  32 #include "radeon_compiler_util.h"
  33 #include "radeon_program.h"
  34
  35 struct read_write_mask_data {
  36         void * UserData;
  37         rc_read_write_mask_fn Cb;
  38 };
  39
  40 static void reads_normal_callback(
  41         void * userdata,
  42         struct rc_instruction * fullinst,
  43         struct rc_src_register * src)
  44 {
  45         struct read_write_mask_data * cb_data = userdata;
  46         unsigned int refmask = 0;
  47         unsigned int chan;
  48         for(chan = 0; chan < 4; chan++) {
  49                 refmask |= 1 << GET_SWZ(src->Swizzle, chan);
  50         }
  51         refmask &= RC_MASK_XYZW;
  52
  53         if (refmask) {
  54                 cb_data->Cb(cb_data->UserData, fullinst, src->File,
  55                                                         src->Index, refmask);
  56         }
  57
  58         if (refmask && src->RelAddr) {
  59                 cb_data->Cb(cb_data->UserData, fullinst, RC_FILE_ADDRESS, 0,
  60                                                                 RC_MASK_X);
  61         }
  62 }
  63
  64 static void pair_get_src_refmasks(unsigned int * refmasks,
  65                                         struct rc_pair_instruction * inst,
  66                                         unsigned int swz, unsigned int src)
  67 {
  68         if (swz >= 4)
  69                 return;
  70
  71         if (swz == RC_SWIZZLE_X || swz == RC_SWIZZLE_Y || swz == RC_SWIZZLE_Z) {
  72                 if(src == RC_PAIR_PRESUB_SRC) {
  73                         unsigned int i;
  74                         int srcp_regs =
  75                                 rc_presubtract_src_reg_count(
  76                                 inst->RGB.Src[src].Index);
  77                         for(i = 0; i < srcp_regs; i++) {
  78                                 refmasks[i] |= 1 << swz;
  79                         }
  80                 }
  81                 else {
  82                         refmasks[src] |= 1 << swz;
  83                 }
  84         }
  85
  86         if (swz == RC_SWIZZLE_W) {
  87                 if (src == RC_PAIR_PRESUB_SRC) {
  88                         unsigned int i;
  89                         int srcp_regs = rc_presubtract_src_reg_count(
  90                                         inst->Alpha.Src[src].Index);
  91                         for(i = 0; i < srcp_regs; i++) {
  92                                 refmasks[i] |= 1 << swz;
  93                         }
  94                 }
  95                 else {
  96                         refmasks[src] |= 1 << swz;
  97                 }
  98         }
  99 }
 100
 101 static void reads_pair(struct rc_instruction * fullinst, rc_read_write_mask_fn cb, void * userdata)
 102 {
 103         struct rc_pair_instruction * inst = &fullinst->U.P;
 104         unsigned int refmasks[3] = { 0, 0, 0 };
 105
 106         unsigned int arg;
 107
 108         for(arg = 0; arg < 3; ++arg) {
 109                 unsigned int chan;
 110                 for(chan = 0; chan < 3; ++chan) {
 111                         unsigned int swz_rgb =
 112                                 GET_SWZ(inst->RGB.Arg[arg].Swizzle, chan);
 113                         unsigned int swz_alpha =
 114                                 GET_SWZ(inst->Alpha.Arg[arg].Swizzle, chan);
 115                         pair_get_src_refmasks(refmasks, inst, swz_rgb,
 116                                                 inst->RGB.Arg[arg].Source);
 117                         pair_get_src_refmasks(refmasks, inst, swz_alpha,
 118                                                 inst->Alpha.Arg[arg].Source);
 119                 }
 120         }
 121
 122         for(unsigned int src = 0; src < 3; ++src) {
 123                 if (inst->RGB.Src[src].Used && (refmasks[src] & RC_MASK_XYZ))
 124                         cb(userdata, fullinst, inst->RGB.Src[src].File, inst->RGB.Src[src].Index,
 125                            refmasks[src] & RC_MASK_XYZ);
 126
 127                 if (inst->Alpha.Src[src].Used && (refmasks[src] & RC_MASK_W))
 128                         cb(userdata, fullinst, inst->Alpha.Src[src].File, inst->Alpha.Src[src].Index, RC_MASK_W);
 129         }
 130 }
 131
 132 static void pair_sub_for_all_args(
 133         struct rc_instruction * fullinst,
 134         struct rc_pair_sub_instruction * sub,
 135         rc_pair_read_arg_fn cb,
 136         void * userdata)
 137 {
 138         int i;
 139         const struct rc_opcode_info * info = rc_get_opcode_info(sub->Opcode);
 140
 141         for(i = 0; i < info->NumSrcRegs; i++) {
 142                 unsigned int src_type;
 143
 144                 src_type = rc_source_type_swz(sub->Arg[i].Swizzle);
 145
 146                 if (src_type == RC_SOURCE_NONE)
 147                         continue;
 148
 149                 if (sub->Arg[i].Source == RC_PAIR_PRESUB_SRC) {
 150                         unsigned int presub_type;
 151                         unsigned int presub_src_count;
 152                         struct rc_pair_instruction_source * src_array;
 153                         unsigned int j;
 154 //                      fprintf(stderr, "Presubtract reader\n");
 155                         if (src_type & RC_SOURCE_RGB) {
 156                                 presub_type = fullinst->
 157                                         U.P.RGB.Src[RC_PAIR_PRESUB_SRC].Index;
 158                                 src_array = fullinst->U.P.RGB.Src;
 159                         } else {
 160                                 presub_type = fullinst->
 161                                         U.P.Alpha.Src[RC_PAIR_PRESUB_SRC].Index;
 162                                 src_array = fullinst->U.P.Alpha.Src;
 163                         }
 164                         presub_src_count
 165                                 = rc_presubtract_src_reg_count(presub_type);
 166                         for(j = 0; j < presub_src_count; j++) {
 167                                 cb(userdata, fullinst, &sub->Arg[i],
 168                                                                 &src_array[j]);
 169 //                              fprintf(stderr, "Callback for presub %u type=%u\n", j, src_type);
 170                         }
 171 //                      fprintf(stderr, "Done presubtract reader\n");
 172                 } else {
 173                         struct rc_pair_instruction_source * src =
 174                                 rc_pair_get_src(&fullinst->U.P, &sub->Arg[i]);
 175                         if (src) {
 176                                 cb(userdata, fullinst, &sub->Arg[i], src);
 177                         }
 178                 }
 179         }
 180 }
 181
 182 /* This function calls the callback function (cb) for each source used by
 183  * the instruction.
 184  * */
 185 void rc_for_all_reads_src(
 186         struct rc_instruction * inst,
 187         rc_read_src_fn cb,
 188         void * userdata)
 189 {
 190         const struct rc_opcode_info * opcode =
 191                                         rc_get_opcode_info(inst->U.I.Opcode);
 192
 193         /* This function only works with normal instructions. */
 194         if (inst->Type != RC_INSTRUCTION_NORMAL) {
 195                 assert(0);
 196                 return;
 197         }
 198
 199         for(unsigned int src = 0; src < opcode->NumSrcRegs; ++src) {
 200
 201                 if (inst->U.I.SrcReg[src].File == RC_FILE_NONE)
 202                         continue;
 203
 204                 if (inst->U.I.SrcReg[src].File == RC_FILE_PRESUB) {
 205                         unsigned int i;
 206                         unsigned int srcp_regs = rc_presubtract_src_reg_count(
 207                                                 inst->U.I.PreSub.Opcode);
 208                         for( i = 0; i < srcp_regs; i++) {
 209                                 cb(userdata, inst, &inst->U.I.PreSub.SrcReg[i]);
 210                         }
 211                 } else {
 212                         cb(userdata, inst, &inst->U.I.SrcReg[src]);
 213                 }
 214         }
 215 }
 216
 217 /**
 218  * This function calls the callback function (cb) for each arg of the RGB and
 219  * alpha components.
 220  */
 221 void rc_pair_for_all_reads_arg(
 222         struct rc_instruction * inst,
 223         rc_pair_read_arg_fn cb,
 224         void * userdata)
 225 {
 226         /* This function only works with pair instructions. */
 227         if (inst->Type != RC_INSTRUCTION_PAIR) {
 228                 assert(0);
 229                 return;
 230         }
 231
 232         pair_sub_for_all_args(inst, &inst->U.P.RGB, cb, userdata);
 233         pair_sub_for_all_args(inst, &inst->U.P.Alpha, cb, userdata);
 234 }
 235
 236 /**
 237  * Calls a callback function for all register reads.
 238  *
 239  * This is conservative, i.e. if the same register is referenced multiple times,
 240  * the callback may also be called multiple times.
 241  * Also, the writemask of the instruction is not taken into account.
 242  */
 243 void rc_for_all_reads_mask(struct rc_instruction * inst, rc_read_write_mask_fn cb, void * userdata)
 244 {
 245         if (inst->Type == RC_INSTRUCTION_NORMAL) {
 246                 struct read_write_mask_data cb_data;
 247                 cb_data.UserData = userdata;
 248                 cb_data.Cb = cb;
 249
 250                 rc_for_all_reads_src(inst, reads_normal_callback, &cb_data);
 251         } else {
 252                 reads_pair(inst, cb, userdata);
 253         }
 254 }
 255
 256
 257
 258 static void writes_normal(struct rc_instruction * fullinst, rc_read_write_mask_fn cb, void * userdata)
 259 {
 260         struct rc_sub_instruction * inst = &fullinst->U.I;
 261         const struct rc_opcode_info * opcode = rc_get_opcode_info(inst->Opcode);
 262
 263         if (opcode->HasDstReg && inst->DstReg.WriteMask)
 264                 cb(userdata, fullinst, inst->DstReg.File, inst->DstReg.Index, inst->DstReg.WriteMask);
 265
 266         if (inst->WriteALUResult)
 267                 cb(userdata, fullinst, RC_FILE_SPECIAL, RC_SPECIAL_ALU_RESULT, RC_MASK_X);
 268 }
 269
 270 static void writes_pair(struct rc_instruction * fullinst, rc_read_write_mask_fn cb, void * userdata)
 271 {
 272         struct rc_pair_instruction * inst = &fullinst->U.P;
 273
 274         if (inst->RGB.WriteMask)
 275                 cb(userdata, fullinst, RC_FILE_TEMPORARY, inst->RGB.DestIndex, inst->RGB.WriteMask);
 276
 277         if (inst->Alpha.WriteMask)
 278                 cb(userdata, fullinst, RC_FILE_TEMPORARY, inst->Alpha.DestIndex, RC_MASK_W);
 279
 280         if (inst->WriteALUResult)
 281                 cb(userdata, fullinst, RC_FILE_SPECIAL, RC_SPECIAL_ALU_RESULT, RC_MASK_X);
 282 }
 283
 284 /**
 285  * Calls a callback function for all register writes in the instruction,
 286  * reporting writemasks to the callback function.
 287  *
 288  * \warning Does not report output registers for paired instructions!
 289  */
 290 void rc_for_all_writes_mask(struct rc_instruction * inst, rc_read_write_mask_fn cb, void * userdata)
 291 {
 292         if (inst->Type == RC_INSTRUCTION_NORMAL) {
 293                 writes_normal(inst, cb, userdata);
 294         } else {
 295                 writes_pair(inst, cb, userdata);
 296         }
 297 }
 298
 299
 300 struct mask_to_chan_data {
 301         void * UserData;
 302         rc_read_write_chan_fn Fn;
 303 };
 304
 305 static void mask_to_chan_cb(void * data, struct rc_instruction * inst,
 306                 rc_register_file file, unsigned int index, unsigned int mask)
 307 {
 308         struct mask_to_chan_data * d = data;
 309         for(unsigned int chan = 0; chan < 4; ++chan) {
 310                 if (GET_BIT(mask, chan))
 311                         d->Fn(d->UserData, inst, file, index, chan);
 312         }
 313 }
 314
 315 /**
 316  * Calls a callback function for all sourced register channels.
 317  *
 318  * This is conservative, i.e. channels may be called multiple times,
 319  * and the writemask of the instruction is not taken into account.
 320  */
 321 void rc_for_all_reads_chan(struct rc_instruction * inst, rc_read_write_chan_fn cb, void * userdata)
 322 {
 323         struct mask_to_chan_data d;
 324         d.UserData = userdata;
 325         d.Fn = cb;
 326         rc_for_all_reads_mask(inst, &mask_to_chan_cb, &d);
 327 }
 328
 329 /**
 330  * Calls a callback function for all written register channels.
 331  *
 332  * \warning Does not report output registers for paired instructions!
 333  */
 334 void rc_for_all_writes_chan(struct rc_instruction * inst, rc_read_write_chan_fn cb, void * userdata)
 335 {
 336         struct mask_to_chan_data d;
 337         d.UserData = userdata;
 338         d.Fn = cb;
 339         rc_for_all_writes_mask(inst, &mask_to_chan_cb, &d);
 340 }
 341
 342 static void remap_normal_instruction(struct rc_instruction * fullinst,
 343                 rc_remap_register_fn cb, void * userdata)
 344 {
 345         struct rc_sub_instruction * inst = &fullinst->U.I;
 346         const struct rc_opcode_info * opcode = rc_get_opcode_info(inst->Opcode);
 347         unsigned int remapped_presub = 0;
 348
 349         if (opcode->HasDstReg) {
 350                 rc_register_file file = inst->DstReg.File;
 351                 unsigned int index = inst->DstReg.Index;
 352
 353                 cb(userdata, fullinst, &file, &index);
 354
 355                 inst->DstReg.File = file;
 356                 inst->DstReg.Index = index;
 357         }
 358
 359         for(unsigned int src = 0; src < opcode->NumSrcRegs; ++src) {
 360                 rc_register_file file = inst->SrcReg[src].File;
 361                 unsigned int index = inst->SrcReg[src].Index;
 362
 363                 if (file == RC_FILE_PRESUB) {
 364                         unsigned int i;
 365                         unsigned int srcp_srcs = rc_presubtract_src_reg_count(
 366                                                 inst->PreSub.Opcode);
 367                         /* Make sure we only remap presubtract sources once in
 368                          * case more than one source register reads the
 369                          * presubtract result. */
 370                         if (remapped_presub)
 371                                 continue;
 372
 373                         for(i = 0; i < srcp_srcs; i++) {
 374                                 file = inst->PreSub.SrcReg[i].File;
 375                                 index = inst->PreSub.SrcReg[i].Index;
 376                                 cb(userdata, fullinst, &file, &index);
 377                                 inst->PreSub.SrcReg[i].File = file;
 378                                 inst->PreSub.SrcReg[i].Index = index;
 379                         }
 380                         remapped_presub = 1;
 381                 }
 382                 else {
 383                         cb(userdata, fullinst, &file, &index);
 384
 385                         inst->SrcReg[src].File = file;
 386                         inst->SrcReg[src].Index = index;
 387                 }
 388         }
 389 }
 390
 391 static void remap_pair_instruction(struct rc_instruction * fullinst,
 392                 rc_remap_register_fn cb, void * userdata)
 393 {
 394         struct rc_pair_instruction * inst = &fullinst->U.P;
 395
 396         if (inst->RGB.WriteMask) {
 397                 rc_register_file file = RC_FILE_TEMPORARY;
 398                 unsigned int index = inst->RGB.DestIndex;
 399
 400                 cb(userdata, fullinst, &file, &index);
 401
 402                 inst->RGB.DestIndex = index;
 403         }
 404
 405         if (inst->Alpha.WriteMask) {
 406                 rc_register_file file = RC_FILE_TEMPORARY;
 407                 unsigned int index = inst->Alpha.DestIndex;
 408
 409                 cb(userdata, fullinst, &file, &index);
 410
 411                 inst->Alpha.DestIndex = index;
 412         }
 413
 414         for(unsigned int src = 0; src < 3; ++src) {
 415                 if (inst->RGB.Src[src].Used) {
 416                         rc_register_file file = inst->RGB.Src[src].File;
 417                         unsigned int index = inst->RGB.Src[src].Index;
 418
 419                         cb(userdata, fullinst, &file, &index);
 420
 421                         inst->RGB.Src[src].File = file;
 422                         inst->RGB.Src[src].Index = index;
 423                 }
 424
 425                 if (inst->Alpha.Src[src].Used) {
 426                         rc_register_file file = inst->Alpha.Src[src].File;
 427                         unsigned int index = inst->Alpha.Src[src].Index;
 428
 429                         cb(userdata, fullinst, &file, &index);
 430
 431                         inst->Alpha.Src[src].File = file;
 432                         inst->Alpha.Src[src].Index = index;
 433                 }
 434         }
 435 }
 436
 437
 438 /**
 439  * Remap all register accesses according to the given function.
 440  * That is, call the function \p cb for each referenced register (both read and written)
 441  * and update the given instruction \p inst accordingly
 442  * if it modifies its \ref pfile and \ref pindex contents.
 443  */
 444 void rc_remap_registers(struct rc_instruction * inst, rc_remap_register_fn cb, void * userdata)
 445 {
 446         if (inst->Type == RC_INSTRUCTION_NORMAL)
 447                 remap_normal_instruction(inst, cb, userdata);
 448         else
 449                 remap_pair_instruction(inst, cb, userdata);
 450 }
 451
 452 struct branch_write_mask {
 453         unsigned int IfWriteMask:4;
 454         unsigned int ElseWriteMask:4;
 455         unsigned int HasElse:1;
 456 };
 457
 458 union get_readers_read_cb {
 459         rc_read_src_fn I;
 460         rc_pair_read_arg_fn P;
 461 };
 462
 463 struct get_readers_callback_data {
 464         struct radeon_compiler * C;
 465         struct rc_reader_data * ReaderData;
 466         rc_read_src_fn ReadNormalCB;
 467         rc_pair_read_arg_fn ReadPairCB;
 468         rc_read_write_mask_fn WriteCB;
 469         rc_register_file DstFile;
 470         unsigned int DstIndex;
 471         unsigned int DstMask;
 472         unsigned int AliveWriteMask;
 473         /*  For convenience, this is indexed starting at 1 */
 474         struct branch_write_mask BranchMasks[R500_PFS_MAX_BRANCH_DEPTH_FULL + 1];
 475 };
 476
 477 static struct rc_reader * add_reader(
 478         struct memory_pool * pool,
 479         struct rc_reader_data * data,
 480         struct rc_instruction * inst,
 481         unsigned int mask)
 482 {
 483         struct rc_reader * new;
 484         memory_pool_array_reserve(pool, struct rc_reader, data->Readers,
 485                                 data->ReaderCount, data->ReadersReserved, 1);
 486         new = &data->Readers[data->ReaderCount++];
 487         new->Inst = inst;
 488         new->WriteMask = mask;
 489         return new;
 490 }
 491
 492 static void add_reader_normal(
 493         struct memory_pool * pool,
 494         struct rc_reader_data * data,
 495         struct rc_instruction * inst,
 496         unsigned int mask,
 497         struct rc_src_register * src)
 498 {
 499         struct rc_reader * new = add_reader(pool, data, inst, mask);
 500         new->U.I.Src = src;
 501 }
 502
 503
 504 static void add_reader_pair(
 505         struct memory_pool * pool,
 506         struct rc_reader_data * data,
 507         struct rc_instruction * inst,
 508         unsigned int mask,
 509         struct rc_pair_instruction_arg * arg,
 510         struct rc_pair_instruction_source * src)
 511 {
 512         struct rc_reader * new = add_reader(pool, data, inst, mask);
 513         new->U.P.Src = src;
 514         new->U.P.Arg = arg;
 515 }
 516
 517 static unsigned int get_readers_read_callback(
 518         struct get_readers_callback_data * cb_data,
 519         unsigned int has_rel_addr,
 520         rc_register_file file,
 521         unsigned int index,
 522         unsigned int swizzle)
 523 {
 524         unsigned int shared_mask, read_mask;
 525
 526         if (has_rel_addr) {
 527                 cb_data->ReaderData->Abort = 1;
 528                 return RC_MASK_NONE;
 529         }
 530
 531         shared_mask = rc_src_reads_dst_mask(file, index, swizzle,
 532                 cb_data->DstFile, cb_data->DstIndex, cb_data->AliveWriteMask);
 533
 534         if (shared_mask == RC_MASK_NONE)
 535                 return shared_mask;
 536
 537         /* If we make it this far, it means that this source reads from the
 538          * same register written to by d->ReaderData->Writer. */
 539
 540         read_mask = rc_swizzle_to_writemask(swizzle);
 541         if (cb_data->ReaderData->AbortOnRead & read_mask) {
 542                 cb_data->ReaderData->Abort = 1;
 543                 return shared_mask;
 544         }
 545
 546         if (cb_data->ReaderData->LoopDepth > 0) {
 547                 cb_data->ReaderData->AbortOnWrite |=
 548                                 (read_mask & cb_data->AliveWriteMask);
 549         }
 550
 551         /* XXX The behavior in this case should be configurable. */
 552         if ((read_mask & cb_data->AliveWriteMask) != read_mask) {
 553                 cb_data->ReaderData->Abort = 1;
 554                 return shared_mask;
 555         }
 556
 557         return shared_mask;
 558 }
 559
 560 static void get_readers_pair_read_callback(
 561         void * userdata,
 562         struct rc_instruction * inst,
 563         struct rc_pair_instruction_arg * arg,
 564         struct rc_pair_instruction_source * src)
 565 {
 566         unsigned int shared_mask;
 567         struct get_readers_callback_data * d = userdata;
 568
 569         shared_mask = get_readers_read_callback(d,
 570                                 0 /*Pair Instructions don't use RelAddr*/,
 571                                 src->File, src->Index, arg->Swizzle);
 572
 573 //      fprintf(stderr, "Shared mask = %u for [%u].%u writemask=%u abort=%u exit=%u\n",
 574 //                      shared_mask, src->Index, arg->Swizzle, d->AliveWriteMask,d->ReaderData->Abort, d->ReaderData->ExitOnAbort);
 575         if (shared_mask == RC_MASK_NONE)
 576                 return;
 577
 578         if (d->ReadPairCB)
 579                 d->ReadPairCB(d->ReaderData, inst, arg, src);
 580
 581         if (d->ReaderData->ExitOnAbort && d->ReaderData->Abort)
 582                 return;
 583
 584         add_reader_pair(&d->C->Pool, d->ReaderData, inst, shared_mask, arg, src);
 585 }
 586
 587 /**
 588  * This function is used by rc_get_readers_normal() to determine whether inst
 589  * is a reader of userdata->ReaderData->Writer
 590  */
 591 static void get_readers_normal_read_callback(
 592         void * userdata,
 593         struct rc_instruction * inst,
 594         struct rc_src_register * src)
 595 {
 596         struct get_readers_callback_data * d = userdata;
 597         unsigned int shared_mask;
 598
 599         shared_mask = get_readers_read_callback(d,
 600                         src->RelAddr, src->File, src->Index, src->Swizzle);
 601
 602         if (shared_mask == RC_MASK_NONE)
 603                 return;
 604         /* The callback function could potentially clear d->ReaderData->Abort,
 605          * so we need to call it before we return. */
 606         if (d->ReadNormalCB)
 607                 d->ReadNormalCB(d->ReaderData, inst, src);
 608
 609         if (d->ReaderData->ExitOnAbort && d->ReaderData->Abort)
 610                 return;
 611
 612         add_reader_normal(&d->C->Pool, d->ReaderData, inst, shared_mask, src);
 613 }
 614
 615 /**
 616  * This function is used by rc_get_readers_normal() to determine when
 617  * userdata->ReaderData->Writer is dead (i. e. All compontents of its
 618  * destination register have been overwritten by other instructions).
 619  */
 620 static void get_readers_write_callback(
 621         void *userdata,
 622         struct rc_instruction * inst,
 623         rc_register_file file,
 624         unsigned int index,
 625         unsigned int mask)
 626 {
 627         struct get_readers_callback_data * d = userdata;
 628
 629         if (index == d->DstIndex && file == d->DstFile) {
 630                 unsigned int shared_mask = mask & d->DstMask;
 631                 d->ReaderData->AbortOnRead &= ~shared_mask;
 632                 d->AliveWriteMask &= ~shared_mask;
 633                 if (d->ReaderData->AbortOnWrite & shared_mask) {
 634                         d->ReaderData->Abort = 1;
 635                 }
 636         }
 637
 638         if(d->WriteCB)
 639                 d->WriteCB(d->ReaderData, inst, file, index, mask);
 640 }
 641
 642 static void push_branch_mask(
 643         struct get_readers_callback_data * d,
 644         unsigned int * branch_depth)
 645 {
 646         (*branch_depth)++;
 647         if (*branch_depth > R500_PFS_MAX_BRANCH_DEPTH_FULL) {
 648                 d->ReaderData->Abort = 1;
 649                 return;
 650         }
 651         d->BranchMasks[*branch_depth].IfWriteMask =
 652                                         d->AliveWriteMask;
 653 }
 654
 655 static void pop_branch_mask(
 656         struct get_readers_callback_data * d,
 657         unsigned int * branch_depth)
 658 {
 659         struct branch_write_mask * masks = &d->BranchMasks[*branch_depth];
 660
 661         if (masks->HasElse) {
 662                 /* Abort on read for components that were written in the IF
 663                  * block. */
 664                 d->ReaderData->AbortOnRead |=
 665                                 masks->IfWriteMask & ~masks->ElseWriteMask;
 666                 /* Abort on read for components that were written in the ELSE
 667                  * block. */
 668                 d->ReaderData->AbortOnRead |=
 669                                 masks->ElseWriteMask & ~d->AliveWriteMask;
 670
 671                 d->AliveWriteMask = masks->IfWriteMask
 672                         ^ ((masks->IfWriteMask ^ masks->ElseWriteMask)
 673                         & (masks->IfWriteMask ^ d->AliveWriteMask));
 674         } else {
 675                 d->ReaderData->AbortOnRead |=
 676                                 masks->IfWriteMask & ~d->AliveWriteMask;
 677                 d->AliveWriteMask = masks->IfWriteMask;
 678
 679         }
 680         memset(masks, 0, sizeof(struct branch_write_mask));
 681         (*branch_depth)--;
 682 }
 683
 684 static void get_readers_for_single_write(
 685         void * userdata,
 686         struct rc_instruction * writer,
 687         rc_register_file dst_file,
 688         unsigned int dst_index,
 689         unsigned int dst_mask)
 690 {
 691         struct rc_instruction * tmp;
 692         unsigned int branch_depth = 0;
 693         struct rc_instruction * endloop = NULL;
 694         unsigned int abort_on_read_at_endloop;
 695         struct get_readers_callback_data * d = userdata;
 696
 697         d->ReaderData->Writer = writer;
 698         d->ReaderData->AbortOnRead = 0;
 699         d->ReaderData->AbortOnWrite = 0;
 700         d->ReaderData->LoopDepth = 0;
 701         d->ReaderData->InElse = 0;
 702         d->DstFile = dst_file;
 703         d->DstIndex = dst_index;
 704         d->DstMask = dst_mask;
 705         d->AliveWriteMask = dst_mask;
 706         memset(d->BranchMasks, 0, sizeof(d->BranchMasks));
 707
 708         if (!dst_mask)
 709                 return;
 710
 711         for(tmp = writer->Next; tmp != &d->C->Program.Instructions;
 712                                                         tmp = tmp->Next){
 713                 rc_opcode opcode = rc_get_flow_control_inst(tmp);
 714                 switch(opcode) {
 715                 case RC_OPCODE_BGNLOOP:
 716                         d->ReaderData->LoopDepth++;
 717                         push_branch_mask(d, &branch_depth);
 718                         break;
 719                 case RC_OPCODE_ENDLOOP:
 720                         if (d->ReaderData->LoopDepth > 0) {
 721                                 d->ReaderData->LoopDepth--;
 722                                 if (d->ReaderData->LoopDepth == 0) {
 723                                         d->ReaderData->AbortOnWrite = 0;
 724                                 }
 725                                 pop_branch_mask(d, &branch_depth);
 726                         } else {
 727                                 /* Here we have reached an ENDLOOP without
 728                                  * seeing its BGNLOOP.  These means that
 729                                  * the writer was written inside of a loop,
 730                                  * so it could have readers that are above it
 731                                  * (i.e. they have a lower IP).  To find these
 732                                  * readers we jump to the BGNLOOP instruction
 733                                  * and check each instruction until we get
 734                                  * back to the writer.
 735                                  */
 736                                 endloop = tmp;
 737                                 tmp = rc_match_endloop(tmp);
 738                                 if (!tmp) {
 739                                         rc_error(d->C, "Failed to match endloop.\n");
 740                                         d->ReaderData->Abort = 1;
 741                                         return;
 742                                 }
 743                                 abort_on_read_at_endloop = d->ReaderData->AbortOnRead;
 744                                 d->ReaderData->AbortOnRead |= d->AliveWriteMask;
 745                                 continue;
 746                         }
 747                         break;
 748                 case RC_OPCODE_IF:
 749                         push_branch_mask(d, &branch_depth);
 750                         break;
 751                 case RC_OPCODE_ELSE:
 752                         if (branch_depth == 0) {
 753                                 d->ReaderData->InElse = 1;
 754                         } else {
 755                                 unsigned int temp_mask = d->AliveWriteMask;
 756                                 d->AliveWriteMask =
 757                                         d->BranchMasks[branch_depth].IfWriteMask;
 758                                 d->BranchMasks[branch_depth].ElseWriteMask =
 759                                                                 temp_mask;
 760                                 d->BranchMasks[branch_depth].HasElse = 1;
 761                         }
 762                         break;
 763                 case RC_OPCODE_ENDIF:
 764                         if (branch_depth == 0) {
 765                                 d->ReaderData->AbortOnRead = d->AliveWriteMask;
 766                                 d->ReaderData->InElse = 0;
 767                         }
 768                         else {
 769                                 pop_branch_mask(d, &branch_depth);
 770                         }
 771                         break;
 772                 default:
 773                         break;
 774                 }
 775
 776                 if (d->ReaderData->InElse)
 777                         continue;
 778
 779                 if (tmp->Type == RC_INSTRUCTION_NORMAL) {
 780                         rc_for_all_reads_src(tmp,
 781                                 get_readers_normal_read_callback, d);
 782                 } else {
 783                         rc_pair_for_all_reads_arg(tmp,
 784                                 get_readers_pair_read_callback, d);
 785                 }
 786
 787                 /* This can happen when we jump from an ENDLOOP to BGNLOOP */
 788                 if (tmp == writer) {
 789                         tmp = endloop;
 790                         endloop = NULL;
 791                         d->ReaderData->AbortOnRead = abort_on_read_at_endloop;
 792                         continue;
 793                 }
 794                 rc_for_all_writes_mask(tmp, get_readers_write_callback, d);
 795
 796                 if (d->ReaderData->ExitOnAbort && d->ReaderData->Abort)
 797                         return;
 798
 799                 if (branch_depth == 0 && !d->AliveWriteMask)
 800                         return;
 801         }
 802 }
 803
 804 static void init_get_readers_callback_data(
 805         struct get_readers_callback_data * d,
 806         struct rc_reader_data * reader_data,
 807         struct radeon_compiler * c,
 808         rc_read_src_fn read_normal_cb,
 809         rc_pair_read_arg_fn read_pair_cb,
 810         rc_read_write_mask_fn write_cb)
 811 {
 812         reader_data->Abort = 0;
 813         reader_data->ReaderCount = 0;
 814         reader_data->ReadersReserved = 0;
 815         reader_data->Readers = NULL;
 816
 817         d->C = c;
 818         d->ReaderData = reader_data;
 819         d->ReadNormalCB = read_normal_cb;
 820         d->ReadPairCB = read_pair_cb;
 821         d->WriteCB = write_cb;
 822 }
 823
 824 /**
 825  * This function will create a list of readers via the rc_reader_data struct.
 826  * This function will abort (set the flag data->Abort) and return if it
 827  * encounters an instruction that reads from @param writer and also a different
 828  * instruction.  Here are some examples:
 829  *
 830  * writer = instruction 0;
 831  * 0 MOV TEMP[0].xy, TEMP[1].xy
 832  * 1 MOV TEMP[0].zw, TEMP[2].xy
 833  * 2 MOV TEMP[3], TEMP[0]
 834  * The Abort flag will be set on instruction 2, because it reads values written
 835  * by instructions 0 and 1.
 836  *
 837  * writer = instruction 1;
 838  * 0 IF TEMP[0].x
 839  * 1 MOV TEMP[1], TEMP[2]
 840  * 2 ELSE
 841  * 3 MOV TEMP[1], TEMP[2]
 842  * 4 ENDIF
 843  * 5 MOV TEMP[3], TEMP[1]
 844  * The Abort flag will be set on instruction 5, because it could read from the
 845  * value written by either instruction 1 or 3, depending on the jump decision
 846  * made at instruction 0.
 847  *
 848  * writer = instruction 0;
 849  * 0 MOV TEMP[0], TEMP[1]
 850  * 2 BGNLOOP
 851  * 3 ADD TEMP[0], TEMP[0], none.1
 852  * 4 ENDLOOP
 853  * The Abort flag will be set on instruction 3, because in the first iteration
 854  * of the loop it reads the value written by instruction 0 and in all other
 855  * iterations it reads the value written by instruction 3.
 856  *
 857  * @param read_cb This function will be called for for every instruction that
 858  * has been determined to be a reader of writer.
 859  * @param write_cb This function will be called for every instruction after
 860  * writer.
 861  */
 862 void rc_get_readers(
 863         struct radeon_compiler * c,
 864         struct rc_instruction * writer,
 865         struct rc_reader_data * data,
 866         rc_read_src_fn read_normal_cb,
 867         rc_pair_read_arg_fn read_pair_cb,
 868         rc_read_write_mask_fn write_cb)
 869 {
 870         struct get_readers_callback_data d;
 871
 872         init_get_readers_callback_data(&d, data, c, read_normal_cb,
 873                                                 read_pair_cb, write_cb);
 874
 875         rc_for_all_writes_mask(writer, get_readers_for_single_write, &d);
 876 }
 877
 878 void rc_get_readers_sub(
 879         struct radeon_compiler * c,
 880         struct rc_instruction * writer,
 881         struct rc_pair_sub_instruction * sub_writer,
 882         struct rc_reader_data * data,
 883         rc_read_src_fn read_normal_cb,
 884         rc_pair_read_arg_fn read_pair_cb,
 885         rc_read_write_mask_fn write_cb)
 886 {
 887         struct get_readers_callback_data d;
 888
 889         init_get_readers_callback_data(&d, data, c, read_normal_cb,
 890                                                 read_pair_cb, write_cb);
 891
 892         if (sub_writer->WriteMask) {
 893                 get_readers_for_single_write(&d, writer, RC_FILE_TEMPORARY,
 894                         sub_writer->DestIndex, sub_writer->WriteMask);
 895         }
 896 }