src/mesa/drivers/dri/r300/compiler/radeon_dataflow.c

   1 /*
   2  * Copyright (C) 2009 Nicolai Haehnle.
   3  * Copyright 2010 Tom Stellard <tstellar@gmail.com>
   4  *
   5  * All Rights Reserved.
   6  *
   7  * Permission is hereby granted, free of charge, to any person obtaining
   8  * a copy of this software and associated documentation files (the
   9  * "Software"), to deal in the Software without restriction, including
  10  * without limitation the rights to use, copy, modify, merge, publish,
  11  * distribute, sublicense, and/or sell copies of the Software, and to
  12  * permit persons to whom the Software is furnished to do so, subject to
  13  * the following conditions:
  14  *
  15  * The above copyright notice and this permission notice (including the
  16  * next paragraph) shall be included in all copies or substantial
  17  * portions of the Software.
  18  *
  19  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  20  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  21  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
  22  * IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
  23  * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
  24  * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
  25  * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  26  *
  27  */
  28
  29 #include "radeon_dataflow.h"
  30
  31 #include "radeon_compiler.h"
  32 #include "radeon_compiler_util.h"
  33 #include "radeon_program.h"
  34
  35 struct read_write_mask_data {
  36         void * UserData;
  37         rc_read_write_mask_fn Cb;
  38 };
  39
  40 static void reads_normal_callback(
  41         void * userdata,
  42         struct rc_instruction * fullinst,
  43         struct rc_src_register * src)
  44 {
  45         struct read_write_mask_data * cb_data = userdata;
  46         unsigned int refmask = 0;
  47         unsigned int chan;
  48         for(chan = 0; chan < 4; chan++) {
  49                 refmask |= 1 << GET_SWZ(src->Swizzle, chan);
  50         }
  51         refmask &= RC_MASK_XYZW;
  52
  53         if (refmask) {
  54                 cb_data->Cb(cb_data->UserData, fullinst, src->File,
  55                                                         src->Index, refmask);
  56         }
  57
  58         if (refmask && src->RelAddr) {
  59                 cb_data->Cb(cb_data->UserData, fullinst, RC_FILE_ADDRESS, 0,
  60                                                                 RC_MASK_X);
  61         }
  62 }
  63
  64 static void pair_get_src_refmasks(unsigned int * refmasks,
  65                                         struct rc_pair_instruction * inst,
  66                                         unsigned int swz, unsigned int src)
  67 {
  68         if (swz >= 4)
  69                 return;
  70
  71         if (swz == RC_SWIZZLE_X || swz == RC_SWIZZLE_Y || swz == RC_SWIZZLE_Z) {
  72                 if(src == RC_PAIR_PRESUB_SRC) {
  73                         unsigned int i;
  74                         int srcp_regs =
  75                                 rc_presubtract_src_reg_count(
  76                                 inst->RGB.Src[src].Index);
  77                         for(i = 0; i < srcp_regs; i++) {
  78                                 refmasks[i] |= 1 << swz;
  79                         }
  80                 }
  81                 else {
  82                         refmasks[src] |= 1 << swz;
  83                 }
  84         }
  85
  86         if (swz == RC_SWIZZLE_W) {
  87                 if (src == RC_PAIR_PRESUB_SRC) {
  88                         unsigned int i;
  89                         int srcp_regs = rc_presubtract_src_reg_count(
  90                                         inst->Alpha.Src[src].Index);
  91                         for(i = 0; i < srcp_regs; i++) {
  92                                 refmasks[i] |= 1 << swz;
  93                         }
  94                 }
  95                 else {
  96                         refmasks[src] |= 1 << swz;
  97                 }
  98         }
  99 }
 100
 101 static void reads_pair(struct rc_instruction * fullinst, rc_read_write_mask_fn cb, void * userdata)
 102 {
 103         struct rc_pair_instruction * inst = &fullinst->U.P;
 104         unsigned int refmasks[3] = { 0, 0, 0 };
 105
 106         unsigned int arg;
 107
 108         for(arg = 0; arg < 3; ++arg) {
 109                 unsigned int chan;
 110                 for(chan = 0; chan < 3; ++chan) {
 111                         unsigned int swz_rgb =
 112                                 GET_SWZ(inst->RGB.Arg[arg].Swizzle, chan);
 113                         unsigned int swz_alpha =
 114                                 GET_SWZ(inst->Alpha.Arg[arg].Swizzle, chan);
 115                         pair_get_src_refmasks(refmasks, inst, swz_rgb,
 116                                                 inst->RGB.Arg[arg].Source);
 117                         pair_get_src_refmasks(refmasks, inst, swz_alpha,
 118                                                 inst->Alpha.Arg[arg].Source);
 119                 }
 120         }
 121
 122         for(unsigned int src = 0; src < 3; ++src) {
 123                 if (inst->RGB.Src[src].Used && (refmasks[src] & RC_MASK_XYZ))
 124                         cb(userdata, fullinst, inst->RGB.Src[src].File, inst->RGB.Src[src].Index,
 125                            refmasks[src] & RC_MASK_XYZ);
 126
 127                 if (inst->Alpha.Src[src].Used && (refmasks[src] & RC_MASK_W))
 128                         cb(userdata, fullinst, inst->Alpha.Src[src].File, inst->Alpha.Src[src].Index, RC_MASK_W);
 129         }
 130 }
 131
 132 static void pair_sub_for_all_args(
 133         struct rc_instruction * fullinst,
 134         struct rc_pair_sub_instruction * sub,
 135         rc_pair_read_arg_fn cb,
 136         void * userdata)
 137 {
 138         int i;
 139         const struct rc_opcode_info * info = rc_get_opcode_info(sub->Opcode);
 140
 141         for(i = 0; i < info->NumSrcRegs; i++) {
 142                 unsigned int src_type;
 143
 144                 src_type = rc_source_type_swz(sub->Arg[i].Swizzle);
 145
 146                 if (src_type == RC_SOURCE_NONE)
 147                         continue;
 148
 149                 if (sub->Arg[i].Source == RC_PAIR_PRESUB_SRC) {
 150                         unsigned int presub_type;
 151                         unsigned int presub_src_count;
 152                         struct rc_pair_instruction_source * src_array;
 153                         unsigned int j;
 154                         if (src_type & RC_SOURCE_RGB) {
 155                                 presub_type = fullinst->
 156                                         U.P.RGB.Src[RC_PAIR_PRESUB_SRC].Index;
 157                                 src_array = fullinst->U.P.RGB.Src;
 158                         } else {
 159                                 presub_type = fullinst->
 160                                         U.P.Alpha.Src[RC_PAIR_PRESUB_SRC].Index;
 161                                 src_array = fullinst->U.P.Alpha.Src;
 162                         }
 163                         presub_src_count
 164                                 = rc_presubtract_src_reg_count(presub_type);
 165                         for(j = 0; j < presub_src_count; j++) {
 166                                 cb(userdata, fullinst, &sub->Arg[i],
 167                                                                 &src_array[j]);
 168                         }
 169                 } else {
 170                         struct rc_pair_instruction_source * src =
 171                                 rc_pair_get_src(&fullinst->U.P, &sub->Arg[i]);
 172                         if (src) {
 173                                 cb(userdata, fullinst, &sub->Arg[i], src);
 174                         }
 175                 }
 176         }
 177 }
 178
 179 /* This function calls the callback function (cb) for each source used by
 180  * the instruction.
 181  * */
 182 void rc_for_all_reads_src(
 183         struct rc_instruction * inst,
 184         rc_read_src_fn cb,
 185         void * userdata)
 186 {
 187         const struct rc_opcode_info * opcode =
 188                                         rc_get_opcode_info(inst->U.I.Opcode);
 189
 190         /* This function only works with normal instructions. */
 191         if (inst->Type != RC_INSTRUCTION_NORMAL) {
 192                 assert(0);
 193                 return;
 194         }
 195
 196         for(unsigned int src = 0; src < opcode->NumSrcRegs; ++src) {
 197
 198                 if (inst->U.I.SrcReg[src].File == RC_FILE_NONE)
 199                         continue;
 200
 201                 if (inst->U.I.SrcReg[src].File == RC_FILE_PRESUB) {
 202                         unsigned int i;
 203                         unsigned int srcp_regs = rc_presubtract_src_reg_count(
 204                                                 inst->U.I.PreSub.Opcode);
 205                         for( i = 0; i < srcp_regs; i++) {
 206                                 cb(userdata, inst, &inst->U.I.PreSub.SrcReg[i]);
 207                         }
 208                 } else {
 209                         cb(userdata, inst, &inst->U.I.SrcReg[src]);
 210                 }
 211         }
 212 }
 213
 214 /**
 215  * This function calls the callback function (cb) for each arg of the RGB and
 216  * alpha components.
 217  */
 218 void rc_pair_for_all_reads_arg(
 219         struct rc_instruction * inst,
 220         rc_pair_read_arg_fn cb,
 221         void * userdata)
 222 {
 223         /* This function only works with pair instructions. */
 224         if (inst->Type != RC_INSTRUCTION_PAIR) {
 225                 assert(0);
 226                 return;
 227         }
 228
 229         pair_sub_for_all_args(inst, &inst->U.P.RGB, cb, userdata);
 230         pair_sub_for_all_args(inst, &inst->U.P.Alpha, cb, userdata);
 231 }
 232
 233 /**
 234  * Calls a callback function for all register reads.
 235  *
 236  * This is conservative, i.e. if the same register is referenced multiple times,
 237  * the callback may also be called multiple times.
 238  * Also, the writemask of the instruction is not taken into account.
 239  */
 240 void rc_for_all_reads_mask(struct rc_instruction * inst, rc_read_write_mask_fn cb, void * userdata)
 241 {
 242         if (inst->Type == RC_INSTRUCTION_NORMAL) {
 243                 struct read_write_mask_data cb_data;
 244                 cb_data.UserData = userdata;
 245                 cb_data.Cb = cb;
 246
 247                 rc_for_all_reads_src(inst, reads_normal_callback, &cb_data);
 248         } else {
 249                 reads_pair(inst, cb, userdata);
 250         }
 251 }
 252
 253
 254
 255 static void writes_normal(struct rc_instruction * fullinst, rc_read_write_mask_fn cb, void * userdata)
 256 {
 257         struct rc_sub_instruction * inst = &fullinst->U.I;
 258         const struct rc_opcode_info * opcode = rc_get_opcode_info(inst->Opcode);
 259
 260         if (opcode->HasDstReg && inst->DstReg.WriteMask)
 261                 cb(userdata, fullinst, inst->DstReg.File, inst->DstReg.Index, inst->DstReg.WriteMask);
 262
 263         if (inst->WriteALUResult)
 264                 cb(userdata, fullinst, RC_FILE_SPECIAL, RC_SPECIAL_ALU_RESULT, RC_MASK_X);
 265 }
 266
 267 static void writes_pair(struct rc_instruction * fullinst, rc_read_write_mask_fn cb, void * userdata)
 268 {
 269         struct rc_pair_instruction * inst = &fullinst->U.P;
 270
 271         if (inst->RGB.WriteMask)
 272                 cb(userdata, fullinst, RC_FILE_TEMPORARY, inst->RGB.DestIndex, inst->RGB.WriteMask);
 273
 274         if (inst->Alpha.WriteMask)
 275                 cb(userdata, fullinst, RC_FILE_TEMPORARY, inst->Alpha.DestIndex, RC_MASK_W);
 276
 277         if (inst->WriteALUResult)
 278                 cb(userdata, fullinst, RC_FILE_SPECIAL, RC_SPECIAL_ALU_RESULT, RC_MASK_X);
 279 }
 280
 281 /**
 282  * Calls a callback function for all register writes in the instruction,
 283  * reporting writemasks to the callback function.
 284  *
 285  * \warning Does not report output registers for paired instructions!
 286  */
 287 void rc_for_all_writes_mask(struct rc_instruction * inst, rc_read_write_mask_fn cb, void * userdata)
 288 {
 289         if (inst->Type == RC_INSTRUCTION_NORMAL) {
 290                 writes_normal(inst, cb, userdata);
 291         } else {
 292                 writes_pair(inst, cb, userdata);
 293         }
 294 }
 295
 296
 297 struct mask_to_chan_data {
 298         void * UserData;
 299         rc_read_write_chan_fn Fn;
 300 };
 301
 302 static void mask_to_chan_cb(void * data, struct rc_instruction * inst,
 303                 rc_register_file file, unsigned int index, unsigned int mask)
 304 {
 305         struct mask_to_chan_data * d = data;
 306         for(unsigned int chan = 0; chan < 4; ++chan) {
 307                 if (GET_BIT(mask, chan))
 308                         d->Fn(d->UserData, inst, file, index, chan);
 309         }
 310 }
 311
 312 /**
 313  * Calls a callback function for all sourced register channels.
 314  *
 315  * This is conservative, i.e. channels may be called multiple times,
 316  * and the writemask of the instruction is not taken into account.
 317  */
 318 void rc_for_all_reads_chan(struct rc_instruction * inst, rc_read_write_chan_fn cb, void * userdata)
 319 {
 320         struct mask_to_chan_data d;
 321         d.UserData = userdata;
 322         d.Fn = cb;
 323         rc_for_all_reads_mask(inst, &mask_to_chan_cb, &d);
 324 }
 325
 326 /**
 327  * Calls a callback function for all written register channels.
 328  *
 329  * \warning Does not report output registers for paired instructions!
 330  */
 331 void rc_for_all_writes_chan(struct rc_instruction * inst, rc_read_write_chan_fn cb, void * userdata)
 332 {
 333         struct mask_to_chan_data d;
 334         d.UserData = userdata;
 335         d.Fn = cb;
 336         rc_for_all_writes_mask(inst, &mask_to_chan_cb, &d);
 337 }
 338
 339 static void remap_normal_instruction(struct rc_instruction * fullinst,
 340                 rc_remap_register_fn cb, void * userdata)
 341 {
 342         struct rc_sub_instruction * inst = &fullinst->U.I;
 343         const struct rc_opcode_info * opcode = rc_get_opcode_info(inst->Opcode);
 344         unsigned int remapped_presub = 0;
 345
 346         if (opcode->HasDstReg) {
 347                 rc_register_file file = inst->DstReg.File;
 348                 unsigned int index = inst->DstReg.Index;
 349
 350                 cb(userdata, fullinst, &file, &index);
 351
 352                 inst->DstReg.File = file;
 353                 inst->DstReg.Index = index;
 354         }
 355
 356         for(unsigned int src = 0; src < opcode->NumSrcRegs; ++src) {
 357                 rc_register_file file = inst->SrcReg[src].File;
 358                 unsigned int index = inst->SrcReg[src].Index;
 359
 360                 if (file == RC_FILE_PRESUB) {
 361                         unsigned int i;
 362                         unsigned int srcp_srcs = rc_presubtract_src_reg_count(
 363                                                 inst->PreSub.Opcode);
 364                         /* Make sure we only remap presubtract sources once in
 365                          * case more than one source register reads the
 366                          * presubtract result. */
 367                         if (remapped_presub)
 368                                 continue;
 369
 370                         for(i = 0; i < srcp_srcs; i++) {
 371                                 file = inst->PreSub.SrcReg[i].File;
 372                                 index = inst->PreSub.SrcReg[i].Index;
 373                                 cb(userdata, fullinst, &file, &index);
 374                                 inst->PreSub.SrcReg[i].File = file;
 375                                 inst->PreSub.SrcReg[i].Index = index;
 376                         }
 377                         remapped_presub = 1;
 378                 }
 379                 else {
 380                         cb(userdata, fullinst, &file, &index);
 381
 382                         inst->SrcReg[src].File = file;
 383                         inst->SrcReg[src].Index = index;
 384                 }
 385         }
 386 }
 387
 388 static void remap_pair_instruction(struct rc_instruction * fullinst,
 389                 rc_remap_register_fn cb, void * userdata)
 390 {
 391         struct rc_pair_instruction * inst = &fullinst->U.P;
 392
 393         if (inst->RGB.WriteMask) {
 394                 rc_register_file file = RC_FILE_TEMPORARY;
 395                 unsigned int index = inst->RGB.DestIndex;
 396
 397                 cb(userdata, fullinst, &file, &index);
 398
 399                 inst->RGB.DestIndex = index;
 400         }
 401
 402         if (inst->Alpha.WriteMask) {
 403                 rc_register_file file = RC_FILE_TEMPORARY;
 404                 unsigned int index = inst->Alpha.DestIndex;
 405
 406                 cb(userdata, fullinst, &file, &index);
 407
 408                 inst->Alpha.DestIndex = index;
 409         }
 410
 411         for(unsigned int src = 0; src < 3; ++src) {
 412                 if (inst->RGB.Src[src].Used) {
 413                         rc_register_file file = inst->RGB.Src[src].File;
 414                         unsigned int index = inst->RGB.Src[src].Index;
 415
 416                         cb(userdata, fullinst, &file, &index);
 417
 418                         inst->RGB.Src[src].File = file;
 419                         inst->RGB.Src[src].Index = index;
 420                 }
 421
 422                 if (inst->Alpha.Src[src].Used) {
 423                         rc_register_file file = inst->Alpha.Src[src].File;
 424                         unsigned int index = inst->Alpha.Src[src].Index;
 425
 426                         cb(userdata, fullinst, &file, &index);
 427
 428                         inst->Alpha.Src[src].File = file;
 429                         inst->Alpha.Src[src].Index = index;
 430                 }
 431         }
 432 }
 433
 434
 435 /**
 436  * Remap all register accesses according to the given function.
 437  * That is, call the function \p cb for each referenced register (both read and written)
 438  * and update the given instruction \p inst accordingly
 439  * if it modifies its \ref pfile and \ref pindex contents.
 440  */
 441 void rc_remap_registers(struct rc_instruction * inst, rc_remap_register_fn cb, void * userdata)
 442 {
 443         if (inst->Type == RC_INSTRUCTION_NORMAL)
 444                 remap_normal_instruction(inst, cb, userdata);
 445         else
 446                 remap_pair_instruction(inst, cb, userdata);
 447 }
 448
 449 /**
 450  * @return RC_OPCODE_NOOP if inst is not a flow control instruction.
 451  * @return The opcode of inst if it is a flow control instruction.
 452  */
 453 static rc_opcode get_flow_control_inst(struct rc_instruction * inst)
 454 {
 455         const struct rc_opcode_info * info;
 456         if (inst->Type == RC_INSTRUCTION_NORMAL) {
 457                 info = rc_get_opcode_info(inst->U.I.Opcode);
 458         } else {
 459                 info = rc_get_opcode_info(inst->U.P.RGB.Opcode);
 460                 /*A flow control instruction shouldn't have an alpha
 461                  * instruction.*/
 462                 assert(!info->IsFlowControl ||
 463                                 inst->U.P.Alpha.Opcode == RC_OPCODE_NOP);
 464         }
 465
 466         if (info->IsFlowControl)
 467                 return info->Opcode;
 468         else
 469                 return RC_OPCODE_NOP;
 470
 471 }
 472
 473 struct branch_write_mask {
 474         unsigned int IfWriteMask:4;
 475         unsigned int ElseWriteMask:4;
 476         unsigned int HasElse:1;
 477 };
 478
 479 union get_readers_read_cb {
 480         rc_read_src_fn I;
 481         rc_pair_read_arg_fn P;
 482 };
 483
 484 struct get_readers_callback_data {
 485         struct radeon_compiler * C;
 486         struct rc_reader_data * ReaderData;
 487         rc_read_src_fn ReadNormalCB;
 488         rc_pair_read_arg_fn ReadPairCB;
 489         rc_read_write_mask_fn WriteCB;
 490         rc_register_file DstFile;
 491         unsigned int DstIndex;
 492         unsigned int DstMask;
 493         unsigned int AliveWriteMask;
 494         /*  For convenience, this is indexed starting at 1 */
 495         struct branch_write_mask BranchMasks[R500_PFS_MAX_BRANCH_DEPTH_FULL + 1];
 496 };
 497
 498 static struct rc_reader * add_reader(
 499         struct memory_pool * pool,
 500         struct rc_reader_data * data,
 501         struct rc_instruction * inst,
 502         unsigned int mask)
 503 {
 504         struct rc_reader * new;
 505         memory_pool_array_reserve(pool, struct rc_reader, data->Readers,
 506                                 data->ReaderCount, data->ReadersReserved, 1);
 507         new = &data->Readers[data->ReaderCount++];
 508         new->Inst = inst;
 509         new->WriteMask = mask;
 510         return new;
 511 }
 512
 513 static void add_reader_normal(
 514         struct memory_pool * pool,
 515         struct rc_reader_data * data,
 516         struct rc_instruction * inst,
 517         unsigned int mask,
 518         struct rc_src_register * src)
 519 {
 520         struct rc_reader * new = add_reader(pool, data, inst, mask);
 521         new->U.I.Src = src;
 522 }
 523
 524
 525 static void add_reader_pair(
 526         struct memory_pool * pool,
 527         struct rc_reader_data * data,
 528         struct rc_instruction * inst,
 529         unsigned int mask,
 530         struct rc_pair_instruction_arg * arg,
 531         struct rc_pair_instruction_source * src)
 532 {
 533         struct rc_reader * new = add_reader(pool, data, inst, mask);
 534         new->U.P.Src = src;
 535         new->U.P.Arg = arg;
 536 }
 537
 538 static unsigned int get_readers_read_callback(
 539         struct get_readers_callback_data * cb_data,
 540         unsigned int has_rel_addr,
 541         rc_register_file file,
 542         unsigned int index,
 543         unsigned int swizzle)
 544 {
 545         unsigned int shared_mask, read_mask;
 546
 547         if (has_rel_addr) {
 548                 cb_data->ReaderData->Abort = 1;
 549                 return RC_MASK_NONE;
 550         }
 551
 552         shared_mask = rc_src_reads_dst_mask(file, index, swizzle,
 553                 cb_data->DstFile, cb_data->DstIndex, cb_data->AliveWriteMask);
 554
 555         if (shared_mask == RC_MASK_NONE)
 556                 return shared_mask;
 557
 558         /* If we make it this far, it means that this source reads from the
 559          * same register written to by d->ReaderData->Writer. */
 560
 561         read_mask = rc_swizzle_to_writemask(swizzle);
 562         if (cb_data->ReaderData->AbortOnRead & read_mask) {
 563                 cb_data->ReaderData->Abort = 1;
 564                 return shared_mask;
 565         }
 566
 567         /* XXX The behavior in this case should be configurable. */
 568         if ((read_mask & cb_data->AliveWriteMask) != read_mask) {
 569                 cb_data->ReaderData->Abort = 1;
 570                 return shared_mask;
 571         }
 572
 573         return shared_mask;
 574 }
 575
 576 static void get_readers_pair_read_callback(
 577         void * userdata,
 578         struct rc_instruction * inst,
 579         struct rc_pair_instruction_arg * arg,
 580         struct rc_pair_instruction_source * src)
 581 {
 582         unsigned int shared_mask;
 583         struct get_readers_callback_data * d = userdata;
 584
 585         shared_mask = get_readers_read_callback(d,
 586                                 0 /*Pair Instructions don't use RelAddr*/,
 587                                 src->File, src->Index, arg->Swizzle);
 588
 589         if (shared_mask == RC_MASK_NONE)
 590                 return;
 591
 592         if (d->ReadPairCB)
 593                 d->ReadPairCB(d->ReaderData, inst, arg, src);
 594
 595         if (d->ReaderData->Abort)
 596                 return;
 597
 598         add_reader_pair(&d->C->Pool, d->ReaderData, inst, shared_mask, arg, src);
 599 }
 600
 601 /**
 602  * This function is used by rc_get_readers_normal() to determine whether inst
 603  * is a reader of userdata->ReaderData->Writer
 604  */
 605 static void get_readers_normal_read_callback(
 606         void * userdata,
 607         struct rc_instruction * inst,
 608         struct rc_src_register * src)
 609 {
 610         struct get_readers_callback_data * d = userdata;
 611         unsigned int shared_mask;
 612
 613         shared_mask = get_readers_read_callback(d,
 614                         src->RelAddr, src->File, src->Index, src->Swizzle);
 615
 616         if (shared_mask == RC_MASK_NONE)
 617                 return;
 618         /* The callback function could potentially clear d->ReaderData->Abort,
 619          * so we need to call it before we return. */
 620         if (d->ReadNormalCB)
 621                 d->ReadNormalCB(d->ReaderData, inst, src);
 622
 623         if (d->ReaderData->Abort)
 624                 return;
 625
 626         add_reader_normal(&d->C->Pool, d->ReaderData, inst, shared_mask, src);
 627 }
 628
 629 /**
 630  * This function is used by rc_get_readers_normal() to determine when
 631  * userdata->ReaderData->Writer is dead (i. e. All compontents of its
 632  * destination register have been overwritten by other instructions).
 633  */
 634 static void get_readers_write_callback(
 635         void *userdata,
 636         struct rc_instruction * inst,
 637         rc_register_file file,
 638         unsigned int index,
 639         unsigned int mask)
 640 {
 641         struct get_readers_callback_data * d = userdata;
 642
 643         if (index == d->DstIndex && file == d->DstFile) {
 644                 unsigned int shared_mask = mask & d->DstMask;
 645                 d->ReaderData->AbortOnRead &= ~shared_mask;
 646                 d->AliveWriteMask &= ~shared_mask;
 647         }
 648
 649         if(d->WriteCB)
 650                 d->WriteCB(d->ReaderData, inst, file, index, mask);
 651 }
 652
 653 static void get_readers_for_single_write(
 654         void * userdata,
 655         struct rc_instruction * writer,
 656         rc_register_file dst_file,
 657         unsigned int dst_index,
 658         unsigned int dst_mask)
 659 {
 660         struct rc_instruction * tmp;
 661         unsigned int branch_depth = 0;
 662         struct get_readers_callback_data * d = userdata;
 663
 664         d->ReaderData->Writer = writer;
 665         d->ReaderData->AbortOnRead = 0;
 666         d->ReaderData->InElse = 0;
 667         d->DstFile = dst_file;
 668         d->DstIndex = dst_index;
 669         d->DstMask = dst_mask;
 670         d->AliveWriteMask = dst_mask;
 671         memset(d->BranchMasks, 0, sizeof(d->BranchMasks));
 672
 673         if (!dst_mask)
 674                 return;
 675
 676         for(tmp = writer->Next; tmp != &d->C->Program.Instructions;
 677                                                         tmp = tmp->Next){
 678                 rc_opcode opcode = get_flow_control_inst(tmp);
 679                 switch(opcode) {
 680                 case RC_OPCODE_BGNLOOP:
 681                         /* XXX We can do better when we see a BGNLOOP if we
 682                          * add a flag called AbortOnWrite to struct
 683                          * rc_reader_data and leave it set until the next
 684                          * ENDLOOP. */
 685                 case RC_OPCODE_ENDLOOP:
 686                         /* XXX We can do better when we see an ENDLOOP by
 687                          * searching backwards from writer and looking for
 688                          * readers of writer's destination index.  If we find a
 689                          * reader before we get to the BGNLOOP, we must abort
 690                          * unless there is another writer between that reader
 691                          * and the BGNLOOP. */
 692                 case RC_OPCODE_BRK:
 693                 case RC_OPCODE_CONT:
 694                         d->ReaderData->Abort = 1;
 695                         return;
 696                 case RC_OPCODE_IF:
 697                         branch_depth++;
 698                         if (branch_depth > R500_PFS_MAX_BRANCH_DEPTH_FULL) {
 699                                 d->ReaderData->Abort = 1;
 700                                 return;
 701                         }
 702                         d->BranchMasks[branch_depth].IfWriteMask =
 703                                                         d->AliveWriteMask;
 704                         break;
 705                 case RC_OPCODE_ELSE:
 706                         if (branch_depth == 0) {
 707                                 d->ReaderData->InElse = 1;
 708                         } else {
 709                                 unsigned int temp_mask = d->AliveWriteMask;
 710                                 d->AliveWriteMask =
 711                                         d->BranchMasks[branch_depth].IfWriteMask;
 712                                 d->BranchMasks[branch_depth].ElseWriteMask =
 713                                                                 temp_mask;
 714                                 d->BranchMasks[branch_depth].HasElse = 1;
 715                         }
 716                         break;
 717                 case RC_OPCODE_ENDIF:
 718                         if (branch_depth == 0) {
 719                                 d->ReaderData->AbortOnRead = d->AliveWriteMask;
 720                                 d->ReaderData->InElse = 0;
 721                         }
 722                         else {
 723                                 struct branch_write_mask * masks =
 724                                         &d->BranchMasks[branch_depth];
 725
 726                                 if (masks->HasElse) {
 727                                         /* Abort on read for components that
 728                                          * were written in the IF block. */
 729                                         d->ReaderData->AbortOnRead |=
 730                                                 masks->IfWriteMask
 731                                                         & ~masks->ElseWriteMask;
 732                                         /* Abort on read for components that
 733                                          * were written in the ELSE block. */
 734                                         d->ReaderData->AbortOnRead |=
 735                                                 masks->ElseWriteMask
 736                                                         & ~d->AliveWriteMask;
 737                                         d->AliveWriteMask = masks->IfWriteMask
 738                                                 ^ ((masks->IfWriteMask ^
 739                                                         masks->ElseWriteMask)
 740                                                 & (masks->IfWriteMask
 741                                                         ^ d->AliveWriteMask));
 742                                 } else {
 743                                         d->ReaderData->AbortOnRead |=
 744                                                 masks->IfWriteMask
 745                                                         & ~d->AliveWriteMask;
 746                                         d->AliveWriteMask = masks->IfWriteMask;
 747
 748                                 }
 749                                 memset(masks, 0,
 750                                         sizeof(struct branch_write_mask));
 751                                 branch_depth--;
 752                         }
 753                         break;
 754                 default:
 755                         break;
 756                 }
 757
 758                 if (d->ReaderData->InElse)
 759                         continue;
 760
 761                 if (tmp->Type == RC_INSTRUCTION_NORMAL) {
 762                         rc_for_all_reads_src(tmp,
 763                                 get_readers_normal_read_callback, d);
 764                 } else {
 765                         rc_pair_for_all_reads_arg(tmp,
 766                                 get_readers_pair_read_callback, d);
 767                 }
 768                 rc_for_all_writes_mask(tmp, get_readers_write_callback, d);
 769
 770                 if (d->ReaderData->Abort)
 771                         return;
 772
 773                 if (branch_depth == 0 && !d->AliveWriteMask)
 774                         return;
 775         }
 776 }
 777
 778 /**
 779  * This function will create a list of readers via the rc_reader_data struct.
 780  * This function will abort (set the flag data->Abort) and return if it
 781  * encounters an instruction that reads from @param writer and also a different
 782  * instruction.  Here are some examples:
 783  *
 784  * writer = instruction 0;
 785  * 0 MOV TEMP[0].xy, TEMP[1].xy
 786  * 1 MOV TEMP[0].zw, TEMP[2].xy
 787  * 2 MOV TEMP[3], TEMP[0]
 788  * The Abort flag will be set on instruction 2, because it reads values written
 789  * by instructions 0 and 1.
 790  *
 791  * writer = instruction 1;
 792  * 0 IF TEMP[0].x
 793  * 1 MOV TEMP[1], TEMP[2]
 794  * 2 ELSE
 795  * 3 MOV TEMP[1], TEMP[2]
 796  * 4 ENDIF
 797  * 5 MOV TEMP[3], TEMP[1]
 798  * The Abort flag will be set on instruction 5, because it could read from the
 799  * value written by either instruction 1 or 3, depending on the jump decision
 800  * made at instruction 0.
 801  *
 802  * writer = instruction 0;
 803  * 0 MOV TEMP[0], TEMP[1]
 804  * 2 BGNLOOP
 805  * 3 ADD TEMP[0], TEMP[0], none.1
 806  * 4 ENDLOOP
 807  * The Abort flag will be set on instruction 3, because in the first iteration
 808  * of the loop it reads the value written by instruction 0 and in all other
 809  * iterations it reads the value written by instruction 3.
 810  *
 811  * @param read_cb This function will be called for for every instruction that
 812  * has been determined to be a reader of writer.
 813  * @param write_cb This function will be called for every instruction after
 814  * writer.
 815  */
 816 void rc_get_readers(
 817         struct radeon_compiler * c,
 818         struct rc_instruction * writer,
 819         struct rc_reader_data * data,
 820         rc_read_src_fn read_normal_cb,
 821         rc_pair_read_arg_fn read_pair_cb,
 822         rc_read_write_mask_fn write_cb)
 823 {
 824         struct get_readers_callback_data d;
 825
 826         data->Abort = 0;
 827         data->ReaderCount = 0;
 828         data->ReadersReserved = 0;
 829         data->Readers = NULL;
 830
 831         d.C = c;
 832         d.ReaderData = data;
 833         d.ReadNormalCB = read_normal_cb;
 834         d.ReadPairCB = read_pair_cb;
 835         d.WriteCB = write_cb;
 836
 837         rc_for_all_writes_mask(writer, get_readers_for_single_write, &d);
 838 }