gl/mesa/src/gallium/drivers/r300/compiler/radeon_dataflow.c

   1 /*
   2  * Copyright (C) 2009 Nicolai Haehnle.
   3  * Copyright 2010 Tom Stellard <tstellar@gmail.com>
   4  *
   5  * All Rights Reserved.
   6  *
   7  * Permission is hereby granted, free of charge, to any person obtaining
   8  * a copy of this software and associated documentation files (the
   9  * "Software"), to deal in the Software without restriction, including
  10  * without limitation the rights to use, copy, modify, merge, publish,
  11  * distribute, sublicense, and/or sell copies of the Software, and to
  12  * permit persons to whom the Software is furnished to do so, subject to
  13  * the following conditions:
  14  *
  15  * The above copyright notice and this permission notice (including the
  16  * next paragraph) shall be included in all copies or substantial
  17  * portions of the Software.
  18  *
  19  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  20  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  21  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
  22  * IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
  23  * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
  24  * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
  25  * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  26  *
  27  */
  28
  29 #include "radeon_dataflow.h"
  30
  31 #include "radeon_compiler.h"
  32 #include "radeon_compiler_util.h"
  33 #include "radeon_program.h"
  34
  35 struct read_write_mask_data {
  36         void * UserData;
  37         rc_read_write_mask_fn Cb;
  38 };
  39
  40 static void reads_normal_callback(
  41         void * userdata,
  42         struct rc_instruction * fullinst,
  43         struct rc_src_register * src)
  44 {
  45         struct read_write_mask_data * cb_data = userdata;
  46         unsigned int refmask = 0;
  47         unsigned int chan;
  48         for(chan = 0; chan < 4; chan++) {
  49                 refmask |= 1 << GET_SWZ(src->Swizzle, chan);
  50         }
  51         refmask &= RC_MASK_XYZW;
  52
  53         if (refmask) {
  54                 cb_data->Cb(cb_data->UserData, fullinst, src->File,
  55                                                         src->Index, refmask);
  56         }
  57
  58         if (refmask && src->RelAddr) {
  59                 cb_data->Cb(cb_data->UserData, fullinst, RC_FILE_ADDRESS, 0,
  60                                                                 RC_MASK_X);
  61         }
  62 }
  63
  64 static void pair_get_src_refmasks(unsigned int * refmasks,
  65                                         struct rc_pair_instruction * inst,
  66                                         unsigned int swz, unsigned int src)
  67 {
  68         if (swz >= 4)
  69                 return;
  70
  71         if (swz == RC_SWIZZLE_X || swz == RC_SWIZZLE_Y || swz == RC_SWIZZLE_Z) {
  72                 if(src == RC_PAIR_PRESUB_SRC) {
  73                         unsigned int i;
  74                         int srcp_regs =
  75                                 rc_presubtract_src_reg_count(
  76                                 inst->RGB.Src[src].Index);
  77                         for(i = 0; i < srcp_regs; i++) {
  78                                 refmasks[i] |= 1 << swz;
  79                         }
  80                 }
  81                 else {
  82                         refmasks[src] |= 1 << swz;
  83                 }
  84         }
  85
  86         if (swz == RC_SWIZZLE_W) {
  87                 if (src == RC_PAIR_PRESUB_SRC) {
  88                         unsigned int i;
  89                         int srcp_regs = rc_presubtract_src_reg_count(
  90                                         inst->Alpha.Src[src].Index);
  91                         for(i = 0; i < srcp_regs; i++) {
  92                                 refmasks[i] |= 1 << swz;
  93                         }
  94                 }
  95                 else {
  96                         refmasks[src] |= 1 << swz;
  97                 }
  98         }
  99 }
 100
 101 static void reads_pair(struct rc_instruction * fullinst, rc_read_write_mask_fn cb, void * userdata)
 102 {
 103         struct rc_pair_instruction * inst = &fullinst->U.P;
 104         unsigned int refmasks[3] = { 0, 0, 0 };
 105
 106         unsigned int arg;
 107
 108         for(arg = 0; arg < 3; ++arg) {
 109                 unsigned int chan;
 110                 for(chan = 0; chan < 3; ++chan) {
 111                         unsigned int swz_rgb =
 112                                 GET_SWZ(inst->RGB.Arg[arg].Swizzle, chan);
 113                         unsigned int swz_alpha =
 114                                 GET_SWZ(inst->Alpha.Arg[arg].Swizzle, chan);
 115                         pair_get_src_refmasks(refmasks, inst, swz_rgb,
 116                                                 inst->RGB.Arg[arg].Source);
 117                         pair_get_src_refmasks(refmasks, inst, swz_alpha,
 118                                                 inst->Alpha.Arg[arg].Source);
 119                 }
 120         }
 121
 122         for(unsigned int src = 0; src < 3; ++src) {
 123                 if (inst->RGB.Src[src].Used && (refmasks[src] & RC_MASK_XYZ))
 124                         cb(userdata, fullinst, inst->RGB.Src[src].File, inst->RGB.Src[src].Index,
 125                            refmasks[src] & RC_MASK_XYZ);
 126
 127                 if (inst->Alpha.Src[src].Used && (refmasks[src] & RC_MASK_W))
 128                         cb(userdata, fullinst, inst->Alpha.Src[src].File, inst->Alpha.Src[src].Index, RC_MASK_W);
 129         }
 130 }
 131
 132 static void pair_sub_for_all_args(
 133         struct rc_instruction * fullinst,
 134         struct rc_pair_sub_instruction * sub,
 135         rc_pair_read_arg_fn cb,
 136         void * userdata)
 137 {
 138         int i;
 139         const struct rc_opcode_info * info = rc_get_opcode_info(sub->Opcode);
 140
 141         for(i = 0; i < info->NumSrcRegs; i++) {
 142                 unsigned int src_type;
 143
 144                 src_type = rc_source_type_swz(sub->Arg[i].Swizzle);
 145
 146                 if (src_type == RC_SOURCE_NONE)
 147                         continue;
 148
 149                 if (sub->Arg[i].Source == RC_PAIR_PRESUB_SRC) {
 150                         unsigned int presub_type;
 151                         unsigned int presub_src_count;
 152                         struct rc_pair_instruction_source * src_array;
 153                         unsigned int j;
 154
 155                         if (src_type & RC_SOURCE_RGB) {
 156                                 presub_type = fullinst->
 157                                         U.P.RGB.Src[RC_PAIR_PRESUB_SRC].Index;
 158                                 src_array = fullinst->U.P.RGB.Src;
 159                         } else {
 160                                 presub_type = fullinst->
 161                                         U.P.Alpha.Src[RC_PAIR_PRESUB_SRC].Index;
 162                                 src_array = fullinst->U.P.Alpha.Src;
 163                         }
 164                         presub_src_count
 165                                 = rc_presubtract_src_reg_count(presub_type);
 166                         for(j = 0; j < presub_src_count; j++) {
 167                                 cb(userdata, fullinst, &sub->Arg[i],
 168                                                                 &src_array[j]);
 169                         }
 170                 } else {
 171                         struct rc_pair_instruction_source * src =
 172                                 rc_pair_get_src(&fullinst->U.P, &sub->Arg[i]);
 173                         if (src) {
 174                                 cb(userdata, fullinst, &sub->Arg[i], src);
 175                         }
 176                 }
 177         }
 178 }
 179
 180 /* This function calls the callback function (cb) for each source used by
 181  * the instruction.
 182  * */
 183 void rc_for_all_reads_src(
 184         struct rc_instruction * inst,
 185         rc_read_src_fn cb,
 186         void * userdata)
 187 {
 188         const struct rc_opcode_info * opcode =
 189                                         rc_get_opcode_info(inst->U.I.Opcode);
 190
 191         /* This function only works with normal instructions. */
 192         if (inst->Type != RC_INSTRUCTION_NORMAL) {
 193                 assert(0);
 194                 return;
 195         }
 196
 197         for(unsigned int src = 0; src < opcode->NumSrcRegs; ++src) {
 198
 199                 if (inst->U.I.SrcReg[src].File == RC_FILE_NONE)
 200                         continue;
 201
 202                 if (inst->U.I.SrcReg[src].File == RC_FILE_PRESUB) {
 203                         unsigned int i;
 204                         unsigned int srcp_regs = rc_presubtract_src_reg_count(
 205                                                 inst->U.I.PreSub.Opcode);
 206                         for( i = 0; i < srcp_regs; i++) {
 207                                 cb(userdata, inst, &inst->U.I.PreSub.SrcReg[i]);
 208                         }
 209                 } else {
 210                         cb(userdata, inst, &inst->U.I.SrcReg[src]);
 211                 }
 212         }
 213 }
 214
 215 /**
 216  * This function calls the callback function (cb) for each arg of the RGB and
 217  * alpha components.
 218  */
 219 void rc_pair_for_all_reads_arg(
 220         struct rc_instruction * inst,
 221         rc_pair_read_arg_fn cb,
 222         void * userdata)
 223 {
 224         /* This function only works with pair instructions. */
 225         if (inst->Type != RC_INSTRUCTION_PAIR) {
 226                 assert(0);
 227                 return;
 228         }
 229
 230         pair_sub_for_all_args(inst, &inst->U.P.RGB, cb, userdata);
 231         pair_sub_for_all_args(inst, &inst->U.P.Alpha, cb, userdata);
 232 }
 233
 234 /**
 235  * Calls a callback function for all register reads.
 236  *
 237  * This is conservative, i.e. if the same register is referenced multiple times,
 238  * the callback may also be called multiple times.
 239  * Also, the writemask of the instruction is not taken into account.
 240  */
 241 void rc_for_all_reads_mask(struct rc_instruction * inst, rc_read_write_mask_fn cb, void * userdata)
 242 {
 243         if (inst->Type == RC_INSTRUCTION_NORMAL) {
 244                 struct read_write_mask_data cb_data;
 245                 cb_data.UserData = userdata;
 246                 cb_data.Cb = cb;
 247
 248                 rc_for_all_reads_src(inst, reads_normal_callback, &cb_data);
 249         } else {
 250                 reads_pair(inst, cb, userdata);
 251         }
 252 }
 253
 254
 255
 256 static void writes_normal(struct rc_instruction * fullinst, rc_read_write_mask_fn cb, void * userdata)
 257 {
 258         struct rc_sub_instruction * inst = &fullinst->U.I;
 259         const struct rc_opcode_info * opcode = rc_get_opcode_info(inst->Opcode);
 260
 261         if (opcode->HasDstReg && inst->DstReg.WriteMask)
 262                 cb(userdata, fullinst, inst->DstReg.File, inst->DstReg.Index, inst->DstReg.WriteMask);
 263
 264         if (inst->WriteALUResult)
 265                 cb(userdata, fullinst, RC_FILE_SPECIAL, RC_SPECIAL_ALU_RESULT, RC_MASK_X);
 266 }
 267
 268 static void writes_pair(struct rc_instruction * fullinst, rc_read_write_mask_fn cb, void * userdata)
 269 {
 270         struct rc_pair_instruction * inst = &fullinst->U.P;
 271
 272         if (inst->RGB.WriteMask)
 273                 cb(userdata, fullinst, RC_FILE_TEMPORARY, inst->RGB.DestIndex, inst->RGB.WriteMask);
 274
 275         if (inst->Alpha.WriteMask)
 276                 cb(userdata, fullinst, RC_FILE_TEMPORARY, inst->Alpha.DestIndex, RC_MASK_W);
 277
 278         if (inst->WriteALUResult)
 279                 cb(userdata, fullinst, RC_FILE_SPECIAL, RC_SPECIAL_ALU_RESULT, RC_MASK_X);
 280 }
 281
 282 /**
 283  * Calls a callback function for all register writes in the instruction,
 284  * reporting writemasks to the callback function.
 285  *
 286  * \warning Does not report output registers for paired instructions!
 287  */
 288 void rc_for_all_writes_mask(struct rc_instruction * inst, rc_read_write_mask_fn cb, void * userdata)
 289 {
 290         if (inst->Type == RC_INSTRUCTION_NORMAL) {
 291                 writes_normal(inst, cb, userdata);
 292         } else {
 293                 writes_pair(inst, cb, userdata);
 294         }
 295 }
 296
 297
 298 struct mask_to_chan_data {
 299         void * UserData;
 300         rc_read_write_chan_fn Fn;
 301 };
 302
 303 static void mask_to_chan_cb(void * data, struct rc_instruction * inst,
 304                 rc_register_file file, unsigned int index, unsigned int mask)
 305 {
 306         struct mask_to_chan_data * d = data;
 307         for(unsigned int chan = 0; chan < 4; ++chan) {
 308                 if (GET_BIT(mask, chan))
 309                         d->Fn(d->UserData, inst, file, index, chan);
 310         }
 311 }
 312
 313 /**
 314  * Calls a callback function for all sourced register channels.
 315  *
 316  * This is conservative, i.e. channels may be called multiple times,
 317  * and the writemask of the instruction is not taken into account.
 318  */
 319 void rc_for_all_reads_chan(struct rc_instruction * inst, rc_read_write_chan_fn cb, void * userdata)
 320 {
 321         struct mask_to_chan_data d;
 322         d.UserData = userdata;
 323         d.Fn = cb;
 324         rc_for_all_reads_mask(inst, &mask_to_chan_cb, &d);
 325 }
 326
 327 /**
 328  * Calls a callback function for all written register channels.
 329  *
 330  * \warning Does not report output registers for paired instructions!
 331  */
 332 void rc_for_all_writes_chan(struct rc_instruction * inst, rc_read_write_chan_fn cb, void * userdata)
 333 {
 334         struct mask_to_chan_data d;
 335         d.UserData = userdata;
 336         d.Fn = cb;
 337         rc_for_all_writes_mask(inst, &mask_to_chan_cb, &d);
 338 }
 339
 340 static void remap_normal_instruction(struct rc_instruction * fullinst,
 341                 rc_remap_register_fn cb, void * userdata)
 342 {
 343         struct rc_sub_instruction * inst = &fullinst->U.I;
 344         const struct rc_opcode_info * opcode = rc_get_opcode_info(inst->Opcode);
 345         unsigned int remapped_presub = 0;
 346
 347         if (opcode->HasDstReg) {
 348                 rc_register_file file = inst->DstReg.File;
 349                 unsigned int index = inst->DstReg.Index;
 350
 351                 cb(userdata, fullinst, &file, &index);
 352
 353                 inst->DstReg.File = file;
 354                 inst->DstReg.Index = index;
 355         }
 356
 357         for(unsigned int src = 0; src < opcode->NumSrcRegs; ++src) {
 358                 rc_register_file file = inst->SrcReg[src].File;
 359                 unsigned int index = inst->SrcReg[src].Index;
 360
 361                 if (file == RC_FILE_PRESUB) {
 362                         unsigned int i;
 363                         unsigned int srcp_srcs = rc_presubtract_src_reg_count(
 364                                                 inst->PreSub.Opcode);
 365                         /* Make sure we only remap presubtract sources once in
 366                          * case more than one source register reads the
 367                          * presubtract result. */
 368                         if (remapped_presub)
 369                                 continue;
 370
 371                         for(i = 0; i < srcp_srcs; i++) {
 372                                 file = inst->PreSub.SrcReg[i].File;
 373                                 index = inst->PreSub.SrcReg[i].Index;
 374                                 cb(userdata, fullinst, &file, &index);
 375                                 inst->PreSub.SrcReg[i].File = file;
 376                                 inst->PreSub.SrcReg[i].Index = index;
 377                         }
 378                         remapped_presub = 1;
 379                 }
 380                 else {
 381                         cb(userdata, fullinst, &file, &index);
 382
 383                         inst->SrcReg[src].File = file;
 384                         inst->SrcReg[src].Index = index;
 385                 }
 386         }
 387 }
 388
 389 static void remap_pair_instruction(struct rc_instruction * fullinst,
 390                 rc_remap_register_fn cb, void * userdata)
 391 {
 392         struct rc_pair_instruction * inst = &fullinst->U.P;
 393
 394         if (inst->RGB.WriteMask) {
 395                 rc_register_file file = RC_FILE_TEMPORARY;
 396                 unsigned int index = inst->RGB.DestIndex;
 397
 398                 cb(userdata, fullinst, &file, &index);
 399
 400                 inst->RGB.DestIndex = index;
 401         }
 402
 403         if (inst->Alpha.WriteMask) {
 404                 rc_register_file file = RC_FILE_TEMPORARY;
 405                 unsigned int index = inst->Alpha.DestIndex;
 406
 407                 cb(userdata, fullinst, &file, &index);
 408
 409                 inst->Alpha.DestIndex = index;
 410         }
 411
 412         for(unsigned int src = 0; src < 3; ++src) {
 413                 if (inst->RGB.Src[src].Used) {
 414                         rc_register_file file = inst->RGB.Src[src].File;
 415                         unsigned int index = inst->RGB.Src[src].Index;
 416
 417                         cb(userdata, fullinst, &file, &index);
 418
 419                         inst->RGB.Src[src].File = file;
 420                         inst->RGB.Src[src].Index = index;
 421                 }
 422
 423                 if (inst->Alpha.Src[src].Used) {
 424                         rc_register_file file = inst->Alpha.Src[src].File;
 425                         unsigned int index = inst->Alpha.Src[src].Index;
 426
 427                         cb(userdata, fullinst, &file, &index);
 428
 429                         inst->Alpha.Src[src].File = file;
 430                         inst->Alpha.Src[src].Index = index;
 431                 }
 432         }
 433 }
 434
 435
 436 /**
 437  * Remap all register accesses according to the given function.
 438  * That is, call the function \p cb for each referenced register (both read and written)
 439  * and update the given instruction \p inst accordingly
 440  * if it modifies its \ref pfile and \ref pindex contents.
 441  */
 442 void rc_remap_registers(struct rc_instruction * inst, rc_remap_register_fn cb, void * userdata)
 443 {
 444         if (inst->Type == RC_INSTRUCTION_NORMAL)
 445                 remap_normal_instruction(inst, cb, userdata);
 446         else
 447                 remap_pair_instruction(inst, cb, userdata);
 448 }
 449
 450 struct branch_write_mask {
 451         unsigned int IfWriteMask:4;
 452         unsigned int ElseWriteMask:4;
 453         unsigned int HasElse:1;
 454 };
 455
 456 union get_readers_read_cb {
 457         rc_read_src_fn I;
 458         rc_pair_read_arg_fn P;
 459 };
 460
 461 struct get_readers_callback_data {
 462         struct radeon_compiler * C;
 463         struct rc_reader_data * ReaderData;
 464         rc_read_src_fn ReadNormalCB;
 465         rc_pair_read_arg_fn ReadPairCB;
 466         rc_read_write_mask_fn WriteCB;
 467         rc_register_file DstFile;
 468         unsigned int DstIndex;
 469         unsigned int DstMask;
 470         unsigned int AliveWriteMask;
 471         /*  For convenience, this is indexed starting at 1 */
 472         struct branch_write_mask BranchMasks[R500_PFS_MAX_BRANCH_DEPTH_FULL + 1];
 473 };
 474
 475 static struct rc_reader * add_reader(
 476         struct memory_pool * pool,
 477         struct rc_reader_data * data,
 478         struct rc_instruction * inst,
 479         unsigned int mask)
 480 {
 481         struct rc_reader * new;
 482         memory_pool_array_reserve(pool, struct rc_reader, data->Readers,
 483                                 data->ReaderCount, data->ReadersReserved, 1);
 484         new = &data->Readers[data->ReaderCount++];
 485         new->Inst = inst;
 486         new->WriteMask = mask;
 487         return new;
 488 }
 489
 490 static void add_reader_normal(
 491         struct memory_pool * pool,
 492         struct rc_reader_data * data,
 493         struct rc_instruction * inst,
 494         unsigned int mask,
 495         struct rc_src_register * src)
 496 {
 497         struct rc_reader * new = add_reader(pool, data, inst, mask);
 498         new->U.I.Src = src;
 499 }
 500
 501
 502 static void add_reader_pair(
 503         struct memory_pool * pool,
 504         struct rc_reader_data * data,
 505         struct rc_instruction * inst,
 506         unsigned int mask,
 507         struct rc_pair_instruction_arg * arg,
 508         struct rc_pair_instruction_source * src)
 509 {
 510         struct rc_reader * new = add_reader(pool, data, inst, mask);
 511         new->U.P.Src = src;
 512         new->U.P.Arg = arg;
 513 }
 514
 515 static unsigned int get_readers_read_callback(
 516         struct get_readers_callback_data * cb_data,
 517         unsigned int has_rel_addr,
 518         rc_register_file file,
 519         unsigned int index,
 520         unsigned int swizzle)
 521 {
 522         unsigned int shared_mask, read_mask;
 523
 524         if (has_rel_addr) {
 525                 cb_data->ReaderData->Abort = 1;
 526                 return RC_MASK_NONE;
 527         }
 528
 529         shared_mask = rc_src_reads_dst_mask(file, index, swizzle,
 530                 cb_data->DstFile, cb_data->DstIndex, cb_data->AliveWriteMask);
 531
 532         if (shared_mask == RC_MASK_NONE)
 533                 return shared_mask;
 534
 535         /* If we make it this far, it means that this source reads from the
 536          * same register written to by d->ReaderData->Writer. */
 537
 538         read_mask = rc_swizzle_to_writemask(swizzle);
 539         if (cb_data->ReaderData->AbortOnRead & read_mask) {
 540                 cb_data->ReaderData->Abort = 1;
 541                 return shared_mask;
 542         }
 543
 544         if (cb_data->ReaderData->LoopDepth > 0) {
 545                 cb_data->ReaderData->AbortOnWrite |=
 546                                 (read_mask & cb_data->AliveWriteMask);
 547         }
 548
 549         /* XXX The behavior in this case should be configurable. */
 550         if ((read_mask & cb_data->AliveWriteMask) != read_mask) {
 551                 cb_data->ReaderData->Abort = 1;
 552                 return shared_mask;
 553         }
 554
 555         return shared_mask;
 556 }
 557
 558 static void get_readers_pair_read_callback(
 559         void * userdata,
 560         struct rc_instruction * inst,
 561         struct rc_pair_instruction_arg * arg,
 562         struct rc_pair_instruction_source * src)
 563 {
 564         unsigned int shared_mask;
 565         struct get_readers_callback_data * d = userdata;
 566
 567         shared_mask = get_readers_read_callback(d,
 568                                 0 /*Pair Instructions don't use RelAddr*/,
 569                                 src->File, src->Index, arg->Swizzle);
 570
 571         if (shared_mask == RC_MASK_NONE)
 572                 return;
 573
 574         if (d->ReadPairCB)
 575                 d->ReadPairCB(d->ReaderData, inst, arg, src);
 576
 577         if (d->ReaderData->ExitOnAbort && d->ReaderData->Abort)
 578                 return;
 579
 580         add_reader_pair(&d->C->Pool, d->ReaderData, inst, shared_mask, arg, src);
 581 }
 582
 583 /**
 584  * This function is used by rc_get_readers_normal() to determine whether inst
 585  * is a reader of userdata->ReaderData->Writer
 586  */
 587 static void get_readers_normal_read_callback(
 588         void * userdata,
 589         struct rc_instruction * inst,
 590         struct rc_src_register * src)
 591 {
 592         struct get_readers_callback_data * d = userdata;
 593         unsigned int shared_mask;
 594
 595         shared_mask = get_readers_read_callback(d,
 596                         src->RelAddr, src->File, src->Index, src->Swizzle);
 597
 598         if (shared_mask == RC_MASK_NONE)
 599                 return;
 600         /* The callback function could potentially clear d->ReaderData->Abort,
 601          * so we need to call it before we return. */
 602         if (d->ReadNormalCB)
 603                 d->ReadNormalCB(d->ReaderData, inst, src);
 604
 605         if (d->ReaderData->ExitOnAbort && d->ReaderData->Abort)
 606                 return;
 607
 608         add_reader_normal(&d->C->Pool, d->ReaderData, inst, shared_mask, src);
 609 }
 610
 611 /**
 612  * This function is used by rc_get_readers_normal() to determine when
 613  * userdata->ReaderData->Writer is dead (i. e. All compontents of its
 614  * destination register have been overwritten by other instructions).
 615  */
 616 static void get_readers_write_callback(
 617         void *userdata,
 618         struct rc_instruction * inst,
 619         rc_register_file file,
 620         unsigned int index,
 621         unsigned int mask)
 622 {
 623         struct get_readers_callback_data * d = userdata;
 624
 625         if (index == d->DstIndex && file == d->DstFile) {
 626                 unsigned int shared_mask = mask & d->DstMask;
 627                 d->ReaderData->AbortOnRead &= ~shared_mask;
 628                 d->AliveWriteMask &= ~shared_mask;
 629                 if (d->ReaderData->AbortOnWrite & shared_mask) {
 630                         d->ReaderData->Abort = 1;
 631                 }
 632         }
 633
 634         if(d->WriteCB)
 635                 d->WriteCB(d->ReaderData, inst, file, index, mask);
 636 }
 637
 638 static void push_branch_mask(
 639         struct get_readers_callback_data * d,
 640         unsigned int * branch_depth)
 641 {
 642         (*branch_depth)++;
 643         if (*branch_depth > R500_PFS_MAX_BRANCH_DEPTH_FULL) {
 644                 d->ReaderData->Abort = 1;
 645                 return;
 646         }
 647         d->BranchMasks[*branch_depth].IfWriteMask =
 648                                         d->AliveWriteMask;
 649 }
 650
 651 static void pop_branch_mask(
 652         struct get_readers_callback_data * d,
 653         unsigned int * branch_depth)
 654 {
 655         struct branch_write_mask * masks = &d->BranchMasks[*branch_depth];
 656
 657         if (masks->HasElse) {
 658                 /* Abort on read for components that were written in the IF
 659                  * block. */
 660                 d->ReaderData->AbortOnRead |=
 661                                 masks->IfWriteMask & ~masks->ElseWriteMask;
 662                 /* Abort on read for components that were written in the ELSE
 663                  * block. */
 664                 d->ReaderData->AbortOnRead |=
 665                                 masks->ElseWriteMask & ~d->AliveWriteMask;
 666
 667                 d->AliveWriteMask = masks->IfWriteMask
 668                         ^ ((masks->IfWriteMask ^ masks->ElseWriteMask)
 669                         & (masks->IfWriteMask ^ d->AliveWriteMask));
 670         } else {
 671                 d->ReaderData->AbortOnRead |=
 672                                 masks->IfWriteMask & ~d->AliveWriteMask;
 673                 d->AliveWriteMask = masks->IfWriteMask;
 674
 675         }
 676         memset(masks, 0, sizeof(struct branch_write_mask));
 677         (*branch_depth)--;
 678 }
 679
 680 static void get_readers_for_single_write(
 681         void * userdata,
 682         struct rc_instruction * writer,
 683         rc_register_file dst_file,
 684         unsigned int dst_index,
 685         unsigned int dst_mask)
 686 {
 687         struct rc_instruction * tmp;
 688         unsigned int branch_depth = 0;
 689         struct rc_instruction * endloop = NULL;
 690         unsigned int abort_on_read_at_endloop = 0;
 691         struct get_readers_callback_data * d = userdata;
 692
 693         d->ReaderData->Writer = writer;
 694         d->ReaderData->AbortOnRead = 0;
 695         d->ReaderData->AbortOnWrite = 0;
 696         d->ReaderData->LoopDepth = 0;
 697         d->ReaderData->InElse = 0;
 698         d->DstFile = dst_file;
 699         d->DstIndex = dst_index;
 700         d->DstMask = dst_mask;
 701         d->AliveWriteMask = dst_mask;
 702         memset(d->BranchMasks, 0, sizeof(d->BranchMasks));
 703
 704         if (!dst_mask)
 705                 return;
 706
 707         for(tmp = writer->Next; tmp != &d->C->Program.Instructions;
 708                                                         tmp = tmp->Next){
 709                 rc_opcode opcode = rc_get_flow_control_inst(tmp);
 710                 switch(opcode) {
 711                 case RC_OPCODE_BGNLOOP:
 712                         d->ReaderData->LoopDepth++;
 713                         push_branch_mask(d, &branch_depth);
 714                         break;
 715                 case RC_OPCODE_ENDLOOP:
 716                         if (d->ReaderData->LoopDepth > 0) {
 717                                 d->ReaderData->LoopDepth--;
 718                                 if (d->ReaderData->LoopDepth == 0) {
 719                                         d->ReaderData->AbortOnWrite = 0;
 720                                 }
 721                                 pop_branch_mask(d, &branch_depth);
 722                         } else {
 723                                 /* Here we have reached an ENDLOOP without
 724                                  * seeing its BGNLOOP.  These means that
 725                                  * the writer was written inside of a loop,
 726                                  * so it could have readers that are above it
 727                                  * (i.e. they have a lower IP).  To find these
 728                                  * readers we jump to the BGNLOOP instruction
 729                                  * and check each instruction until we get
 730                                  * back to the writer.
 731                                  */
 732                                 endloop = tmp;
 733                                 tmp = rc_match_endloop(tmp);
 734                                 if (!tmp) {
 735                                         rc_error(d->C, "Failed to match endloop.\n");
 736                                         d->ReaderData->Abort = 1;
 737                                         return;
 738                                 }
 739                                 abort_on_read_at_endloop = d->ReaderData->AbortOnRead;
 740                                 d->ReaderData->AbortOnRead |= d->AliveWriteMask;
 741                                 continue;
 742                         }
 743                         break;
 744                 case RC_OPCODE_IF:
 745                         push_branch_mask(d, &branch_depth);
 746                         break;
 747                 case RC_OPCODE_ELSE:
 748                         if (branch_depth == 0) {
 749                                 d->ReaderData->InElse = 1;
 750                         } else {
 751                                 unsigned int temp_mask = d->AliveWriteMask;
 752                                 d->AliveWriteMask =
 753                                         d->BranchMasks[branch_depth].IfWriteMask;
 754                                 d->BranchMasks[branch_depth].ElseWriteMask =
 755                                                                 temp_mask;
 756                                 d->BranchMasks[branch_depth].HasElse = 1;
 757                         }
 758                         break;
 759                 case RC_OPCODE_ENDIF:
 760                         if (branch_depth == 0) {
 761                                 d->ReaderData->AbortOnRead = d->AliveWriteMask;
 762                                 d->ReaderData->InElse = 0;
 763                         }
 764                         else {
 765                                 pop_branch_mask(d, &branch_depth);
 766                         }
 767                         break;
 768                 default:
 769                         break;
 770                 }
 771
 772                 if (d->ReaderData->InElse)
 773                         continue;
 774
 775                 if (tmp->Type == RC_INSTRUCTION_NORMAL) {
 776                         rc_for_all_reads_src(tmp,
 777                                 get_readers_normal_read_callback, d);
 778                 } else {
 779                         rc_pair_for_all_reads_arg(tmp,
 780                                 get_readers_pair_read_callback, d);
 781                 }
 782
 783                 /* This can happen when we jump from an ENDLOOP to BGNLOOP */
 784                 if (tmp == writer) {
 785                         tmp = endloop;
 786                         endloop = NULL;
 787                         d->ReaderData->AbortOnRead = abort_on_read_at_endloop;
 788                         continue;
 789                 }
 790                 rc_for_all_writes_mask(tmp, get_readers_write_callback, d);
 791
 792                 if (d->ReaderData->ExitOnAbort && d->ReaderData->Abort)
 793                         return;
 794
 795                 if (branch_depth == 0 && !d->AliveWriteMask)
 796                         return;
 797         }
 798 }
 799
 800 static void init_get_readers_callback_data(
 801         struct get_readers_callback_data * d,
 802         struct rc_reader_data * reader_data,
 803         struct radeon_compiler * c,
 804         rc_read_src_fn read_normal_cb,
 805         rc_pair_read_arg_fn read_pair_cb,
 806         rc_read_write_mask_fn write_cb)
 807 {
 808         reader_data->Abort = 0;
 809         reader_data->ReaderCount = 0;
 810         reader_data->ReadersReserved = 0;
 811         reader_data->Readers = NULL;
 812
 813         d->C = c;
 814         d->ReaderData = reader_data;
 815         d->ReadNormalCB = read_normal_cb;
 816         d->ReadPairCB = read_pair_cb;
 817         d->WriteCB = write_cb;
 818 }
 819
 820 /**
 821  * This function will create a list of readers via the rc_reader_data struct.
 822  * This function will abort (set the flag data->Abort) and return if it
 823  * encounters an instruction that reads from @param writer and also a different
 824  * instruction.  Here are some examples:
 825  *
 826  * writer = instruction 0;
 827  * 0 MOV TEMP[0].xy, TEMP[1].xy
 828  * 1 MOV TEMP[0].zw, TEMP[2].xy
 829  * 2 MOV TEMP[3], TEMP[0]
 830  * The Abort flag will be set on instruction 2, because it reads values written
 831  * by instructions 0 and 1.
 832  *
 833  * writer = instruction 1;
 834  * 0 IF TEMP[0].x
 835  * 1 MOV TEMP[1], TEMP[2]
 836  * 2 ELSE
 837  * 3 MOV TEMP[1], TEMP[2]
 838  * 4 ENDIF
 839  * 5 MOV TEMP[3], TEMP[1]
 840  * The Abort flag will be set on instruction 5, because it could read from the
 841  * value written by either instruction 1 or 3, depending on the jump decision
 842  * made at instruction 0.
 843  *
 844  * writer = instruction 0;
 845  * 0 MOV TEMP[0], TEMP[1]
 846  * 2 BGNLOOP
 847  * 3 ADD TEMP[0], TEMP[0], none.1
 848  * 4 ENDLOOP
 849  * The Abort flag will be set on instruction 3, because in the first iteration
 850  * of the loop it reads the value written by instruction 0 and in all other
 851  * iterations it reads the value written by instruction 3.
 852  *
 853  * @param read_cb This function will be called for for every instruction that
 854  * has been determined to be a reader of writer.
 855  * @param write_cb This function will be called for every instruction after
 856  * writer.
 857  */
 858 void rc_get_readers(
 859         struct radeon_compiler * c,
 860         struct rc_instruction * writer,
 861         struct rc_reader_data * data,
 862         rc_read_src_fn read_normal_cb,
 863         rc_pair_read_arg_fn read_pair_cb,
 864         rc_read_write_mask_fn write_cb)
 865 {
 866         struct get_readers_callback_data d;
 867
 868         init_get_readers_callback_data(&d, data, c, read_normal_cb,
 869                                                 read_pair_cb, write_cb);
 870
 871         rc_for_all_writes_mask(writer, get_readers_for_single_write, &d);
 872 }
 873
 874 void rc_get_readers_sub(
 875         struct radeon_compiler * c,
 876         struct rc_instruction * writer,
 877         struct rc_pair_sub_instruction * sub_writer,
 878         struct rc_reader_data * data,
 879         rc_read_src_fn read_normal_cb,
 880         rc_pair_read_arg_fn read_pair_cb,
 881         rc_read_write_mask_fn write_cb)
 882 {
 883         struct get_readers_callback_data d;
 884
 885         init_get_readers_callback_data(&d, data, c, read_normal_cb,
 886                                                 read_pair_cb, write_cb);
 887
 888         if (sub_writer->WriteMask) {
 889                 get_readers_for_single_write(&d, writer, RC_FILE_TEMPORARY,
 890                         sub_writer->DestIndex, sub_writer->WriteMask);
 891         }
 892 }