src/amd/compiler/aco_validate.cpp

   1 /*
   2  * Copyright © 2018 Valve Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  */
  24
  25 #include "aco_ir.h"
  26
  27 #include "util/memstream.h"
  28 #include "util/ralloc.h"
  29
  30 #include <array>
  31 #include <map>
  32 #include <set>
  33 #include <vector>
  34
  35 namespace aco {
  36
  37 static void
  38 aco_log(Program* program, enum aco_compiler_debug_level level, const char* prefix,
  39         const char* file, unsigned line, const char* fmt, va_list args)
  40 {
  41    char* msg;
  42
  43    if (program->debug.shorten_messages) {
  44       msg = ralloc_vasprintf(NULL, fmt, args);
  45    } else {
  46       msg = ralloc_strdup(NULL, prefix);
  47       ralloc_asprintf_append(&msg, "    In file %s:%u\n", file, line);
  48       ralloc_asprintf_append(&msg, "    ");
  49       ralloc_vasprintf_append(&msg, fmt, args);
  50    }
  51
  52    if (program->debug.func)
  53       program->debug.func(program->debug.private_data, level, msg);
  54
  55    fprintf(program->debug.output, "%s\n", msg);
  56
  57    ralloc_free(msg);
  58 }
  59
  60 void
  61 _aco_perfwarn(Program* program, const char* file, unsigned line, const char* fmt, ...)
  62 {
  63    va_list args;
  64
  65    va_start(args, fmt);
  66    aco_log(program, ACO_COMPILER_DEBUG_LEVEL_PERFWARN, "ACO PERFWARN:\n", file, line, fmt, args);
  67    va_end(args);
  68 }
  69
  70 void
  71 _aco_err(Program* program, const char* file, unsigned line, const char* fmt, ...)
  72 {
  73    va_list args;
  74
  75    va_start(args, fmt);
  76    aco_log(program, ACO_COMPILER_DEBUG_LEVEL_ERROR, "ACO ERROR:\n", file, line, fmt, args);
  77    va_end(args);
  78 }
  79
  80 bool
  81 validate_ir(Program* program)
  82 {
  83    bool is_valid = true;
  84    auto check = [&program, &is_valid](bool success, const char* msg,
  85                                       aco::Instruction* instr) -> void
  86    {
  87       if (!success) {
  88          char* out;
  89          size_t outsize;
  90          struct u_memstream mem;
  91          u_memstream_open(&mem, &out, &outsize);
  92          FILE* const memf = u_memstream_get(&mem);
  93
  94          fprintf(memf, "%s: ", msg);
  95          aco_print_instr(program->gfx_level, instr, memf);
  96          u_memstream_close(&mem);
  97
  98          aco_err(program, "%s", out);
  99          free(out);
 100
 101          is_valid = false;
 102       }
 103    };
 104
 105    auto check_block = [&program, &is_valid](bool success, const char* msg,
 106                                             aco::Block* block) -> void
 107    {
 108       if (!success) {
 109          aco_err(program, "%s: BB%u", msg, block->index);
 110          is_valid = false;
 111       }
 112    };
 113
 114    for (Block& block : program->blocks) {
 115       for (aco_ptr<Instruction>& instr : block.instructions) {
 116
 117          /* check base format */
 118          Format base_format = instr->format;
 119          base_format = (Format)((uint32_t)base_format & ~(uint32_t)Format::SDWA);
 120          base_format = (Format)((uint32_t)base_format & ~(uint32_t)Format::DPP16);
 121          base_format = (Format)((uint32_t)base_format & ~(uint32_t)Format::DPP8);
 122          if ((uint32_t)base_format & (uint32_t)Format::VOP1)
 123             base_format = Format::VOP1;
 124          else if ((uint32_t)base_format & (uint32_t)Format::VOP2)
 125             base_format = Format::VOP2;
 126          else if ((uint32_t)base_format & (uint32_t)Format::VOPC)
 127             base_format = Format::VOPC;
 128          else if ((uint32_t)base_format & (uint32_t)Format::VINTRP) {
 129             if (instr->opcode == aco_opcode::v_interp_p1ll_f16 ||
 130                 instr->opcode == aco_opcode::v_interp_p1lv_f16 ||
 131                 instr->opcode == aco_opcode::v_interp_p2_legacy_f16 ||
 132                 instr->opcode == aco_opcode::v_interp_p2_f16) {
 133                /* v_interp_*_fp16 are considered VINTRP by the compiler but
 134                 * they are emitted as VOP3.
 135                 */
 136                base_format = Format::VOP3;
 137             } else {
 138                base_format = Format::VINTRP;
 139             }
 140          }
 141          check(base_format == instr_info.format[(int)instr->opcode],
 142                "Wrong base format for instruction", instr.get());
 143
 144          /* check VOP3 modifiers */
 145          if (instr->isVOP3() && withoutDPP(instr->format) != Format::VOP3) {
 146             check(base_format == Format::VOP2 || base_format == Format::VOP1 ||
 147                      base_format == Format::VOPC || base_format == Format::VINTRP,
 148                   "Format cannot have VOP3/VOP3B applied", instr.get());
 149          }
 150
 151          if (instr->isDPP()) {
 152             check(base_format == Format::VOP2 || base_format == Format::VOP1 ||
 153                      base_format == Format::VOPC || base_format == Format::VOP3 ||
 154                      base_format == Format::VOP3P,
 155                   "Format cannot have DPP applied", instr.get());
 156             check((!instr->isVOP3() && !instr->isVOP3P()) || program->gfx_level >= GFX11,
 157                   "VOP3+DPP is GFX11+ only", instr.get());
 158          }
 159
 160          /* check SDWA */
 161          if (instr->isSDWA()) {
 162             check(base_format == Format::VOP2 || base_format == Format::VOP1 ||
 163                      base_format == Format::VOPC,
 164                   "Format cannot have SDWA applied", instr.get());
 165
 166             check(program->gfx_level >= GFX8, "SDWA is GFX8 to GFX10.3 only", instr.get());
 167             check(program->gfx_level < GFX11, "SDWA is GFX8 to GFX10.3 only", instr.get());
 168
 169             SDWA_instruction& sdwa = instr->sdwa();
 170             check(sdwa.omod == 0 || program->gfx_level >= GFX9, "SDWA omod only supported on GFX9+",
 171                   instr.get());
 172             if (base_format == Format::VOPC) {
 173                check(sdwa.clamp == false || program->gfx_level == GFX8,
 174                      "SDWA VOPC clamp only supported on GFX8", instr.get());
 175                check((instr->definitions[0].isFixed() && instr->definitions[0].physReg() == vcc) ||
 176                         program->gfx_level >= GFX9,
 177                      "SDWA+VOPC definition must be fixed to vcc on GFX8", instr.get());
 178             } else {
 179                const Definition& def = instr->definitions[0];
 180                check(def.bytes() <= 4, "SDWA definitions must not be larger than 4 bytes",
 181                      instr.get());
 182                check(def.bytes() >= sdwa.dst_sel.size() + sdwa.dst_sel.offset(),
 183                      "SDWA definition selection size must be at most definition size", instr.get());
 184                check(
 185                   sdwa.dst_sel.size() == 1 || sdwa.dst_sel.size() == 2 || sdwa.dst_sel.size() == 4,
 186                   "SDWA definition selection size must be 1, 2 or 4 bytes", instr.get());
 187                check(sdwa.dst_sel.offset() % sdwa.dst_sel.size() == 0, "Invalid selection offset",
 188                      instr.get());
 189                check(def.bytes() == 4 || def.bytes() == sdwa.dst_sel.size(),
 190                      "SDWA dst_sel size must be definition size for subdword definitions",
 191                      instr.get());
 192                check(def.bytes() == 4 || sdwa.dst_sel.offset() == 0,
 193                      "SDWA dst_sel offset must be 0 for subdword definitions", instr.get());
 194             }
 195
 196             for (unsigned i = 0; i < std::min<unsigned>(2, instr->operands.size()); i++) {
 197                const Operand& op = instr->operands[i];
 198                check(op.bytes() <= 4, "SDWA operands must not be larger than 4 bytes", instr.get());
 199                check(op.bytes() >= sdwa.sel[i].size() + sdwa.sel[i].offset(),
 200                      "SDWA operand selection size must be at most operand size", instr.get());
 201                check(sdwa.sel[i].size() == 1 || sdwa.sel[i].size() == 2 || sdwa.sel[i].size() == 4,
 202                      "SDWA operand selection size must be 1, 2 or 4 bytes", instr.get());
 203                check(sdwa.sel[i].offset() % sdwa.sel[i].size() == 0, "Invalid selection offset",
 204                      instr.get());
 205             }
 206             if (instr->operands.size() >= 3) {
 207                check(instr->operands[2].isFixed() && instr->operands[2].physReg() == vcc,
 208                      "3rd operand must be fixed to vcc with SDWA", instr.get());
 209             }
 210             if (instr->definitions.size() >= 2) {
 211                check(instr->definitions[1].isFixed() && instr->definitions[1].physReg() == vcc,
 212                      "2nd definition must be fixed to vcc with SDWA", instr.get());
 213             }
 214
 215             const bool sdwa_opcodes =
 216                instr->opcode != aco_opcode::v_fmac_f32 && instr->opcode != aco_opcode::v_fmac_f16 &&
 217                instr->opcode != aco_opcode::v_fmamk_f32 &&
 218                instr->opcode != aco_opcode::v_fmaak_f32 &&
 219                instr->opcode != aco_opcode::v_fmamk_f16 &&
 220                instr->opcode != aco_opcode::v_fmaak_f16 &&
 221                instr->opcode != aco_opcode::v_madmk_f32 &&
 222                instr->opcode != aco_opcode::v_madak_f32 &&
 223                instr->opcode != aco_opcode::v_madmk_f16 &&
 224                instr->opcode != aco_opcode::v_madak_f16 &&
 225                instr->opcode != aco_opcode::v_readfirstlane_b32 &&
 226                instr->opcode != aco_opcode::v_clrexcp && instr->opcode != aco_opcode::v_swap_b32;
 227
 228             const bool feature_mac =
 229                program->gfx_level == GFX8 &&
 230                (instr->opcode == aco_opcode::v_mac_f32 && instr->opcode == aco_opcode::v_mac_f16);
 231
 232             check(sdwa_opcodes || feature_mac, "SDWA can't be used with this opcode", instr.get());
 233          }
 234
 235          /* check opsel */
 236          if (instr->isVOP3() || instr->isVOP1() || instr->isVOP2() || instr->isVOPC()) {
 237             VALU_instruction& valu = instr->valu();
 238             check(valu.opsel == 0 || program->gfx_level >= GFX9, "Opsel is only supported on GFX9+",
 239                   instr.get());
 240             check(valu.opsel == 0 || instr->format == Format::VOP3 || program->gfx_level >= GFX11,
 241                   "Opsel is only supported for VOP3 before GFX11", instr.get());
 242
 243             for (unsigned i = 0; i < 3; i++) {
 244                if (i >= instr->operands.size() ||
 245                    (!instr->isVOP3() && !instr->operands[i].isOfType(RegType::vgpr)) ||
 246                    (instr->operands[i].hasRegClass() &&
 247                     instr->operands[i].regClass().is_subdword() && !instr->operands[i].isFixed()))
 248                   check(!valu.opsel[i], "Unexpected opsel for operand", instr.get());
 249             }
 250             if (instr->definitions[0].regClass().is_subdword() && !instr->definitions[0].isFixed())
 251                check(!valu.opsel[3], "Unexpected opsel for sub-dword definition", instr.get());
 252          } else if (instr->opcode == aco_opcode::v_fma_mixlo_f16 ||
 253                     instr->opcode == aco_opcode::v_fma_mixhi_f16 ||
 254                     instr->opcode == aco_opcode::v_fma_mix_f32) {
 255             check(instr->definitions[0].regClass() ==
 256                      (instr->opcode == aco_opcode::v_fma_mix_f32 ? v1 : v2b),
 257                   "v_fma_mix_f32/v_fma_mix_f16 must have v1/v2b definition", instr.get());
 258          } else if (instr->isVOP3P()) {
 259             VALU_instruction& vop3p = instr->valu();
 260             for (unsigned i = 0; i < instr->operands.size(); i++) {
 261                if (instr->operands[i].hasRegClass() &&
 262                    instr->operands[i].regClass().is_subdword() && !instr->operands[i].isFixed())
 263                   check(!vop3p.opsel_lo[i] && !vop3p.opsel_hi[i],
 264                         "Unexpected opsel for subdword operand", instr.get());
 265             }
 266             check(instr->definitions[0].regClass() == v1, "VOP3P must have v1 definition",
 267                   instr.get());
 268          }
 269
 270          /* check for undefs */
 271          for (unsigned i = 0; i < instr->operands.size(); i++) {
 272             if (instr->operands[i].isUndefined()) {
 273                bool flat = instr->isFlatLike();
 274                bool can_be_undef = is_phi(instr) || instr->isEXP() || instr->isReduction() ||
 275                                    instr->opcode == aco_opcode::p_create_vector ||
 276                                    instr->opcode == aco_opcode::p_jump_to_epilog ||
 277                                    instr->opcode == aco_opcode::p_dual_src_export_gfx11 ||
 278                                    (instr->opcode == aco_opcode::p_interp_gfx11 && i == 0) ||
 279                                    (instr->opcode == aco_opcode::p_bpermute_gfx11w64 && i == 0) ||
 280                                    (flat && i == 1) || (instr->isMIMG() && (i == 1 || i == 2)) ||
 281                                    ((instr->isMUBUF() || instr->isMTBUF()) && i == 1) ||
 282                                    (instr->isScratch() && i == 0) ||
 283                                    (instr->isDS() && i == 0) ||
 284                                    (instr->opcode == aco_opcode::p_init_scratch && i == 0);
 285                check(can_be_undef, "Undefs can only be used in certain operands", instr.get());
 286             } else {
 287                check(instr->operands[i].isFixed() || instr->operands[i].isTemp() ||
 288                         instr->operands[i].isConstant(),
 289                      "Uninitialized Operand", instr.get());
 290             }
 291          }
 292
 293          /* check subdword definitions */
 294          for (unsigned i = 0; i < instr->definitions.size(); i++) {
 295             if (instr->definitions[i].regClass().is_subdword())
 296                check(instr->definitions[i].bytes() <= 4 || instr->isPseudo() || instr->isVMEM(),
 297                      "Only Pseudo and VMEM instructions can write subdword registers > 4 bytes",
 298                      instr.get());
 299          }
 300
 301          if (instr->isSALU() || instr->isVALU()) {
 302             /* check literals */
 303             Operand literal(s1);
 304             for (unsigned i = 0; i < instr->operands.size(); i++) {
 305                Operand op = instr->operands[i];
 306                if (!op.isLiteral())
 307                   continue;
 308
 309                check(!instr->isDPP() && !instr->isSDWA() &&
 310                         (!instr->isVOP3() || program->gfx_level >= GFX10) &&
 311                         (!instr->isVOP3P() || program->gfx_level >= GFX10),
 312                      "Literal applied on wrong instruction format", instr.get());
 313
 314                check(literal.isUndefined() || (literal.size() == op.size() &&
 315                                                literal.constantValue() == op.constantValue()),
 316                      "Only 1 Literal allowed", instr.get());
 317                literal = op;
 318                check(instr->isSALU() || instr->isVOP3() || instr->isVOP3P() || i == 0 || i == 2,
 319                      "Wrong source position for Literal argument", instr.get());
 320             }
 321
 322             /* check num sgprs for VALU */
 323             if (instr->isVALU()) {
 324                bool is_shift64 = instr->opcode == aco_opcode::v_lshlrev_b64 ||
 325                                  instr->opcode == aco_opcode::v_lshrrev_b64 ||
 326                                  instr->opcode == aco_opcode::v_ashrrev_i64;
 327                unsigned const_bus_limit = 1;
 328                if (program->gfx_level >= GFX10 && !is_shift64)
 329                   const_bus_limit = 2;
 330
 331                uint32_t scalar_mask =
 332                   instr->isVOP3() || instr->isVOP3P() || instr->isVINTERP_INREG() ? 0x7 : 0x5;
 333                if (instr->isSDWA())
 334                   scalar_mask = program->gfx_level >= GFX9 ? 0x7 : 0x4;
 335                else if (instr->isDPP())
 336                   scalar_mask = 0x4;
 337
 338                if (instr->isVOPC() || instr->opcode == aco_opcode::v_readfirstlane_b32 ||
 339                    instr->opcode == aco_opcode::v_readlane_b32 ||
 340                    instr->opcode == aco_opcode::v_readlane_b32_e64) {
 341                   check(instr->definitions[0].regClass().type() == RegType::sgpr,
 342                         "Wrong Definition type for VALU instruction", instr.get());
 343                } else {
 344                   check(instr->definitions[0].regClass().type() == RegType::vgpr,
 345                         "Wrong Definition type for VALU instruction", instr.get());
 346                }
 347
 348                unsigned num_sgprs = 0;
 349                unsigned sgpr[] = {0, 0};
 350                for (unsigned i = 0; i < instr->operands.size(); i++) {
 351                   Operand op = instr->operands[i];
 352                   if (instr->opcode == aco_opcode::v_readfirstlane_b32 ||
 353                       instr->opcode == aco_opcode::v_readlane_b32 ||
 354                       instr->opcode == aco_opcode::v_readlane_b32_e64) {
 355                      check(i != 1 || op.isOfType(RegType::sgpr) || op.isConstant(),
 356                            "Must be a SGPR or a constant", instr.get());
 357                      check(i == 1 || (op.isOfType(RegType::vgpr) && op.bytes() <= 4),
 358                            "Wrong Operand type for VALU instruction", instr.get());
 359                      continue;
 360                   }
 361                   if (instr->opcode == aco_opcode::v_permlane16_b32 ||
 362                       instr->opcode == aco_opcode::v_permlanex16_b32) {
 363                      check(i != 0 || op.isOfType(RegType::vgpr),
 364                            "Operand 0 of v_permlane must be VGPR", instr.get());
 365                      check(i == 0 || op.isOfType(RegType::sgpr) || op.isConstant(),
 366                            "Lane select operands of v_permlane must be SGPR or constant",
 367                            instr.get());
 368                   }
 369
 370                   if (instr->opcode == aco_opcode::v_writelane_b32 ||
 371                       instr->opcode == aco_opcode::v_writelane_b32_e64) {
 372                      check(i != 2 || (op.isOfType(RegType::vgpr) && op.bytes() <= 4),
 373                            "Wrong Operand type for VALU instruction", instr.get());
 374                      check(i == 2 || op.isOfType(RegType::sgpr) || op.isConstant(),
 375                            "Must be a SGPR or a constant", instr.get());
 376                      continue;
 377                   }
 378                   if (op.isOfType(RegType::sgpr)) {
 379                      check(scalar_mask & (1 << i), "Wrong source position for SGPR argument",
 380                            instr.get());
 381
 382                      if (op.tempId() != sgpr[0] && op.tempId() != sgpr[1]) {
 383                         if (num_sgprs < 2)
 384                            sgpr[num_sgprs++] = op.tempId();
 385                      }
 386                   }
 387
 388                   if (op.isConstant() && !op.isLiteral())
 389                      check(scalar_mask & (1 << i), "Wrong source position for constant argument",
 390                            instr.get());
 391                }
 392                check(num_sgprs + (literal.isUndefined() ? 0 : 1) <= const_bus_limit,
 393                      "Too many SGPRs/literals", instr.get());
 394
 395                /* Validate modifiers. */
 396                check(!instr->valu().opsel || instr->isVOP3() || instr->isVOP1() ||
 397                         instr->isVOP2() || instr->isVOPC() || instr->isVINTERP_INREG(),
 398                      "OPSEL set for unsupported instruction format", instr.get());
 399                check(!instr->valu().opsel_lo || instr->isVOP3P(),
 400                      "OPSEL_LO set for unsupported instruction format", instr.get());
 401                check(!instr->valu().opsel_hi || instr->isVOP3P(),
 402                      "OPSEL_HI set for unsupported instruction format", instr.get());
 403                check(!instr->valu().omod || instr->isVOP3() ||instr->isSDWA(),
 404                      "OMOD set for unsupported instruction format", instr.get());
 405                check(!instr->valu().clamp || instr->isVOP3() || instr->isVOP3P() ||
 406                         instr->isSDWA() || instr->isVINTERP_INREG(),
 407                      "CLAMP set for unsupported instruction format", instr.get());
 408
 409                for (bool abs : instr->valu().abs) {
 410                   check(!abs || instr->isVOP3() || instr->isVOP3P() || instr->isSDWA() ||
 411                            instr->isDPP16(),
 412                         "ABS/NEG_HI set for unsupported instruction format", instr.get());
 413                }
 414                for (bool neg : instr->valu().neg) {
 415                   check(!neg || instr->isVOP3() || instr->isVOP3P() || instr->isSDWA() ||
 416                            instr->isDPP16() || instr->isVINTERP_INREG(),
 417                         "NEG/NEG_LO set for unsupported instruction format", instr.get());
 418                }
 419             }
 420
 421             if (instr->isSOP1() || instr->isSOP2()) {
 422                if (!instr->definitions.empty())
 423                   check(instr->definitions[0].regClass().type() == RegType::sgpr,
 424                         "Wrong Definition type for SALU instruction", instr.get());
 425                for (const Operand& op : instr->operands) {
 426                   check(op.isConstant() || op.isOfType(RegType::sgpr),
 427                         "Wrong Operand type for SALU instruction", instr.get());
 428                }
 429             }
 430          }
 431
 432          switch (instr->format) {
 433          case Format::PSEUDO: {
 434             if (instr->opcode == aco_opcode::p_create_vector) {
 435                unsigned size = 0;
 436                for (const Operand& op : instr->operands) {
 437                   check(op.bytes() < 4 || size % 4 == 0, "Operand is not aligned", instr.get());
 438                   size += op.bytes();
 439                }
 440                check(size == instr->definitions[0].bytes(),
 441                      "Definition size does not match operand sizes", instr.get());
 442                if (instr->definitions[0].regClass().type() == RegType::sgpr) {
 443                   for (const Operand& op : instr->operands) {
 444                      check(op.isConstant() || op.regClass().type() == RegType::sgpr,
 445                            "Wrong Operand type for scalar vector", instr.get());
 446                   }
 447                }
 448             } else if (instr->opcode == aco_opcode::p_extract_vector) {
 449                check(!instr->operands[0].isConstant() && instr->operands[1].isConstant(),
 450                      "Wrong Operand types", instr.get());
 451                check((instr->operands[1].constantValue() + 1) * instr->definitions[0].bytes() <=
 452                         instr->operands[0].bytes(),
 453                      "Index out of range", instr.get());
 454                check(instr->definitions[0].regClass().type() == RegType::vgpr ||
 455                         instr->operands[0].regClass().type() == RegType::sgpr,
 456                      "Cannot extract SGPR value from VGPR vector", instr.get());
 457                check(program->gfx_level >= GFX9 ||
 458                         !instr->definitions[0].regClass().is_subdword() ||
 459                         instr->operands[0].regClass().type() == RegType::vgpr,
 460                      "Cannot extract subdword from SGPR before GFX9+", instr.get());
 461             } else if (instr->opcode == aco_opcode::p_split_vector) {
 462                check(!instr->operands[0].isConstant(), "Operand must not be constant", instr.get());
 463                unsigned size = 0;
 464                for (const Definition& def : instr->definitions) {
 465                   size += def.bytes();
 466                }
 467                check(size == instr->operands[0].bytes(),
 468                      "Operand size does not match definition sizes", instr.get());
 469                if (instr->operands[0].isOfType(RegType::vgpr)) {
 470                   for (const Definition& def : instr->definitions)
 471                      check(def.regClass().type() == RegType::vgpr,
 472                            "Wrong Definition type for VGPR split_vector", instr.get());
 473                } else {
 474                   for (const Definition& def : instr->definitions)
 475                      check(program->gfx_level >= GFX9 || !def.regClass().is_subdword(),
 476                            "Cannot split SGPR into subdword VGPRs before GFX9+", instr.get());
 477                }
 478             } else if (instr->opcode == aco_opcode::p_parallelcopy) {
 479                check(instr->definitions.size() == instr->operands.size(),
 480                      "Number of Operands does not match number of Definitions", instr.get());
 481                for (unsigned i = 0; i < instr->operands.size(); i++) {
 482                   check(instr->definitions[i].bytes() == instr->operands[i].bytes(),
 483                         "Operand and Definition size must match", instr.get());
 484                   if (instr->operands[i].hasRegClass()) {
 485                      check((instr->definitions[i].regClass().type() ==
 486                             instr->operands[i].regClass().type()) ||
 487                               (instr->definitions[i].regClass().type() == RegType::vgpr &&
 488                                instr->operands[i].regClass().type() == RegType::sgpr),
 489                            "Operand and Definition types do not match", instr.get());
 490                      check(instr->definitions[i].regClass().is_linear_vgpr() ==
 491                               instr->operands[i].regClass().is_linear_vgpr(),
 492                            "Operand and Definition types do not match", instr.get());
 493                   } else {
 494                      check(!instr->definitions[i].regClass().is_linear_vgpr(),
 495                            "Can only copy linear VGPRs into linear VGPRs, not constant/undef",
 496                            instr.get());
 497                   }
 498                }
 499             } else if (instr->opcode == aco_opcode::p_phi) {
 500                check(instr->operands.size() == block.logical_preds.size(),
 501                      "Number of Operands does not match number of predecessors", instr.get());
 502                check(instr->definitions[0].regClass().type() == RegType::vgpr,
 503                      "Logical Phi Definition must be vgpr", instr.get());
 504                for (const Operand& op : instr->operands)
 505                   check(instr->definitions[0].size() == op.size(),
 506                         "Operand sizes must match Definition size", instr.get());
 507             } else if (instr->opcode == aco_opcode::p_linear_phi) {
 508                for (const Operand& op : instr->operands) {
 509                   check(!op.isTemp() || op.getTemp().is_linear(), "Wrong Operand type",
 510                         instr.get());
 511                   check(instr->definitions[0].size() == op.size(),
 512                         "Operand sizes must match Definition size", instr.get());
 513                }
 514                check(instr->operands.size() == block.linear_preds.size(),
 515                      "Number of Operands does not match number of predecessors", instr.get());
 516             } else if (instr->opcode == aco_opcode::p_extract ||
 517                        instr->opcode == aco_opcode::p_insert) {
 518                check(!instr->operands[0].isConstant(), "Data operand must not be constant",
 519                      instr.get());
 520                check(instr->operands[1].isConstant(), "Index must be constant", instr.get());
 521                if (instr->opcode == aco_opcode::p_extract)
 522                   check(instr->operands[3].isConstant(), "Sign-extend flag must be constant",
 523                         instr.get());
 524
 525                check(instr->definitions[0].regClass().type() != RegType::sgpr ||
 526                         instr->operands[0].regClass().type() == RegType::sgpr,
 527                      "Can't extract/insert VGPR to SGPR", instr.get());
 528
 529                if (instr->opcode == aco_opcode::p_insert)
 530                   check(instr->operands[0].bytes() == instr->definitions[0].bytes(),
 531                         "Sizes of p_insert data operand and definition must match", instr.get());
 532
 533                if (instr->definitions[0].regClass().type() == RegType::sgpr)
 534                   check(instr->definitions.size() >= 2 && instr->definitions[1].isFixed() &&
 535                            instr->definitions[1].physReg() == scc,
 536                         "SGPR extract/insert needs an SCC definition", instr.get());
 537
 538                unsigned data_bits = instr->operands[0].bytes() * 8u;
 539                unsigned op_bits = instr->operands[2].constantValue();
 540
 541                if (instr->opcode == aco_opcode::p_insert) {
 542                   check(op_bits == 8 || op_bits == 16, "Size must be 8 or 16", instr.get());
 543                   check(op_bits < data_bits, "Size must be smaller than source", instr.get());
 544                } else if (instr->opcode == aco_opcode::p_extract) {
 545                   check(op_bits == 8 || op_bits == 16 || op_bits == 32,
 546                         "Size must be 8 or 16 or 32", instr.get());
 547                   check(data_bits >= op_bits, "Can't extract more bits than what the data has.",
 548                         instr.get());
 549                }
 550
 551                unsigned comp = data_bits / MAX2(op_bits, 1);
 552                check(instr->operands[1].constantValue() < comp, "Index must be in-bounds",
 553                      instr.get());
 554             } else if (instr->opcode == aco_opcode::p_jump_to_epilog) {
 555                check(instr->definitions.size() == 0, "p_jump_to_epilog must have 0 definitions",
 556                      instr.get());
 557                check(instr->operands.size() > 0 && instr->operands[0].isOfType(RegType::sgpr) &&
 558                         instr->operands[0].size() == 2,
 559                      "First operand of p_jump_to_epilog must be a SGPR", instr.get());
 560                for (unsigned i = 1; i < instr->operands.size(); i++) {
 561                   check(
 562                      instr->operands[i].isOfType(RegType::vgpr) || instr->operands[i].isUndefined(),
 563                      "Other operands of p_jump_to_epilog must be VGPRs or undef", instr.get());
 564                }
 565             } else if (instr->opcode == aco_opcode::p_dual_src_export_gfx11) {
 566                check(instr->definitions.size() == 6,
 567                      "p_dual_src_export_gfx11 must have 6 definitions", instr.get());
 568                check(instr->definitions[2].regClass().type() == RegType::vgpr &&
 569                         instr->definitions[2].regClass().size() == 1,
 570                      "Third definition of p_dual_src_export_gfx11 must be a v1", instr.get());
 571                check(instr->definitions[3].regClass() == program->lane_mask,
 572                      "Fourth definition of p_dual_src_export_gfx11 must be a lane mask", instr.get());
 573                check(instr->definitions[4].physReg() == vcc,
 574                      "Fifth definition of p_dual_src_export_gfx11 must be vcc", instr.get());
 575                check(instr->definitions[5].physReg() == scc,
 576                      "Sixth definition of p_dual_src_export_gfx11 must be scc", instr.get());
 577                check(instr->operands.size() == 8, "p_dual_src_export_gfx11 must have 8 operands",
 578                      instr.get());
 579                for (unsigned i = 0; i < instr->operands.size(); i++) {
 580                   check(
 581                      instr->operands[i].isOfType(RegType::vgpr) || instr->operands[i].isUndefined(),
 582                      "Operands of p_dual_src_export_gfx11 must be VGPRs or undef", instr.get());
 583                }
 584             } else if (instr->opcode == aco_opcode::p_start_linear_vgpr) {
 585                check(instr->definitions.size() == 1, "Must have one definition", instr.get());
 586                check(instr->operands.size() <= 1, "Must have one or zero operands", instr.get());
 587                if (!instr->definitions.empty())
 588                   check(instr->definitions[0].regClass().is_linear_vgpr(),
 589                         "Definition must be linear VGPR", instr.get());
 590                if (!instr->definitions.empty() && !instr->operands.empty())
 591                   check(instr->definitions[0].bytes() == instr->operands[0].bytes(),
 592                         "Operand size must match definition", instr.get());
 593             }
 594             break;
 595          }
 596          case Format::PSEUDO_REDUCTION: {
 597             for (const Operand& op : instr->operands)
 598                check(op.regClass().type() == RegType::vgpr,
 599                      "All operands of PSEUDO_REDUCTION instructions must be in VGPRs.",
 600                      instr.get());
 601
 602             if (instr->opcode == aco_opcode::p_reduce &&
 603                 instr->reduction().cluster_size == program->wave_size)
 604                check(instr->definitions[0].regClass().type() == RegType::sgpr ||
 605                         program->wave_size == 32,
 606                      "The result of unclustered reductions must go into an SGPR.", instr.get());
 607             else
 608                check(instr->definitions[0].regClass().type() == RegType::vgpr,
 609                      "The result of scans and clustered reductions must go into a VGPR.",
 610                      instr.get());
 611
 612             break;
 613          }
 614          case Format::SMEM: {
 615             if (instr->operands.size() >= 1)
 616                check(instr->operands[0].isOfType(RegType::sgpr), "SMEM operands must be sgpr",
 617                      instr.get());
 618             if (instr->operands.size() >= 2)
 619                check(instr->operands[1].isConstant() || instr->operands[1].isOfType(RegType::sgpr),
 620                      "SMEM offset must be constant or sgpr", instr.get());
 621             if (!instr->definitions.empty())
 622                check(instr->definitions[0].regClass().type() == RegType::sgpr,
 623                      "SMEM result must be sgpr", instr.get());
 624             break;
 625          }
 626          case Format::MTBUF:
 627          case Format::MUBUF: {
 628             check(instr->operands.size() > 1, "VMEM instructions must have at least one operand",
 629                   instr.get());
 630             check(instr->operands[1].isOfType(RegType::vgpr),
 631                   "VADDR must be in vgpr for VMEM instructions", instr.get());
 632             check(instr->operands[0].isOfType(RegType::sgpr), "VMEM resource constant must be sgpr",
 633                   instr.get());
 634             check(instr->operands.size() < 4 || instr->operands[3].isOfType(RegType::vgpr),
 635                   "VMEM write data must be vgpr", instr.get());
 636
 637             const bool d16 = instr->opcode == aco_opcode::buffer_load_dword || // FIXME: used to spill subdword variables
 638                              instr->opcode == aco_opcode::buffer_load_ubyte ||
 639                              instr->opcode == aco_opcode::buffer_load_sbyte ||
 640                              instr->opcode == aco_opcode::buffer_load_ushort ||
 641                              instr->opcode == aco_opcode::buffer_load_sshort ||
 642                              instr->opcode == aco_opcode::buffer_load_ubyte_d16 ||
 643                              instr->opcode == aco_opcode::buffer_load_ubyte_d16_hi ||
 644                              instr->opcode == aco_opcode::buffer_load_sbyte_d16 ||
 645                              instr->opcode == aco_opcode::buffer_load_sbyte_d16_hi ||
 646                              instr->opcode == aco_opcode::buffer_load_short_d16 ||
 647                              instr->opcode == aco_opcode::buffer_load_short_d16_hi ||
 648                              instr->opcode == aco_opcode::buffer_load_format_d16_x ||
 649                              instr->opcode == aco_opcode::buffer_load_format_d16_hi_x ||
 650                              instr->opcode == aco_opcode::buffer_load_format_d16_xy ||
 651                              instr->opcode == aco_opcode::buffer_load_format_d16_xyz ||
 652                              instr->opcode == aco_opcode::buffer_load_format_d16_xyzw ||
 653                              instr->opcode == aco_opcode::tbuffer_load_format_d16_x ||
 654                              instr->opcode == aco_opcode::tbuffer_load_format_d16_xy ||
 655                              instr->opcode == aco_opcode::tbuffer_load_format_d16_xyz ||
 656                              instr->opcode == aco_opcode::tbuffer_load_format_d16_xyzw;
 657             if (instr->definitions.size()) {
 658                check(instr->definitions[0].regClass().type() == RegType::vgpr,
 659                      "VMEM definitions[0] (VDATA) must be VGPR", instr.get());
 660                check(d16 || !instr->definitions[0].regClass().is_subdword(),
 661                      "Only D16 opcodes can load subdword values.", instr.get());
 662                check(instr->definitions[0].bytes() <= 8 || !d16,
 663                      "D16 opcodes can only load up to 8 bytes.", instr.get());
 664             }
 665             break;
 666          }
 667          case Format::MIMG: {
 668             check(instr->operands.size() >= 4, "MIMG instructions must have at least 4 operands",
 669                   instr.get());
 670             check(instr->operands[0].hasRegClass() &&
 671                      (instr->operands[0].regClass() == s4 || instr->operands[0].regClass() == s8),
 672                   "MIMG operands[0] (resource constant) must be in 4 or 8 SGPRs", instr.get());
 673             if (instr->operands[1].hasRegClass())
 674                check(instr->operands[1].regClass() == s4,
 675                      "MIMG operands[1] (sampler constant) must be 4 SGPRs", instr.get());
 676             if (!instr->operands[2].isUndefined()) {
 677                bool is_cmpswap = instr->opcode == aco_opcode::image_atomic_cmpswap ||
 678                                  instr->opcode == aco_opcode::image_atomic_fcmpswap;
 679                check(instr->definitions.empty() ||
 680                         (instr->definitions[0].regClass() == instr->operands[2].regClass() ||
 681                          is_cmpswap),
 682                      "MIMG operands[2] (VDATA) must be the same as definitions[0] for atomics and "
 683                      "TFE/LWE loads",
 684                      instr.get());
 685             }
 686
 687             if (instr->mimg().strict_wqm) {
 688                check(instr->operands[3].hasRegClass() &&
 689                         instr->operands[3].regClass().is_linear_vgpr(),
 690                      "MIMG operands[3] must be temp linear VGPR.", instr.get());
 691
 692                unsigned total_size = 0;
 693                for (unsigned i = 4; i < instr->operands.size(); i++) {
 694                   check(instr->operands[i].hasRegClass() && instr->operands[i].regClass() == v1,
 695                         "MIMG operands[4+] (VADDR) must be v1", instr.get());
 696                   total_size += instr->operands[i].bytes();
 697                }
 698                check(total_size <= instr->operands[3].bytes(),
 699                      "MIMG operands[4+] must fit within operands[3].", instr.get());
 700             } else {
 701                check(instr->operands.size() == 4 || program->gfx_level >= GFX10,
 702                      "NSA is only supported on GFX10+", instr.get());
 703                for (unsigned i = 3; i < instr->operands.size(); i++) {
 704                   check(instr->operands[i].hasRegClass() &&
 705                            instr->operands[i].regClass().type() == RegType::vgpr,
 706                         "MIMG operands[3+] (VADDR) must be VGPR", instr.get());
 707                   if (instr->operands.size() > 4) {
 708                      if (program->gfx_level < GFX11) {
 709                         check(instr->operands[i].regClass() == v1,
 710                               "GFX10 MIMG VADDR must be v1 if NSA is used", instr.get());
 711                      } else {
 712                         if (instr->opcode != aco_opcode::image_bvh_intersect_ray &&
 713                             instr->opcode != aco_opcode::image_bvh64_intersect_ray && i < 7) {
 714                            check(instr->operands[i].regClass() == v1,
 715                                  "first 4 GFX11 MIMG VADDR must be v1 if NSA is used", instr.get());
 716                         }
 717                      }
 718                   }
 719                }
 720             }
 721
 722             if (instr->definitions.size()) {
 723                check(instr->definitions[0].regClass().type() == RegType::vgpr,
 724                      "MIMG definitions[0] (VDATA) must be VGPR", instr.get());
 725                check(instr->mimg().d16 || !instr->definitions[0].regClass().is_subdword(),
 726                      "Only D16 MIMG instructions can load subdword values.", instr.get());
 727                check(instr->definitions[0].bytes() <= 8 || !instr->mimg().d16,
 728                      "D16 MIMG instructions can only load up to 8 bytes.", instr.get());
 729             }
 730             break;
 731          }
 732          case Format::DS: {
 733             for (const Operand& op : instr->operands) {
 734                check(op.isOfType(RegType::vgpr) || op.physReg() == m0 || op.isUndefined(),
 735                      "Only VGPRs are valid DS instruction operands", instr.get());
 736             }
 737             if (!instr->definitions.empty())
 738                check(instr->definitions[0].regClass().type() == RegType::vgpr,
 739                      "DS instruction must return VGPR", instr.get());
 740             break;
 741          }
 742          case Format::EXP: {
 743             for (unsigned i = 0; i < 4; i++)
 744                check(instr->operands[i].isOfType(RegType::vgpr),
 745                      "Only VGPRs are valid Export arguments", instr.get());
 746             break;
 747          }
 748          case Format::FLAT:
 749             check(instr->operands[1].isUndefined(), "Flat instructions don't support SADDR",
 750                   instr.get());
 751             FALLTHROUGH;
 752          case Format::GLOBAL:
 753             check(instr->operands[0].isOfType(RegType::vgpr), "FLAT/GLOBAL address must be vgpr",
 754                   instr.get());
 755             FALLTHROUGH;
 756          case Format::SCRATCH: {
 757             check(instr->operands[0].isOfType(RegType::vgpr),
 758                   "FLAT/GLOBAL/SCRATCH address must be undefined or vgpr", instr.get());
 759             check(instr->operands[1].isOfType(RegType::sgpr),
 760                   "FLAT/GLOBAL/SCRATCH sgpr address must be undefined or sgpr", instr.get());
 761             if (instr->format == Format::SCRATCH && program->gfx_level < GFX10_3)
 762                check(!instr->operands[0].isUndefined() || !instr->operands[1].isUndefined(),
 763                      "SCRATCH must have either SADDR or ADDR operand", instr.get());
 764             if (!instr->definitions.empty())
 765                check(instr->definitions[0].regClass().type() == RegType::vgpr,
 766                      "FLAT/GLOBAL/SCRATCH result must be vgpr", instr.get());
 767             else
 768                check(instr->operands[2].isOfType(RegType::vgpr),
 769                      "FLAT/GLOBAL/SCRATCH data must be vgpr", instr.get());
 770             break;
 771          }
 772          case Format::LDSDIR: {
 773             check(instr->definitions.size() == 1 && instr->definitions[0].regClass() == v1, "LDSDIR must have an v1 definition", instr.get());
 774             check(instr->operands.size() == 1, "LDSDIR must have an operand", instr.get());
 775             if (!instr->operands.empty()) {
 776                check(instr->operands[0].regClass() == s1, "LDSDIR must have an s1 operand", instr.get());
 777                check(instr->operands[0].isFixed() && instr->operands[0].physReg() == m0, "LDSDIR must have an operand fixed to m0", instr.get());
 778             }
 779             break;
 780          }
 781          default: break;
 782          }
 783       }
 784    }
 785
 786    /* validate CFG */
 787    for (unsigned i = 0; i < program->blocks.size(); i++) {
 788       Block& block = program->blocks[i];
 789       check_block(block.index == i, "block.index must match actual index", &block);
 790
 791       /* predecessors/successors should be sorted */
 792       for (unsigned j = 0; j + 1 < block.linear_preds.size(); j++)
 793          check_block(block.linear_preds[j] < block.linear_preds[j + 1],
 794                      "linear predecessors must be sorted", &block);
 795       for (unsigned j = 0; j + 1 < block.logical_preds.size(); j++)
 796          check_block(block.logical_preds[j] < block.logical_preds[j + 1],
 797                      "logical predecessors must be sorted", &block);
 798       for (unsigned j = 0; j + 1 < block.linear_succs.size(); j++)
 799          check_block(block.linear_succs[j] < block.linear_succs[j + 1],
 800                      "linear successors must be sorted", &block);
 801       for (unsigned j = 0; j + 1 < block.logical_succs.size(); j++)
 802          check_block(block.logical_succs[j] < block.logical_succs[j + 1],
 803                      "logical successors must be sorted", &block);
 804
 805       /* critical edges are not allowed */
 806       if (block.linear_preds.size() > 1) {
 807          for (unsigned pred : block.linear_preds)
 808             check_block(program->blocks[pred].linear_succs.size() == 1,
 809                         "linear critical edges are not allowed", &program->blocks[pred]);
 810          for (unsigned pred : block.logical_preds)
 811             check_block(program->blocks[pred].logical_succs.size() == 1,
 812                         "logical critical edges are not allowed", &program->blocks[pred]);
 813       }
 814    }
 815
 816    return is_valid;
 817 }
 818
 819 /* RA validation */
 820 namespace {
 821
 822 struct Location {
 823    Location() : block(NULL), instr(NULL) {}
 824
 825    Block* block;
 826    Instruction* instr; // NULL if it's the block's live-in
 827 };
 828
 829 struct Assignment {
 830    Location defloc;
 831    Location firstloc;
 832    PhysReg reg;
 833    bool valid;
 834 };
 835
 836 bool
 837 ra_fail(Program* program, Location loc, Location loc2, const char* fmt, ...)
 838 {
 839    va_list args;
 840    va_start(args, fmt);
 841    char msg[1024];
 842    vsprintf(msg, fmt, args);
 843    va_end(args);
 844
 845    char* out;
 846    size_t outsize;
 847    struct u_memstream mem;
 848    u_memstream_open(&mem, &out, &outsize);
 849    FILE* const memf = u_memstream_get(&mem);
 850
 851    fprintf(memf, "RA error found at instruction in BB%d:\n", loc.block->index);
 852    if (loc.instr) {
 853       aco_print_instr(program->gfx_level, loc.instr, memf);
 854       fprintf(memf, "\n%s", msg);
 855    } else {
 856       fprintf(memf, "%s", msg);
 857    }
 858    if (loc2.block) {
 859       fprintf(memf, " in BB%d:\n", loc2.block->index);
 860       aco_print_instr(program->gfx_level, loc2.instr, memf);
 861    }
 862    fprintf(memf, "\n\n");
 863    u_memstream_close(&mem);
 864
 865    aco_err(program, "%s", out);
 866    free(out);
 867
 868    return true;
 869 }
 870
 871 bool
 872 validate_subdword_operand(amd_gfx_level gfx_level, const aco_ptr<Instruction>& instr,
 873                           unsigned index)
 874 {
 875    Operand op = instr->operands[index];
 876    unsigned byte = op.physReg().byte();
 877
 878    if (instr->opcode == aco_opcode::p_as_uniform)
 879       return byte == 0;
 880    if (instr->isPseudo() && gfx_level >= GFX8)
 881       return true;
 882    if (instr->isSDWA())
 883       return byte + instr->sdwa().sel[index].offset() + instr->sdwa().sel[index].size() <= 4 &&
 884              byte % instr->sdwa().sel[index].size() == 0;
 885    if (instr->isVOP3P()) {
 886       bool fma_mix = instr->opcode == aco_opcode::v_fma_mixlo_f16 ||
 887                      instr->opcode == aco_opcode::v_fma_mixhi_f16 ||
 888                      instr->opcode == aco_opcode::v_fma_mix_f32;
 889       return instr->valu().opsel_lo[index] == (byte >> 1) &&
 890              instr->valu().opsel_hi[index] == (fma_mix || (byte >> 1));
 891    }
 892    if (byte == 2 && can_use_opsel(gfx_level, instr->opcode, index))
 893       return true;
 894
 895    switch (instr->opcode) {
 896    case aco_opcode::v_cvt_f32_ubyte1:
 897       if (byte == 1)
 898          return true;
 899       break;
 900    case aco_opcode::v_cvt_f32_ubyte2:
 901       if (byte == 2)
 902          return true;
 903       break;
 904    case aco_opcode::v_cvt_f32_ubyte3:
 905       if (byte == 3)
 906          return true;
 907       break;
 908    case aco_opcode::ds_write_b8_d16_hi:
 909    case aco_opcode::ds_write_b16_d16_hi:
 910       if (byte == 2 && index == 1)
 911          return true;
 912       break;
 913    case aco_opcode::buffer_store_byte_d16_hi:
 914    case aco_opcode::buffer_store_short_d16_hi:
 915    case aco_opcode::buffer_store_format_d16_hi_x:
 916       if (byte == 2 && index == 3)
 917          return true;
 918       break;
 919    case aco_opcode::flat_store_byte_d16_hi:
 920    case aco_opcode::flat_store_short_d16_hi:
 921    case aco_opcode::scratch_store_byte_d16_hi:
 922    case aco_opcode::scratch_store_short_d16_hi:
 923    case aco_opcode::global_store_byte_d16_hi:
 924    case aco_opcode::global_store_short_d16_hi:
 925       if (byte == 2 && index == 2)
 926          return true;
 927       break;
 928    default: break;
 929    }
 930
 931    return byte == 0;
 932 }
 933
 934 bool
 935 validate_subdword_definition(amd_gfx_level gfx_level, const aco_ptr<Instruction>& instr)
 936 {
 937    Definition def = instr->definitions[0];
 938    unsigned byte = def.physReg().byte();
 939
 940    if (instr->isPseudo() && gfx_level >= GFX8)
 941       return true;
 942    if (instr->isSDWA())
 943       return byte + instr->sdwa().dst_sel.offset() + instr->sdwa().dst_sel.size() <= 4 &&
 944              byte % instr->sdwa().dst_sel.size() == 0;
 945    if (byte == 2 && can_use_opsel(gfx_level, instr->opcode, -1))
 946       return true;
 947
 948    switch (instr->opcode) {
 949    case aco_opcode::v_fma_mixhi_f16:
 950    case aco_opcode::buffer_load_ubyte_d16_hi:
 951    case aco_opcode::buffer_load_sbyte_d16_hi:
 952    case aco_opcode::buffer_load_short_d16_hi:
 953    case aco_opcode::buffer_load_format_d16_hi_x:
 954    case aco_opcode::flat_load_ubyte_d16_hi:
 955    case aco_opcode::flat_load_short_d16_hi:
 956    case aco_opcode::scratch_load_ubyte_d16_hi:
 957    case aco_opcode::scratch_load_short_d16_hi:
 958    case aco_opcode::global_load_ubyte_d16_hi:
 959    case aco_opcode::global_load_short_d16_hi:
 960    case aco_opcode::ds_read_u8_d16_hi:
 961    case aco_opcode::ds_read_u16_d16_hi: return byte == 2;
 962    default: break;
 963    }
 964
 965    return byte == 0;
 966 }
 967
 968 unsigned
 969 get_subdword_bytes_written(Program* program, const aco_ptr<Instruction>& instr, unsigned index)
 970 {
 971    amd_gfx_level gfx_level = program->gfx_level;
 972    Definition def = instr->definitions[index];
 973
 974    if (instr->isPseudo())
 975       return gfx_level >= GFX8 ? def.bytes() : def.size() * 4u;
 976    if (instr->isVALU()) {
 977       assert(def.bytes() <= 2);
 978       if (instr->isSDWA())
 979          return instr->sdwa().dst_sel.size();
 980
 981       if (instr_is_16bit(gfx_level, instr->opcode))
 982          return 2;
 983
 984       return 4;
 985    }
 986
 987    if (instr->isMIMG()) {
 988       assert(instr->mimg().d16);
 989       return program->dev.sram_ecc_enabled ? def.size() * 4u : def.bytes();
 990    }
 991
 992    switch (instr->opcode) {
 993    case aco_opcode::buffer_load_ubyte_d16:
 994    case aco_opcode::buffer_load_sbyte_d16:
 995    case aco_opcode::buffer_load_short_d16:
 996    case aco_opcode::buffer_load_format_d16_x:
 997    case aco_opcode::tbuffer_load_format_d16_x:
 998    case aco_opcode::flat_load_ubyte_d16:
 999    case aco_opcode::flat_load_short_d16:
1000    case aco_opcode::scratch_load_ubyte_d16:
1001    case aco_opcode::scratch_load_short_d16:
1002    case aco_opcode::global_load_ubyte_d16:
1003    case aco_opcode::global_load_short_d16:
1004    case aco_opcode::ds_read_u8_d16:
1005    case aco_opcode::ds_read_u16_d16:
1006    case aco_opcode::buffer_load_ubyte_d16_hi:
1007    case aco_opcode::buffer_load_sbyte_d16_hi:
1008    case aco_opcode::buffer_load_short_d16_hi:
1009    case aco_opcode::buffer_load_format_d16_hi_x:
1010    case aco_opcode::flat_load_ubyte_d16_hi:
1011    case aco_opcode::flat_load_short_d16_hi:
1012    case aco_opcode::scratch_load_ubyte_d16_hi:
1013    case aco_opcode::scratch_load_short_d16_hi:
1014    case aco_opcode::global_load_ubyte_d16_hi:
1015    case aco_opcode::global_load_short_d16_hi:
1016    case aco_opcode::ds_read_u8_d16_hi:
1017    case aco_opcode::ds_read_u16_d16_hi: return program->dev.sram_ecc_enabled ? 4 : 2;
1018    case aco_opcode::buffer_load_format_d16_xyz:
1019    case aco_opcode::tbuffer_load_format_d16_xyz: return program->dev.sram_ecc_enabled ? 8 : 6;
1020    default: return def.size() * 4;
1021    }
1022 }
1023
1024 bool
1025 validate_instr_defs(Program* program, std::array<unsigned, 2048>& regs,
1026                     const std::vector<Assignment>& assignments, const Location& loc,
1027                     aco_ptr<Instruction>& instr)
1028 {
1029    bool err = false;
1030
1031    for (unsigned i = 0; i < instr->definitions.size(); i++) {
1032       Definition& def = instr->definitions[i];
1033       if (!def.isTemp())
1034          continue;
1035       Temp tmp = def.getTemp();
1036       PhysReg reg = assignments[tmp.id()].reg;
1037       for (unsigned j = 0; j < tmp.bytes(); j++) {
1038          if (regs[reg.reg_b + j])
1039             err |=
1040                ra_fail(program, loc, assignments[regs[reg.reg_b + j]].defloc,
1041                        "Assignment of element %d of %%%d already taken by %%%d from instruction", i,
1042                        tmp.id(), regs[reg.reg_b + j]);
1043          regs[reg.reg_b + j] = tmp.id();
1044       }
1045       if (def.regClass().is_subdword() && def.bytes() < 4) {
1046          unsigned written = get_subdword_bytes_written(program, instr, i);
1047          /* If written=4, the instruction still might write the upper half. In that case, it's
1048           * the lower half that isn't preserved */
1049          for (unsigned j = reg.byte() & ~(written - 1); j < written; j++) {
1050             unsigned written_reg = reg.reg() * 4u + j;
1051             if (regs[written_reg] && regs[written_reg] != def.tempId())
1052                err |= ra_fail(program, loc, assignments[regs[written_reg]].defloc,
1053                               "Assignment of element %d of %%%d overwrites the full register "
1054                               "taken by %%%d from instruction",
1055                               i, tmp.id(), regs[written_reg]);
1056          }
1057       }
1058    }
1059
1060    for (const Definition& def : instr->definitions) {
1061       if (!def.isTemp())
1062          continue;
1063       if (def.isKill()) {
1064          for (unsigned j = 0; j < def.getTemp().bytes(); j++)
1065             regs[def.physReg().reg_b + j] = 0;
1066       }
1067    }
1068
1069    return err;
1070 }
1071
1072 } /* end namespace */
1073
1074 bool
1075 validate_ra(Program* program)
1076 {
1077    if (!(debug_flags & DEBUG_VALIDATE_RA))
1078       return false;
1079
1080    bool err = false;
1081    aco::live live_vars = aco::live_var_analysis(program);
1082    std::vector<std::vector<Temp>> phi_sgpr_ops(program->blocks.size());
1083    uint16_t sgpr_limit = get_addr_sgpr_from_waves(program, program->num_waves);
1084
1085    std::vector<Assignment> assignments(program->peekAllocationId());
1086    for (Block& block : program->blocks) {
1087       Location loc;
1088       loc.block = &block;
1089       for (aco_ptr<Instruction>& instr : block.instructions) {
1090          if (instr->opcode == aco_opcode::p_phi) {
1091             for (unsigned i = 0; i < instr->operands.size(); i++) {
1092                if (instr->operands[i].isTemp() &&
1093                    instr->operands[i].getTemp().type() == RegType::sgpr &&
1094                    instr->operands[i].isFirstKill())
1095                   phi_sgpr_ops[block.logical_preds[i]].emplace_back(instr->operands[i].getTemp());
1096             }
1097          }
1098
1099          loc.instr = instr.get();
1100          for (unsigned i = 0; i < instr->operands.size(); i++) {
1101             Operand& op = instr->operands[i];
1102             if (!op.isTemp())
1103                continue;
1104             if (!op.isFixed())
1105                err |= ra_fail(program, loc, Location(), "Operand %d is not assigned a register", i);
1106             if (assignments[op.tempId()].valid && assignments[op.tempId()].reg != op.physReg())
1107                err |=
1108                   ra_fail(program, loc, assignments[op.tempId()].firstloc,
1109                           "Operand %d has an inconsistent register assignment with instruction", i);
1110             if ((op.getTemp().type() == RegType::vgpr &&
1111                  op.physReg().reg_b + op.bytes() > (256 + program->config->num_vgprs) * 4) ||
1112                 (op.getTemp().type() == RegType::sgpr &&
1113                  op.physReg() + op.size() > program->config->num_sgprs &&
1114                  op.physReg() < sgpr_limit))
1115                err |= ra_fail(program, loc, assignments[op.tempId()].firstloc,
1116                               "Operand %d has an out-of-bounds register assignment", i);
1117             if (op.physReg() == vcc && !program->needs_vcc)
1118                err |= ra_fail(program, loc, Location(),
1119                               "Operand %d fixed to vcc but needs_vcc=false", i);
1120             if (op.regClass().is_subdword() &&
1121                 !validate_subdword_operand(program->gfx_level, instr, i))
1122                err |= ra_fail(program, loc, Location(), "Operand %d not aligned correctly", i);
1123             if (!assignments[op.tempId()].firstloc.block)
1124                assignments[op.tempId()].firstloc = loc;
1125             if (!assignments[op.tempId()].defloc.block) {
1126                assignments[op.tempId()].reg = op.physReg();
1127                assignments[op.tempId()].valid = true;
1128             }
1129          }
1130
1131          for (unsigned i = 0; i < instr->definitions.size(); i++) {
1132             Definition& def = instr->definitions[i];
1133             if (!def.isTemp())
1134                continue;
1135             if (!def.isFixed())
1136                err |=
1137                   ra_fail(program, loc, Location(), "Definition %d is not assigned a register", i);
1138             if (assignments[def.tempId()].defloc.block)
1139                err |= ra_fail(program, loc, assignments[def.tempId()].defloc,
1140                               "Temporary %%%d also defined by instruction", def.tempId());
1141             if ((def.getTemp().type() == RegType::vgpr &&
1142                  def.physReg().reg_b + def.bytes() > (256 + program->config->num_vgprs) * 4) ||
1143                 (def.getTemp().type() == RegType::sgpr &&
1144                  def.physReg() + def.size() > program->config->num_sgprs &&
1145                  def.physReg() < sgpr_limit))
1146                err |= ra_fail(program, loc, assignments[def.tempId()].firstloc,
1147                               "Definition %d has an out-of-bounds register assignment", i);
1148             if (def.physReg() == vcc && !program->needs_vcc)
1149                err |= ra_fail(program, loc, Location(),
1150                               "Definition %d fixed to vcc but needs_vcc=false", i);
1151             if (def.regClass().is_subdword() &&
1152                 !validate_subdword_definition(program->gfx_level, instr))
1153                err |= ra_fail(program, loc, Location(), "Definition %d not aligned correctly", i);
1154             if (!assignments[def.tempId()].firstloc.block)
1155                assignments[def.tempId()].firstloc = loc;
1156             assignments[def.tempId()].defloc = loc;
1157             assignments[def.tempId()].reg = def.physReg();
1158             assignments[def.tempId()].valid = true;
1159          }
1160       }
1161    }
1162
1163    for (Block& block : program->blocks) {
1164       Location loc;
1165       loc.block = &block;
1166
1167       std::array<unsigned, 2048> regs; /* register file in bytes */
1168       regs.fill(0);
1169
1170       IDSet live = live_vars.live_out[block.index];
1171       /* remove killed p_phi sgpr operands */
1172       for (Temp tmp : phi_sgpr_ops[block.index])
1173          live.erase(tmp.id());
1174
1175       /* check live out */
1176       for (unsigned id : live) {
1177          Temp tmp(id, program->temp_rc[id]);
1178          PhysReg reg = assignments[id].reg;
1179          for (unsigned i = 0; i < tmp.bytes(); i++) {
1180             if (regs[reg.reg_b + i]) {
1181                err |= ra_fail(program, loc, Location(),
1182                               "Assignment of element %d of %%%d already taken by %%%d in live-out",
1183                               i, id, regs[reg.reg_b + i]);
1184             }
1185             regs[reg.reg_b + i] = id;
1186          }
1187       }
1188       regs.fill(0);
1189
1190       for (auto it = block.instructions.rbegin(); it != block.instructions.rend(); ++it) {
1191          aco_ptr<Instruction>& instr = *it;
1192
1193          /* check killed p_phi sgpr operands */
1194          if (instr->opcode == aco_opcode::p_logical_end) {
1195             for (Temp tmp : phi_sgpr_ops[block.index]) {
1196                PhysReg reg = assignments[tmp.id()].reg;
1197                for (unsigned i = 0; i < tmp.bytes(); i++) {
1198                   if (regs[reg.reg_b + i])
1199                      err |= ra_fail(
1200                         program, loc, Location(),
1201                         "Assignment of element %d of %%%d already taken by %%%d in live-out", i,
1202                         tmp.id(), regs[reg.reg_b + i]);
1203                }
1204                live.insert(tmp.id());
1205             }
1206          }
1207
1208          for (const Definition& def : instr->definitions) {
1209             if (!def.isTemp())
1210                continue;
1211             live.erase(def.tempId());
1212          }
1213
1214          /* don't count phi operands as live-in, since they are actually
1215           * killed when they are copied at the predecessor */
1216          if (instr->opcode != aco_opcode::p_phi && instr->opcode != aco_opcode::p_linear_phi) {
1217             for (const Operand& op : instr->operands) {
1218                if (!op.isTemp())
1219                   continue;
1220                live.insert(op.tempId());
1221             }
1222          }
1223       }
1224
1225       for (unsigned id : live) {
1226          Temp tmp(id, program->temp_rc[id]);
1227          PhysReg reg = assignments[id].reg;
1228          for (unsigned i = 0; i < tmp.bytes(); i++)
1229             regs[reg.reg_b + i] = id;
1230       }
1231
1232       for (aco_ptr<Instruction>& instr : block.instructions) {
1233          loc.instr = instr.get();
1234
1235          /* remove killed p_phi operands from regs */
1236          if (instr->opcode == aco_opcode::p_logical_end) {
1237             for (Temp tmp : phi_sgpr_ops[block.index]) {
1238                PhysReg reg = assignments[tmp.id()].reg;
1239                for (unsigned i = 0; i < tmp.bytes(); i++)
1240                   regs[reg.reg_b + i] = 0;
1241             }
1242          }
1243
1244          if (instr->opcode != aco_opcode::p_phi && instr->opcode != aco_opcode::p_linear_phi) {
1245             for (const Operand& op : instr->operands) {
1246                if (!op.isTemp())
1247                   continue;
1248                if (op.isFirstKillBeforeDef()) {
1249                   for (unsigned j = 0; j < op.getTemp().bytes(); j++)
1250                      regs[op.physReg().reg_b + j] = 0;
1251                }
1252             }
1253          }
1254
1255          if (!instr->isBranch() || block.linear_succs.size() != 1)
1256             err |= validate_instr_defs(program, regs, assignments, loc, instr);
1257
1258          if (!is_phi(instr)) {
1259             for (const Operand& op : instr->operands) {
1260                if (!op.isTemp())
1261                   continue;
1262                if (op.isLateKill() && op.isFirstKill()) {
1263                   for (unsigned j = 0; j < op.getTemp().bytes(); j++)
1264                      regs[op.physReg().reg_b + j] = 0;
1265                }
1266             }
1267          } else if (block.linear_preds.size() != 1 ||
1268                     program->blocks[block.linear_preds[0]].linear_succs.size() == 1) {
1269             for (unsigned pred : block.linear_preds) {
1270                aco_ptr<Instruction>& br = program->blocks[pred].instructions.back();
1271                assert(br->isBranch());
1272                err |= validate_instr_defs(program, regs, assignments, loc, br);
1273             }
1274          }
1275       }
1276    }
1277
1278    return err;
1279 }
1280 } // namespace aco