src/compiler/nir/nir_lower_bit_size.c

   1 /*
   2  * Copyright © 2018 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  */
  23
  24 #include "nir_builder.h"
  25
  26 /**
  27  * Some ALU operations may not be supported in hardware in specific bit-sizes.
  28  * This pass allows implementations to selectively lower such operations to
  29  * a bit-size that is supported natively and then converts the result back to
  30  * the original bit-size.
  31  */
  32
  33 static nir_ssa_def *convert_to_bit_size(nir_builder *bld, nir_ssa_def *src,
  34                                         nir_alu_type type, unsigned bit_size)
  35 {
  36    assert(src->bit_size < bit_size);
  37
  38    /* create b2i32(a) instead of i2i32(b2i8(a))/i2i32(b2i16(a)) */
  39    nir_alu_instr *alu = nir_src_as_alu_instr(nir_src_for_ssa(src));
  40    if ((type & (nir_type_uint | nir_type_int)) && bit_size == 32 &&
  41        alu && (alu->op == nir_op_b2i8 || alu->op == nir_op_b2i16)) {
  42       nir_alu_instr *instr = nir_alu_instr_create(bld->shader, nir_op_b2i32);
  43       nir_alu_src_copy(&instr->src[0], &alu->src[0], instr);
  44       return nir_builder_alu_instr_finish_and_insert(bld, instr);
  45    }
  46
  47    return nir_convert_to_bit_size(bld, src, type, bit_size);
  48 }
  49
  50 static void
  51 lower_alu_instr(nir_builder *bld, nir_alu_instr *alu, unsigned bit_size)
  52 {
  53    const nir_op op = alu->op;
  54    unsigned dst_bit_size = alu->dest.dest.ssa.bit_size;
  55
  56    bld->cursor = nir_before_instr(&alu->instr);
  57
  58    /* Convert each source to the requested bit-size */
  59    nir_ssa_def *srcs[NIR_MAX_VEC_COMPONENTS] = { NULL };
  60    for (unsigned i = 0; i < nir_op_infos[op].num_inputs; i++) {
  61       nir_ssa_def *src = nir_ssa_for_alu_src(bld, alu, i);
  62
  63       nir_alu_type type = nir_op_infos[op].input_types[i];
  64       if (nir_alu_type_get_type_size(type) == 0)
  65          src = convert_to_bit_size(bld, src, type, bit_size);
  66
  67       if (i == 1 && (op == nir_op_ishl || op == nir_op_ishr || op == nir_op_ushr ||
  68                      op == nir_op_bitz || op == nir_op_bitz8 || op == nir_op_bitz16 ||
  69                      op == nir_op_bitz32 || op == nir_op_bitnz || op == nir_op_bitnz8 ||
  70                      op == nir_op_bitnz16 || op == nir_op_bitnz32)) {
  71          assert(util_is_power_of_two_nonzero(dst_bit_size));
  72          src = nir_iand(bld, src, nir_imm_int(bld, dst_bit_size - 1));
  73       }
  74
  75       srcs[i] = src;
  76    }
  77
  78    /* Emit the lowered ALU instruction */
  79    nir_ssa_def *lowered_dst = NULL;
  80    if (op == nir_op_imul_high || op == nir_op_umul_high) {
  81       assert(dst_bit_size * 2 <= bit_size);
  82       lowered_dst = nir_imul(bld, srcs[0], srcs[1]);
  83       if (nir_op_infos[op].output_type & nir_type_uint)
  84          lowered_dst = nir_ushr_imm(bld, lowered_dst, dst_bit_size);
  85       else
  86          lowered_dst = nir_ishr_imm(bld, lowered_dst, dst_bit_size);
  87    } else if (op == nir_op_iadd_sat || op == nir_op_isub_sat || op == nir_op_uadd_sat ||
  88               op == nir_op_uadd_carry) {
  89       if (op == nir_op_isub_sat)
  90          lowered_dst = nir_isub(bld, srcs[0], srcs[1]);
  91       else
  92          lowered_dst = nir_iadd(bld, srcs[0], srcs[1]);
  93
  94       /* The add_sat and sub_sat instructions need to clamp the result to the
  95        * range of the original type.
  96        */
  97       if (op == nir_op_iadd_sat || op == nir_op_isub_sat) {
  98          const int64_t int_max = u_intN_max(dst_bit_size);
  99          const int64_t int_min = u_intN_min(dst_bit_size);
 100
 101          lowered_dst = nir_iclamp(bld, lowered_dst,
 102                                   nir_imm_intN_t(bld, int_min, bit_size),
 103                                   nir_imm_intN_t(bld, int_max, bit_size));
 104       } else if (op == nir_op_uadd_sat) {
 105          const uint64_t uint_max = u_uintN_max(dst_bit_size);
 106
 107          lowered_dst = nir_umin(bld, lowered_dst,
 108                                 nir_imm_intN_t(bld, uint_max, bit_size));
 109       } else {
 110          assert(op == nir_op_uadd_carry);
 111          lowered_dst = nir_ushr_imm(bld, lowered_dst, dst_bit_size);
 112       }
 113    } else {
 114       lowered_dst = nir_build_alu_src_arr(bld, op, srcs);
 115    }
 116
 117
 118    /* Convert result back to the original bit-size */
 119    if (nir_alu_type_get_type_size(nir_op_infos[op].output_type) == 0 &&
 120        dst_bit_size != bit_size) {
 121       nir_alu_type type = nir_op_infos[op].output_type;
 122       nir_ssa_def *dst = nir_convert_to_bit_size(bld, lowered_dst, type, dst_bit_size);
 123       nir_ssa_def_rewrite_uses(&alu->dest.dest.ssa, dst);
 124    } else {
 125       nir_ssa_def_rewrite_uses(&alu->dest.dest.ssa, lowered_dst);
 126    }
 127 }
 128
 129 static void
 130 lower_intrinsic_instr(nir_builder *b, nir_intrinsic_instr *intrin,
 131                       unsigned bit_size)
 132 {
 133    switch (intrin->intrinsic) {
 134    case nir_intrinsic_read_invocation:
 135    case nir_intrinsic_read_first_invocation:
 136    case nir_intrinsic_vote_feq:
 137    case nir_intrinsic_vote_ieq:
 138    case nir_intrinsic_shuffle:
 139    case nir_intrinsic_shuffle_xor:
 140    case nir_intrinsic_shuffle_up:
 141    case nir_intrinsic_shuffle_down:
 142    case nir_intrinsic_quad_broadcast:
 143    case nir_intrinsic_quad_swap_horizontal:
 144    case nir_intrinsic_quad_swap_vertical:
 145    case nir_intrinsic_quad_swap_diagonal:
 146    case nir_intrinsic_reduce:
 147    case nir_intrinsic_inclusive_scan:
 148    case nir_intrinsic_exclusive_scan: {
 149       assert(intrin->src[0].is_ssa && intrin->dest.is_ssa);
 150       const unsigned old_bit_size = intrin->dest.ssa.bit_size;
 151       assert(old_bit_size < bit_size);
 152
 153       nir_alu_type type = nir_type_uint;
 154       if (nir_intrinsic_has_reduction_op(intrin))
 155          type = nir_op_infos[nir_intrinsic_reduction_op(intrin)].input_types[0];
 156       else if (intrin->intrinsic == nir_intrinsic_vote_feq)
 157          type = nir_type_float;
 158
 159       b->cursor = nir_before_instr(&intrin->instr);
 160       nir_intrinsic_instr *new_intrin =
 161          nir_instr_as_intrinsic(nir_instr_clone(b->shader, &intrin->instr));
 162
 163       nir_ssa_def *new_src = nir_convert_to_bit_size(b, intrin->src[0].ssa,
 164                                                      type, bit_size);
 165       new_intrin->src[0] = nir_src_for_ssa(new_src);
 166
 167       if (intrin->intrinsic == nir_intrinsic_vote_feq ||
 168           intrin->intrinsic == nir_intrinsic_vote_ieq) {
 169          /* These return a Boolean; it's always 1-bit */
 170          assert(new_intrin->dest.ssa.bit_size == 1);
 171       } else {
 172          /* These return the same bit size as the source; we need to adjust
 173           * the size and then we'll have to emit a down-cast.
 174           */
 175          assert(intrin->src[0].ssa->bit_size == intrin->dest.ssa.bit_size);
 176          new_intrin->dest.ssa.bit_size = bit_size;
 177       }
 178
 179       nir_builder_instr_insert(b, &new_intrin->instr);
 180
 181       nir_ssa_def *res = &new_intrin->dest.ssa;
 182       if (intrin->intrinsic == nir_intrinsic_exclusive_scan) {
 183          /* For exclusive scan, we have to be careful because the identity
 184           * value for the higher bit size may get added into the mix by
 185           * disabled channels.  For some cases (imin/imax in particular),
 186           * this value won't convert to the right identity value when we
 187           * down-cast so we have to clamp it.
 188           */
 189          switch (nir_intrinsic_reduction_op(intrin)) {
 190          case nir_op_imin: {
 191             int64_t int_max = (1ull << (old_bit_size - 1)) - 1;
 192             res = nir_imin(b, res, nir_imm_intN_t(b, int_max, bit_size));
 193             break;
 194          }
 195          case nir_op_imax: {
 196             int64_t int_min = -(int64_t)(1ull << (old_bit_size - 1));
 197             res = nir_imax(b, res, nir_imm_intN_t(b, int_min, bit_size));
 198             break;
 199          }
 200          default:
 201             break;
 202          }
 203       }
 204
 205       if (intrin->intrinsic != nir_intrinsic_vote_feq &&
 206           intrin->intrinsic != nir_intrinsic_vote_ieq)
 207          res = nir_u2uN(b, res, old_bit_size);
 208
 209       nir_ssa_def_rewrite_uses(&intrin->dest.ssa, res);
 210       break;
 211    }
 212
 213    default:
 214       unreachable("Unsupported instruction");
 215    }
 216 }
 217
 218 static void
 219 lower_phi_instr(nir_builder *b, nir_phi_instr *phi, unsigned bit_size,
 220                 nir_phi_instr *last_phi)
 221 {
 222    assert(phi->dest.is_ssa);
 223    unsigned old_bit_size = phi->dest.ssa.bit_size;
 224    assert(old_bit_size < bit_size);
 225
 226    nir_foreach_phi_src(src, phi) {
 227       b->cursor = nir_after_block_before_jump(src->pred);
 228       assert(src->src.is_ssa);
 229       nir_ssa_def *new_src = nir_u2uN(b, src->src.ssa, bit_size);
 230
 231       nir_instr_rewrite_src(&phi->instr, &src->src, nir_src_for_ssa(new_src));
 232    }
 233
 234    phi->dest.ssa.bit_size = bit_size;
 235
 236    b->cursor = nir_after_instr(&last_phi->instr);
 237
 238    nir_ssa_def *new_dest = nir_u2uN(b, &phi->dest.ssa, old_bit_size);
 239    nir_ssa_def_rewrite_uses_after(&phi->dest.ssa, new_dest,
 240                                   new_dest->parent_instr);
 241 }
 242
 243 static bool
 244 lower_impl(nir_function_impl *impl,
 245            nir_lower_bit_size_callback callback,
 246            void *callback_data)
 247 {
 248    nir_builder b = nir_builder_create(impl);
 249    bool progress = false;
 250
 251    nir_foreach_block(block, impl) {
 252       /* Stash this so we can rewrite phi destinations quickly. */
 253       nir_phi_instr *last_phi = nir_block_last_phi_instr(block);
 254
 255       nir_foreach_instr_safe(instr, block) {
 256          unsigned lower_bit_size = callback(instr, callback_data);
 257          if (lower_bit_size == 0)
 258             continue;
 259
 260          switch (instr->type) {
 261          case nir_instr_type_alu:
 262             lower_alu_instr(&b, nir_instr_as_alu(instr), lower_bit_size);
 263             break;
 264
 265          case nir_instr_type_intrinsic:
 266             lower_intrinsic_instr(&b, nir_instr_as_intrinsic(instr),
 267                                   lower_bit_size);
 268             break;
 269
 270          case nir_instr_type_phi:
 271             lower_phi_instr(&b, nir_instr_as_phi(instr),
 272                             lower_bit_size, last_phi);
 273             break;
 274
 275          default:
 276             unreachable("Unsupported instruction type");
 277          }
 278          progress = true;
 279       }
 280    }
 281
 282    if (progress) {
 283       nir_metadata_preserve(impl, nir_metadata_block_index |
 284                                   nir_metadata_dominance);
 285    } else {
 286       nir_metadata_preserve(impl, nir_metadata_all);
 287    }
 288
 289    return progress;
 290 }
 291
 292 bool
 293 nir_lower_bit_size(nir_shader *shader,
 294                    nir_lower_bit_size_callback callback,
 295                    void *callback_data)
 296 {
 297    bool progress = false;
 298
 299    nir_foreach_function_impl(impl, shader) {
 300       progress |= lower_impl(impl, callback, callback_data);
 301    }
 302
 303    return progress;
 304 }
 305
 306 static void
 307 split_phi(nir_builder *b, nir_phi_instr *phi)
 308 {
 309    nir_phi_instr *lowered[2] = {
 310       nir_phi_instr_create(b->shader),
 311       nir_phi_instr_create(b->shader)
 312    };
 313    int num_components = phi->dest.ssa.num_components;
 314    assert(phi->dest.ssa.bit_size == 64);
 315
 316    nir_foreach_phi_src(src, phi) {
 317       assert(num_components == src->src.ssa->num_components);
 318
 319       b->cursor = nir_before_src(&src->src);
 320
 321       nir_ssa_def *x = nir_unpack_64_2x32_split_x(b, src->src.ssa);
 322       nir_ssa_def *y = nir_unpack_64_2x32_split_y(b, src->src.ssa);
 323
 324       nir_phi_instr_add_src(lowered[0], src->pred, nir_src_for_ssa(x));
 325       nir_phi_instr_add_src(lowered[1], src->pred, nir_src_for_ssa(y));
 326    }
 327
 328    nir_ssa_dest_init(&lowered[0]->instr, &lowered[0]->dest, num_components,
 329                      32);
 330    nir_ssa_dest_init(&lowered[1]->instr, &lowered[1]->dest, num_components,
 331                      32);
 332
 333    b->cursor = nir_before_instr(&phi->instr);
 334    nir_builder_instr_insert(b, &lowered[0]->instr);
 335    nir_builder_instr_insert(b, &lowered[1]->instr);
 336
 337    b->cursor = nir_after_phis(nir_cursor_current_block(b->cursor));
 338    nir_ssa_def *merged = nir_pack_64_2x32_split(b, &lowered[0]->dest.ssa, &lowered[1]->dest.ssa);
 339    nir_ssa_def_rewrite_uses(&phi->dest.ssa, merged);
 340    nir_instr_remove(&phi->instr);
 341 }
 342
 343 static bool
 344 lower_64bit_phi_instr(nir_builder *b, nir_instr *instr, UNUSED void *cb_data)
 345 {
 346    if (instr->type != nir_instr_type_phi)
 347       return false;
 348
 349    nir_phi_instr *phi = nir_instr_as_phi(instr);
 350    assert(phi->dest.is_ssa);
 351
 352    if (phi->dest.ssa.bit_size <= 32)
 353       return false;
 354
 355    split_phi(b, phi);
 356    return true;
 357 }
 358
 359 bool
 360 nir_lower_64bit_phis(nir_shader *shader)
 361 {
 362    return nir_shader_instructions_pass(shader, lower_64bit_phi_instr,
 363                                        nir_metadata_block_index |
 364                                        nir_metadata_dominance,
 365                                        NULL);
 366 }