From 03b2c34793b62b285627f18753559c1c5f360756 Mon Sep 17 00:00:00 2001
From: Alyssa Rosenzweig <alyssa@rosenzweig.io>
Date: Thu, 20 Jul 2023 08:21:30 -0400
Subject: [PATCH] nir: Remove register arrays

Nothing produces them any more, so remove them from NIR. This massively reduces
the size of nir_src, which should improve performance all over.

nir_src size reduced from 56 bytes -> 40 bytes (pahole results on arm64, x86_64
should be similar.)

Signed-off-by: Alyssa Rosenzweig <alyssa@rosenzweig.io>
Reviewed-by: Faith Ekstrand <faith.ekstrand@collabora.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/24253>
---
 src/broadcom/compiler/nir_to_vir.c       |  2 -
 src/compiler/nir/nir.c                   | 99 +++++---------------------------
 src/compiler/nir/nir.h                   |  6 --
 src/compiler/nir/nir_clone.c             | 10 ----
 src/compiler/nir/nir_from_ssa.c          |  5 +-
 src/compiler/nir/nir_inline_helpers.h    | 23 +-------
 src/compiler/nir/nir_instr_set.c         | 11 +---
 src/compiler/nir/nir_lower_vec_to_movs.c |  5 +-
 src/compiler/nir/nir_print.c             | 18 ------
 src/compiler/nir/nir_serialize.c         | 27 +--------
 src/compiler/nir/nir_sweep.c             | 21 -------
 src/compiler/nir/nir_validate.c          | 23 --------
 src/gallium/drivers/vc4/vc4_program.c    |  2 -
 13 files changed, 21 insertions(+), 231 deletions(-)

diff --git a/src/broadcom/compiler/nir_to_vir.c b/src/broadcom/compiler/nir_to_vir.c
index ca07297..9935d95 100644
--- a/src/broadcom/compiler/nir_to_vir.c
+++ b/src/broadcom/compiler/nir_to_vir.c
@@ -787,7 +787,6 @@ ntq_store_dest(struct v3d_compile *c, nir_dest *dest, int chan,
                 qregs[chan] = result;
         } else {
                 nir_register *reg = dest->reg.reg;
-                assert(dest->reg.base_offset == 0);
                 assert(reg->num_array_elems == 0);
                 struct hash_entry *entry =
                         _mesa_hash_table_search(c->def_ht, reg);
@@ -854,7 +853,6 @@ ntq_get_src(struct v3d_compile *c, nir_src src, int i)
         } else {
                 nir_register *reg = src.reg.reg;
                 assert(reg->num_array_elems == 0);
-                assert(src.reg.base_offset == 0);
                 assert(i < reg->num_components);
 
                 if (_mesa_set_search(c->tmu.outstanding_regs, reg))
diff --git a/src/compiler/nir/nir.c b/src/compiler/nir/nir.c
index 7214c55..92dc2bf 100644
--- a/src/compiler/nir/nir.c
+++ b/src/compiler/nir/nir.c
@@ -517,46 +517,14 @@ nir_function_create(nir_shader *shader, const char *name)
    return func;
 }
 
-static bool src_has_indirect(nir_src *src)
-{
-   return !src->is_ssa && src->reg.indirect;
-}
-
-static void src_free_indirects(nir_src *src)
-{
-   if (src_has_indirect(src)) {
-      assert(src->reg.indirect->is_ssa || !src->reg.indirect->reg.indirect);
-      gc_free(src->reg.indirect);
-      src->reg.indirect = NULL;
-   }
-}
-
-static void dest_free_indirects(nir_dest *dest)
-{
-   if (!dest->is_ssa && dest->reg.indirect) {
-      assert(dest->reg.indirect->is_ssa || !dest->reg.indirect->reg.indirect);
-      gc_free(dest->reg.indirect);
-      dest->reg.indirect = NULL;
-   }
-}
-
 static void
 src_copy(nir_src *dest, const nir_src *src, gc_ctx *ctx)
 {
-   src_free_indirects(dest);
-
    dest->is_ssa = src->is_ssa;
    if (src->is_ssa) {
       dest->ssa = src->ssa;
    } else {
-      dest->reg.base_offset = src->reg.base_offset;
       dest->reg.reg = src->reg.reg;
-      if (src->reg.indirect) {
-         dest->reg.indirect = gc_zalloc(ctx, nir_src, 1);
-         src_copy(dest->reg.indirect, src->reg.indirect, ctx);
-      } else {
-         dest->reg.indirect = NULL;
-      }
    }
 }
 
@@ -573,18 +541,9 @@ void nir_dest_copy(nir_dest *dest, const nir_dest *src, nir_instr *instr)
    /* Copying an SSA definition makes no sense whatsoever. */
    assert(!src->is_ssa);
 
-   dest_free_indirects(dest);
-
    dest->is_ssa = false;
 
-   dest->reg.base_offset = src->reg.base_offset;
    dest->reg.reg = src->reg.reg;
-   if (src->reg.indirect) {
-      dest->reg.indirect = gc_zalloc(gc_get_context(instr), nir_src, 1);
-      nir_src_copy(dest->reg.indirect, src->reg.indirect, instr);
-   } else {
-      dest->reg.indirect = NULL;
-   }
 }
 
 void
@@ -703,8 +662,6 @@ src_init(nir_src *src)
 {
    src->is_ssa = false;
    src->reg.reg = NULL;
-   src->reg.indirect = NULL;
-   src->reg.base_offset = 0;
 }
 
 nir_if *
@@ -765,8 +722,6 @@ dest_init(nir_dest *dest)
 {
    dest->is_ssa = false;
    dest->reg.reg = NULL;
-   dest->reg.indirect = NULL;
-   dest->reg.base_offset = 0;
 }
 
 static void
@@ -1312,23 +1267,8 @@ void nir_instr_remove_v(nir_instr *instr)
    }
 }
 
-static bool free_src_indirects_cb(nir_src *src, void *state)
-{
-   src_free_indirects(src);
-   return true;
-}
-
-static bool free_dest_indirects_cb(nir_dest *dest, void *state)
-{
-   dest_free_indirects(dest);
-   return true;
-}
-
 void nir_instr_free(nir_instr *instr)
 {
-   nir_foreach_src(instr, free_src_indirects_cb, NULL);
-   nir_foreach_dest(instr, free_dest_indirects_cb, NULL);
-
    switch (instr->type) {
    case nir_instr_type_tex:
       gc_free(nir_instr_as_tex(instr)->src);
@@ -1711,33 +1651,30 @@ nir_src_is_always_uniform(nir_src src)
 static void
 src_remove_all_uses(nir_src *src)
 {
-   for (; src; src = src->is_ssa ? NULL : src->reg.indirect) {
-      if (!src_is_valid(src))
-         continue;
-
+   if (src && src_is_valid(src))
       list_del(&src->use_link);
-   }
 }
 
 static void
 src_add_all_uses(nir_src *src, nir_instr *parent_instr, nir_if *parent_if)
 {
-   for (; src; src = src->is_ssa ? NULL : src->reg.indirect) {
-      if (!src_is_valid(src))
-         continue;
+   if (!src)
+      return;
 
-      if (parent_instr) {
-         nir_src_set_parent_instr(src, parent_instr);
-      } else {
-         assert(parent_if);
-         nir_src_set_parent_if(src, parent_if);
-      }
+   if (!src_is_valid(src))
+      return;
 
-      if (src->is_ssa)
-         list_addtail(&src->use_link, &src->ssa->uses);
-      else
-         list_addtail(&src->use_link, &src->reg.reg->uses);
+   if (parent_instr) {
+      nir_src_set_parent_instr(src, parent_instr);
+   } else {
+      assert(parent_if);
+      nir_src_set_parent_if(src, parent_if);
    }
+
+   if (src->is_ssa)
+      list_addtail(&src->use_link, &src->ssa->uses);
+   else
+      list_addtail(&src->use_link, &src->reg.reg->uses);
 }
 
 void
@@ -1756,7 +1693,6 @@ nir_instr_move_src(nir_instr *dest_instr, nir_src *dest, nir_src *src)
    assert(!src_is_valid(dest) || dest->parent_instr == dest_instr);
 
    src_remove_all_uses(dest);
-   src_free_indirects(dest);
    src_remove_all_uses(src);
    *dest = *src;
    *src = NIR_SRC_INIT;
@@ -1783,8 +1719,6 @@ nir_instr_rewrite_dest(nir_instr *instr, nir_dest *dest, nir_dest new_dest)
       assert(nir_ssa_def_is_unused(&dest->ssa));
    } else {
       list_del(&dest->reg.def_link);
-      if (dest->reg.indirect)
-         src_remove_all_uses(dest->reg.indirect);
    }
 
    /* We can't re-write with an SSA def */
@@ -1794,9 +1728,6 @@ nir_instr_rewrite_dest(nir_instr *instr, nir_dest *dest, nir_dest new_dest)
 
    dest->reg.parent_instr = instr;
    list_addtail(&dest->reg.def_link, &new_dest.reg.reg->defs);
-
-   if (dest->reg.indirect)
-      src_add_all_uses(dest->reg.indirect, instr, NULL);
 }
 
 void
diff --git a/src/compiler/nir/nir.h b/src/compiler/nir/nir.h
index 2ace569..6e0a927 100644
--- a/src/compiler/nir/nir.h
+++ b/src/compiler/nir/nir.h
@@ -994,8 +994,6 @@ struct nir_src;
 
 typedef struct {
    nir_register *reg;
-   struct nir_src *indirect; /** < NULL for no indirect offset */
-   unsigned base_offset;
 } nir_register_src;
 
 typedef struct {
@@ -1003,8 +1001,6 @@ typedef struct {
    struct list_head def_link;
 
    nir_register *reg;
-   struct nir_src *indirect; /** < NULL for no indirect offset */
-   unsigned base_offset;
 } nir_register_dest;
 
 struct nir_if;
@@ -1123,8 +1119,6 @@ nir_src_for_reg(nir_register *reg)
 
    src.is_ssa = false;
    src.reg.reg = reg;
-   src.reg.indirect = NULL;
-   src.reg.base_offset = 0;
 
    return src;
 }
diff --git a/src/compiler/nir/nir_clone.c b/src/compiler/nir/nir_clone.c
index 4d96c1d..8fadc78 100644
--- a/src/compiler/nir/nir_clone.c
+++ b/src/compiler/nir/nir_clone.c
@@ -243,11 +243,6 @@ __clone_src(clone_state *state, void *ninstr_or_if,
       nsrc->ssa = remap_local(state, src->ssa);
    } else {
       nsrc->reg.reg = remap_reg(state, src->reg.reg);
-      if (src->reg.indirect) {
-         nsrc->reg.indirect = gc_alloc(state->ns->gctx, nir_src, 1);
-         __clone_src(state, ninstr_or_if, nsrc->reg.indirect, src->reg.indirect);
-      }
-      nsrc->reg.base_offset = src->reg.base_offset;
    }
 }
 
@@ -263,11 +258,6 @@ __clone_dst(clone_state *state, nir_instr *ninstr,
          add_remap(state, &ndst->ssa, &dst->ssa);
    } else {
       ndst->reg.reg = remap_reg(state, dst->reg.reg);
-      if (dst->reg.indirect) {
-         ndst->reg.indirect = gc_alloc(state->ns->gctx, nir_src, 1);
-         __clone_src(state, ninstr, ndst->reg.indirect, dst->reg.indirect);
-      }
-      ndst->reg.base_offset = dst->reg.base_offset;
    }
 }
 
diff --git a/src/compiler/nir/nir_from_ssa.c b/src/compiler/nir/nir_from_ssa.c
index dc9f594..ff47684 100644
--- a/src/compiler/nir/nir_from_ssa.c
+++ b/src/compiler/nir/nir_from_ssa.c
@@ -858,10 +858,7 @@ resolve_registers_impl(nir_function_impl *impl, struct from_ssa_state *state)
 static void
 emit_copy(nir_builder *b, nir_src src, nir_src dest_src)
 {
-   assert(!dest_src.is_ssa &&
-          dest_src.reg.indirect == NULL &&
-          dest_src.reg.base_offset == 0);
-
+   assert(!dest_src.is_ssa);
    assert(!nir_src_is_divergent(src) || nir_src_is_divergent(dest_src));
 
    if (src.is_ssa)
diff --git a/src/compiler/nir/nir_inline_helpers.h b/src/compiler/nir/nir_inline_helpers.h
index 21d552e..57095cc 100644
--- a/src/compiler/nir/nir_inline_helpers.h
+++ b/src/compiler/nir/nir_inline_helpers.h
@@ -46,24 +46,6 @@ _nir_visit_src(nir_src *src, nir_foreach_src_cb cb, void *state)
 {
    if (!cb(src, state))
       return false;
-   if (!src->is_ssa && src->reg.indirect)
-      return cb(src->reg.indirect, state);
-   return true;
-}
-
-typedef struct {
-   void *state;
-   nir_foreach_src_cb cb;
-} _nir_visit_dest_indirect_state;
-
-static ALWAYS_INLINE bool
-_nir_visit_dest_indirect(nir_dest *dest, void *_state)
-{
-   _nir_visit_dest_indirect_state *state = (_nir_visit_dest_indirect_state *) _state;
-
-   if (!dest->is_ssa && dest->reg.indirect)
-      return state->cb(dest->reg.indirect, state->state);
-
    return true;
 }
 
@@ -159,8 +141,5 @@ nir_foreach_src(nir_instr *instr, nir_foreach_src_cb cb, void *state)
       break;
    }
 
-   _nir_visit_dest_indirect_state dest_state;
-   dest_state.state = state;
-   dest_state.cb = cb;
-   return _nir_foreach_dest(instr, _nir_visit_dest_indirect, &dest_state);
+   return true;
 }
diff --git a/src/compiler/nir/nir_instr_set.c b/src/compiler/nir/nir_instr_set.c
index 985db65..8794e59 100644
--- a/src/compiler/nir/nir_instr_set.c
+++ b/src/compiler/nir/nir_instr_set.c
@@ -338,16 +338,7 @@ nir_srcs_equal(nir_src src1, nir_src src2)
       if (src2.is_ssa) {
          return false;
       } else {
-         if ((src1.reg.indirect == NULL) != (src2.reg.indirect == NULL))
-            return false;
-
-         if (src1.reg.indirect) {
-            if (!nir_srcs_equal(*src1.reg.indirect, *src2.reg.indirect))
-               return false;
-         }
-
-         return src1.reg.reg == src2.reg.reg &&
-                src1.reg.base_offset == src2.reg.base_offset;
+         return src1.reg.reg == src2.reg.reg;
       }
    }
 }
diff --git a/src/compiler/nir/nir_lower_vec_to_movs.c b/src/compiler/nir/nir_lower_vec_to_movs.c
index 94bf296..e3ff04e 100644
--- a/src/compiler/nir/nir_lower_vec_to_movs.c
+++ b/src/compiler/nir/nir_lower_vec_to_movs.c
@@ -40,10 +40,7 @@ src_matches_dest_reg(nir_dest *dest, nir_src *src)
    if (dest->is_ssa || src->is_ssa)
       return false;
 
-   return (dest->reg.reg == src->reg.reg &&
-           dest->reg.base_offset == src->reg.base_offset &&
-           !dest->reg.indirect &&
-           !src->reg.indirect);
+   return (dest->reg.reg == src->reg.reg);
 }
 
 /**
diff --git a/src/compiler/nir/nir_print.c b/src/compiler/nir/nir_print.c
index 7c2bc3e..74c1382 100644
--- a/src/compiler/nir/nir_print.c
+++ b/src/compiler/nir/nir_print.c
@@ -405,16 +405,7 @@ static void print_src(const nir_src *src, print_state *state, nir_alu_type src_t
 static void
 print_reg_src(const nir_register_src *src, print_state *state)
 {
-   FILE *fp = state->fp;
    print_register(src->reg, state);
-   if (src->reg->num_array_elems != 0) {
-      fprintf(fp, "[%u", src->base_offset);
-      if (src->indirect != NULL) {
-         fprintf(fp, " + ");
-         print_src(src->indirect, state, nir_type_invalid);
-      }
-      fprintf(fp, "]");
-   }
 }
 
 static void
@@ -428,15 +419,6 @@ print_reg_dest(nir_register_dest *dest, print_state *state)
    count_digits(state->max_dest_index) - count_digits(dest->reg->index) : 0;
    fprintf(fp, "%s      %*sr%u", divergence_status(state, dest->reg->divergent),
            padding, "", dest->reg->index);
-
-   if (dest->reg->num_array_elems != 0) {
-      fprintf(fp, "[%u", dest->base_offset);
-      if (dest->indirect != NULL) {
-         fprintf(fp, " + ");
-         print_src(dest->indirect, state, nir_type_invalid);
-      }
-      fprintf(fp, "]");
-   }
 }
 
 static void
diff --git a/src/compiler/nir/nir_serialize.c b/src/compiler/nir/nir_serialize.c
index 1648b3b..9d5d345 100644
--- a/src/compiler/nir/nir_serialize.c
+++ b/src/compiler/nir/nir_serialize.c
@@ -531,13 +531,8 @@ write_src_full(write_ctx *ctx, const nir_src *src, union packed_src header)
       blob_write_uint32(ctx->blob, header.u32);
    } else {
       header.any.object_idx = write_lookup_object(ctx, src->reg.reg);
-      header.any.is_indirect = !!src->reg.indirect;
+      header.any.is_indirect = false;
       blob_write_uint32(ctx->blob, header.u32);
-      blob_write_uint32(ctx->blob, src->reg.base_offset);
-      if (src->reg.indirect) {
-         union packed_src header = {0};
-         write_src_full(ctx, src->reg.indirect, header);
-      }
    }
 }
 
@@ -560,13 +555,6 @@ read_src(read_ctx *ctx, nir_src *src)
       src->ssa = read_lookup_object(ctx, header.any.object_idx);
    } else {
       src->reg.reg = read_lookup_object(ctx, header.any.object_idx);
-      src->reg.base_offset = blob_read_uint32(ctx->blob);
-      if (header.any.is_indirect) {
-         src->reg.indirect = gc_alloc(ctx->nir->gctx, nir_src, 1);
-         read_src(ctx, src->reg.indirect);
-      } else {
-         src->reg.indirect = NULL;
-      }
    }
    return header;
 }
@@ -581,8 +569,7 @@ union packed_dest {
    } ssa;
    struct {
       uint8_t is_ssa:1;
-      uint8_t is_indirect:1;
-      uint8_t _pad:6;
+      uint8_t _pad:7;
    } reg;
 };
 
@@ -704,8 +691,6 @@ write_dest(write_ctx *ctx, const nir_dest *dst, union packed_instr header,
          encode_num_components_in_3bits(dst->ssa.num_components);
       dest.ssa.bit_size = encode_bit_size_3bits(dst->ssa.bit_size);
       dest.ssa.divergent = dst->ssa.divergent;
-   } else {
-      dest.reg.is_indirect = !!(dst->reg.indirect);
    }
    header.any.dest = dest.u8;
 
@@ -756,9 +741,6 @@ write_dest(write_ctx *ctx, const nir_dest *dst, union packed_instr header,
       write_add_object(ctx, &dst->ssa);
    } else {
       blob_write_uint32(ctx->blob, write_lookup_object(ctx, dst->reg.reg));
-      blob_write_uint32(ctx->blob, dst->reg.base_offset);
-      if (dst->reg.indirect)
-         write_src(ctx, dst->reg.indirect);
    }
 }
 
@@ -781,11 +763,6 @@ read_dest(read_ctx *ctx, nir_dest *dst, nir_instr *instr,
       read_add_object(ctx, &dst->ssa);
    } else {
       dst->reg.reg = read_object(ctx);
-      dst->reg.base_offset = blob_read_uint32(ctx->blob);
-      if (dest.reg.is_indirect) {
-         dst->reg.indirect = gc_alloc(ctx->nir->gctx, nir_src, 1);
-         read_src(ctx, dst->reg.indirect);
-      }
    }
 }
 
diff --git a/src/compiler/nir/nir_sweep.c b/src/compiler/nir/nir_sweep.c
index cdb75c6..4ebe12f 100644
--- a/src/compiler/nir/nir_sweep.c
+++ b/src/compiler/nir/nir_sweep.c
@@ -40,24 +40,6 @@
 
 static void sweep_cf_node(nir_shader *nir, nir_cf_node *cf_node);
 
-static bool
-sweep_src_indirect(nir_src *src, void *nir)
-{
-   if (!src->is_ssa && src->reg.indirect)
-      gc_mark_live(((nir_shader*)nir)->gctx, src->reg.indirect);
-
-   return true;
-}
-
-static bool
-sweep_dest_indirect(nir_dest *dest, void *nir)
-{
-   if (!dest->is_ssa && dest->reg.indirect)
-      gc_mark_live(((nir_shader*)nir)->gctx, dest->reg.indirect);
-
-   return true;
-}
-
 static void
 sweep_block(nir_shader *nir, nir_block *block)
 {
@@ -86,9 +68,6 @@ sweep_block(nir_shader *nir, nir_block *block)
       default:
          break;
       }
-
-      nir_foreach_src(instr, sweep_src_indirect, nir);
-      nir_foreach_dest(instr, sweep_dest_indirect, nir);
    }
 }
 
diff --git a/src/compiler/nir/nir_validate.c b/src/compiler/nir/nir_validate.c
index 6f2eab2..7dc23a3 100644
--- a/src/compiler/nir/nir_validate.c
+++ b/src/compiler/nir/nir_validate.c
@@ -174,18 +174,6 @@ validate_reg_src(nir_src *src, validate_state *state,
       validate_assert(state, src->reg.reg->bit_size & bit_sizes);
    if (num_components)
       validate_assert(state, src->reg.reg->num_components == num_components);
-
-   validate_assert(state, (src->reg.reg->num_array_elems == 0 ||
-          src->reg.base_offset < src->reg.reg->num_array_elems) &&
-          "definitely out-of-bounds array access");
-
-   if (src->reg.indirect) {
-      validate_assert(state, src->reg.reg->num_array_elems != 0);
-      validate_assert(state, (src->reg.indirect->is_ssa ||
-              src->reg.indirect->reg.indirect == NULL) &&
-             "only one level of indirection allowed");
-      validate_src(src->reg.indirect, state, 32, 1);
-   }
 }
 
 static void
@@ -269,17 +257,6 @@ validate_reg_dest(nir_register_dest *dest, validate_state *state,
       validate_assert(state, dest->reg->bit_size & bit_sizes);
    if (num_components)
       validate_assert(state, dest->reg->num_components == num_components);
-
-   validate_assert(state, (dest->reg->num_array_elems == 0 ||
-          dest->base_offset < dest->reg->num_array_elems) &&
-          "definitely out-of-bounds array access");
-
-   if (dest->indirect) {
-      validate_assert(state, dest->reg->num_array_elems != 0);
-      validate_assert(state, (dest->indirect->is_ssa || dest->indirect->reg.indirect == NULL) &&
-             "only one level of indirection allowed");
-      validate_src(dest->indirect, state, 32, 1);
-   }
 }
 
 static void
diff --git a/src/gallium/drivers/vc4/vc4_program.c b/src/gallium/drivers/vc4/vc4_program.c
index 03a5d65..99ffa99 100644
--- a/src/gallium/drivers/vc4/vc4_program.c
+++ b/src/gallium/drivers/vc4/vc4_program.c
@@ -211,7 +211,6 @@ ntq_store_dest(struct vc4_compile *c, nir_dest *dest, int chan,
                 qregs[chan] = result;
         } else {
                 nir_register *reg = dest->reg.reg;
-                assert(dest->reg.base_offset == 0);
                 assert(reg->num_array_elems == 0);
                 struct hash_entry *entry =
                         _mesa_hash_table_search(c->def_ht, reg);
@@ -260,7 +259,6 @@ ntq_get_src(struct vc4_compile *c, nir_src src, int i)
                 nir_register *reg = src.reg.reg;
                 entry = _mesa_hash_table_search(c->def_ht, reg);
                 assert(reg->num_array_elems == 0);
-                assert(src.reg.base_offset == 0);
                 assert(i < reg->num_components);
         }
 
-- 
2.7.4