pan/bi: Simplify spill code

Now allow spilling all nodes. Fixes failed spilling in

dEQP-GLES31.functional.ssbo.layout.random.all_shared_buffer.21

Signed-off-by: Alyssa Rosenzweig <alyssa@collabora.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/11123>

diff --git a/src/panfrost/bifrost/bi_ra.c b/src/panfrost/bifrost/bi_ra.c
index 16218f1..b5befa7 100644 (file)
--- a/src/panfrost/bifrost/bi_ra.c
+++ b/src/panfrost/bifrost/bi_ra.c
@@ -361,8 +361,10 @@ bi_choose_spill_node(bi_context *ctx, struct lcra_state *l)
 
         bi_foreach_instr_global(ctx, ins) {
                 bi_foreach_dest(ins, d) {
-                        if (ins->no_spill || ins->dest[d].offset)
-                                BITSET_SET(no_spill, bi_get_node(ins->dest[d]));
+                        unsigned node = bi_get_node(ins->dest[d]);
+
+                        if (node < l->node_count && ins->no_spill)
+                                BITSET_SET(no_spill, node);
                 }
         }
 
@@ -371,8 +373,7 @@ bi_choose_spill_node(bi_context *ctx, struct lcra_state *l)
         unsigned best_benefit = 0.0;
         signed best_node = -1;
 
-        /* Skip register nodes (with bit 0 set) */
-        for (unsigned i = 0; i < l->node_count; i += 2) {
+        for (unsigned i = 0; i < l->node_count; ++i) {
                 if (BITSET_TEST(no_spill, i)) continue;
 
                 unsigned benefit = lcra_count_constraints(l, i);
@@ -387,100 +388,62 @@ bi_choose_spill_node(bi_context *ctx, struct lcra_state *l)
         return best_node;
 }
 
-static void
-bi_spill_dest(bi_builder *b, bi_index index, bi_index temp, uint32_t offset,
-                bi_instr *instr, bi_block *block, unsigned channels)
+static unsigned
+bi_count_read_index(bi_instr *I, bi_index index)
 {
-        b->cursor = bi_after_instr(instr);
-        bi_store(b, channels * 32, temp, bi_imm_u32(offset), bi_zero(),
-                        BI_SEG_TL);
+        unsigned max = 0;
 
-        b->shader->spills++;
-}
-
-static void
-bi_fill_src(bi_builder *b, bi_index index, bi_index temp, uint32_t offset,
-                bi_instr *instr, bi_block *block, unsigned channels)
-{
-        b->cursor = bi_before_instr(instr);
-        bi_instr *ld = bi_load_to(b, channels * 32, temp,
-                        bi_imm_u32(offset), bi_zero(), BI_SEG_TL);
-        ld->no_spill = true;
+        bi_foreach_src(I, s) {
+                if (bi_is_equiv(I->src[s], index)) {
+                        unsigned count = bi_count_read_registers(I, s);
+                        max = MAX2(max, count + I->src[s].offset);
+                }
+        }
 
-        b->shader->fills++;
+        return max;
 }
 
+/* Once we've chosen a spill node, spill it and returns bytes spilled */
+
 static unsigned
-bi_instr_mark_spill(bi_context *ctx, bi_block *block,
-                bi_instr *ins, bi_index index, bi_index *temp)
+bi_spill_register(bi_context *ctx, bi_index index, uint32_t offset)
 {
+        bi_builder b = { .shader = ctx };
         unsigned channels = 0;
 
-        bi_foreach_dest(ins, d) {
-                if (!bi_is_equiv(ins->dest[d], index)) continue;
-                if (bi_is_null(*temp)) *temp = bi_temp_reg(ctx);
-                ins->no_spill = true;
-
-                unsigned offset = ins->dest[d].offset;
-                ins->dest[d] = bi_replace_index(ins->dest[d], *temp);
-                ins->dest[d].offset = offset;
-
-                unsigned newc = util_last_bit(bi_writemask(ins, d)) >> 2;
-                channels = MAX2(channels, newc);
-        }
-
-        return channels;
-}
+        /* Spill after every store, fill before every load */
+        bi_foreach_instr_global_safe(ctx, I) {
+                bi_foreach_dest(I, d) {
+                        if (!bi_is_equiv(I->dest[d], index)) continue;
 
-static bool
-bi_instr_mark_fill(bi_context *ctx, bi_block *block, bi_instr *ins,
-                bi_index index, bi_index *temp)
-{
-        if (!bi_has_arg(ins, index)) return false;
-        if (bi_is_null(*temp)) *temp = bi_temp_reg(ctx);
-        bi_rewrite_index_src_single(ins, index, *temp);
-        return true;
-}
+                        unsigned extra = I->dest[d].offset;
+                        bi_index tmp = bi_temp(ctx);
 
-/* Once we've chosen a spill node, spill it. Precondition: node is a valid
- * SSA node in the non-optimized scheduled IR that was not already
- * spilled (enforced by bi_choose_spill_node). Returns bytes spilled */
+                        I->dest[d] = bi_replace_index(I->dest[d], tmp);
+                        I->no_spill = true;
 
-static unsigned
-bi_spill_register(bi_context *ctx, bi_index index, uint32_t offset)
-{
-        assert(!index.reg);
+                        unsigned count = bi_count_write_registers(I, d);
+                        unsigned bits = count * 32;
 
-        bi_builder _b = { .shader = ctx };
-        unsigned channels = 1;
+                        b.cursor = bi_after_instr(I);
+                        bi_index loc = bi_imm_u32(offset + 4 * extra);
+                        bi_store(&b, bits, tmp, loc, bi_zero(), BI_SEG_TL);
 
-        /* Spill after every store, fill before every load */
-        bi_foreach_block(ctx, _block) {
-                bi_block *block = (bi_block *) _block;
-                bi_foreach_instr_in_block_safe(block, instr) {
-                        bi_index tmp;
-                        unsigned local_channels = bi_instr_mark_spill(ctx,
-                                        block, instr, index, &tmp);
-
-                        channels = MAX2(channels, local_channels);
-
-                        if (local_channels) {
-                                bi_spill_dest(&_b, index, tmp, offset,
-                                                instr, block, channels);
-                        }
+                        ctx->spills++;
+                        channels = MAX2(channels, extra + count);
+                }
 
-                        /* For SSA form, if we write/spill, there was no prior
-                         * contents to fill, so don't waste time reading
-                         * garbage */
+                if (bi_has_arg(I, index)) {
+                        b.cursor = bi_before_instr(I);
+                        bi_index tmp = bi_temp(ctx);
 
-                        bool should_fill = !local_channels || index.reg;
-                        should_fill &= bi_instr_mark_fill(ctx, block, instr,
-                                        index, &tmp);
+                        unsigned bits = bi_count_read_index(I, index) * 32;
+                        bi_rewrite_index_src_single(I, index, tmp);
 
-                        if (should_fill) {
-                                bi_fill_src(&_b, index, tmp, offset, instr,
-                                                block, channels);
-                        }
+                        bi_instr *ld = bi_load_to(&b, bits, tmp,
+                                        bi_imm_u32(offset), bi_zero(), BI_SEG_TL);
+                        ld->no_spill = true;
+                        ctx->fills++;
                 }
         }