radv: vectorize scratch access

fossil-db (gfx1100):
Totals from 20 (0.01% of 133461) affected shaders:
Instrs: 49421 -> 49134 (-0.58%)
CodeSize: 251668 -> 249604 (-0.82%); split: -0.83%, +0.01%
Latency: 178126 -> 178412 (+0.16%); split: -0.16%, +0.32%
InvThroughput: 23565 -> 23646 (+0.34%); split: -0.05%, +0.39%
VClause: 957 -> 943 (-1.46%)
Copies: 5770 -> 5801 (+0.54%); split: -0.36%, +0.90%
PreVGPRs: 1368 -> 1359 (-0.66%)

Regressions seem to be a couple of cases of bad RA luck.

Signed-off-by: Rhys Perry <pendingchaos02@gmail.com>
Reviewed-by: Samuel Pitoiset <samuel.pitoiset@gmail.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/24350>

diff --git a/src/amd/vulkan/radv_pipeline.c b/src/amd/vulkan/radv_pipeline.c
index bab0694..9766f32 100644 (file)
--- a/src/amd/vulkan/radv_pipeline.c
+++ b/src/amd/vulkan/radv_pipeline.c
@@ -330,7 +330,9 @@ radv_mem_vectorize_callback(unsigned align_mul, unsigned align_offset, unsigned
    bool is_scratch = false;
    switch (low->intrinsic) {
    case nir_intrinsic_load_stack:
+   case nir_intrinsic_load_scratch:
    case nir_intrinsic_store_stack:
+   case nir_intrinsic_store_scratch:
       is_scratch = true;
       break;
    default:
@@ -356,7 +358,9 @@ radv_mem_vectorize_callback(unsigned align_mul, unsigned align_offset, unsigned
    case nir_intrinsic_load_ubo:
    case nir_intrinsic_load_push_constant:
    case nir_intrinsic_load_stack:
-   case nir_intrinsic_store_stack: {
+   case nir_intrinsic_load_scratch:
+   case nir_intrinsic_store_stack:
+   case nir_intrinsic_store_scratch: {
       unsigned max_components;
       if (align % 4 == 0)
          max_components = NIR_MAX_VEC_COMPONENTS;
@@ -565,7 +569,8 @@ radv_postprocess_nir(struct radv_device *device, const struct radv_pipeline_key
    NIR_PASS(_, stage->nir, nir_lower_memory_model);
 
    nir_load_store_vectorize_options vectorize_opts = {
-      .modes = nir_var_mem_ssbo | nir_var_mem_ubo | nir_var_mem_push_const | nir_var_mem_shared | nir_var_mem_global,
+      .modes = nir_var_mem_ssbo | nir_var_mem_ubo | nir_var_mem_push_const | nir_var_mem_shared | nir_var_mem_global |
+               nir_var_shader_temp,
       .callback = radv_mem_vectorize_callback,
       .cb_data = &gfx_level,
       .robust_modes = 0,