From: Jason Ekstrand <jason@jlekstrand.net>
Date: Mon, 20 Jul 2020 21:30:37 +0000 (-0500)
Subject: nir: Use a single list for all shader variables
X-Git-Tag: upstream/21.0.0~7020
X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=d70fff99c5bc3a721e20869e7f0be8024ffe5ecd;p=platform%2Fupstream%2Fmesa.git

nir: Use a single list for all shader variables

Instead of having separate lists of variables, roughly sorted by mode,
use a single list for all shader-level NIR variables.  This makes a few
list walks a bit longer here and there but list walks aren't a very
common thing in NIR at all.  On the other hand, it makes a lot of things
like validation, printing, etc. way simpler.  Also, there are a number
of cases where we move variables from inputs/outputs to globals and this
makes it way easier because we no longer have to move them between
lists.  We only have to deal with that if moving them from the shader to
a nir_function_impl.

Reviewed-by: Rob Clark <robdclark@chromium.org>
Reviewed-By: Mike Blumenkrantz <michael.blumenkrantz@gmail.com>
Reviewed-by: Vasily Khoruzhick <anarsoul@gmail.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/5966>
---

diff --git a/src/compiler/glsl/gl_nir_link_uniform_blocks.c b/src/compiler/glsl/gl_nir_link_uniform_blocks.c
index 07f1614..1a7fdf3 100644
--- a/src/compiler/glsl/gl_nir_link_uniform_blocks.c
+++ b/src/compiler/glsl/gl_nir_link_uniform_blocks.c
@@ -417,7 +417,7 @@ allocate_uniform_blocks(void *mem_ctx,
    *num_variables = 0;
    *num_blocks = 0;
 
-   nir_foreach_variable(var, &shader->Program->nir->uniforms) {
+   nir_foreach_variable_in_shader(var, shader->Program->nir) {
       if (block_type == BLOCK_UBO && !nir_variable_is_in_ubo(var))
          continue;
 
@@ -557,7 +557,7 @@ link_linked_shader_uniform_blocks(void *mem_ctx,
    unsigned variable_index = 0;
    struct gl_uniform_block *blks = *blocks;
 
-   nir_foreach_variable(var, &shader->Program->nir->uniforms) {
+   nir_foreach_variable_in_shader(var, shader->Program->nir) {
       if (block_type == BLOCK_UBO && !nir_variable_is_in_ubo(var))
          continue;
 
diff --git a/src/compiler/glsl/gl_nir_linker.h b/src/compiler/glsl/gl_nir_linker.h
index 8240d5f..5171a2c 100644
--- a/src/compiler/glsl/gl_nir_linker.h
+++ b/src/compiler/glsl/gl_nir_linker.h
@@ -36,7 +36,9 @@ struct gl_nir_linker_options {
 };
 
 #define nir_foreach_gl_uniform_variable(var, shader) \
-   nir_foreach_variable(var, &(shader)->uniforms)
+   nir_foreach_variable_with_modes(var, shader, nir_var_uniform | \
+                                                nir_var_mem_ubo | \
+                                                nir_var_mem_ssbo)
 
 bool gl_nir_link_spirv(struct gl_context *ctx,
                        struct gl_shader_program *prog,
diff --git a/src/compiler/nir/nir.c b/src/compiler/nir/nir.c
index a9fe16f..dc56c49 100644
--- a/src/compiler/nir/nir.c
+++ b/src/compiler/nir/nir.c
@@ -44,10 +44,7 @@ nir_shader_create(void *mem_ctx,
 {
    nir_shader *shader = rzalloc(mem_ctx, nir_shader);
 
-   exec_list_make_empty(&shader->uniforms);
-   exec_list_make_empty(&shader->inputs);
-   exec_list_make_empty(&shader->outputs);
-   exec_list_make_empty(&shader->shared);
+   exec_list_make_empty(&shader->variables);
 
    shader->options = options;
 
@@ -59,8 +56,6 @@ nir_shader_create(void *mem_ctx,
    }
 
    exec_list_make_empty(&shader->functions);
-   exec_list_make_empty(&shader->globals);
-   exec_list_make_empty(&shader->system_values);
 
    shader->num_inputs = 0;
    shader->num_outputs = 0;
@@ -104,56 +99,38 @@ nir_reg_remove(nir_register *reg)
    exec_node_remove(&reg->node);
 }
 
-struct exec_list *
-nir_variable_list_for_mode(nir_shader *shader, nir_variable_mode mode)
+void
+nir_shader_add_variable(nir_shader *shader, nir_variable *var)
 {
-   switch (mode) {
+   switch (var->data.mode) {
    case nir_var_function_temp:
       assert(!"nir_shader_add_variable cannot be used for local variables");
-      return NULL;
+      return;
 
    case nir_var_shader_temp:
-      return &shader->globals;
-
    case nir_var_shader_in:
-      return &shader->inputs;
-
    case nir_var_shader_out:
-      return &shader->outputs;
-
    case nir_var_uniform:
    case nir_var_mem_ubo:
    case nir_var_mem_ssbo:
-      return &shader->uniforms;
-
    case nir_var_mem_shared:
-      assert(gl_shader_stage_is_compute(shader->info.stage));
-      return &shader->shared;
+   case nir_var_system_value:
+      break;
 
    case nir_var_mem_global:
       assert(!"nir_shader_add_variable cannot be used for global memory");
-      return NULL;
-
-   case nir_var_system_value:
-      return &shader->system_values;
+      return;
 
    case nir_var_mem_push_const:
       assert(!"nir_var_push_constant is not supposed to be used for variables");
-      return NULL;
+      return;
 
    default:
       assert(!"invalid mode");
-      return NULL;
+      return;
    }
-}
 
-void
-nir_shader_add_variable(nir_shader *shader, nir_variable *var)
-{
-   struct exec_list *var_list =
-      nir_variable_list_for_mode(shader, var->data.mode);
-   if (var_list)
-      exec_list_push_tail(var_list, &var->node);
+   exec_list_push_tail(&shader->variables, &var->node);
 }
 
 nir_variable *
@@ -1820,35 +1797,12 @@ nir_index_instrs(nir_function_impl *impl)
    return index;
 }
 
-static void
-index_var_list(struct exec_list *list, unsigned *count)
-{
-   nir_foreach_variable(var, list)
-      var->index = (*count)++;
-}
-
 unsigned
 nir_shader_index_vars(nir_shader *shader, nir_variable_mode modes)
 {
    unsigned count = 0;
-   if (modes & nir_var_shader_temp)
-      index_var_list(&shader->globals, &count);
-
-   if (modes & nir_var_shader_in)
-      index_var_list(&shader->inputs, &count);
-
-   if (modes & nir_var_shader_out)
-      index_var_list(&shader->outputs, &count);
-
-   if (modes & (nir_var_uniform | nir_var_mem_ubo | nir_var_mem_ssbo))
-      index_var_list(&shader->uniforms, &count);
-
-   if (modes & nir_var_mem_shared)
-      index_var_list(&shader->shared, &count);
-
-   if (modes & nir_var_system_value)
-      index_var_list(&shader->system_values, &count);
-
+   nir_foreach_variable_with_modes(var, shader, modes)
+      var->index = count++;
    return count;
 }
 
@@ -1856,7 +1810,8 @@ unsigned
 nir_function_impl_index_vars(nir_function_impl *impl)
 {
    unsigned count = 0;
-   index_var_list(&impl->locals, &count);
+   nir_foreach_function_temp_variable(var, impl)
+      var->index = count++;
    return count;
 }
 
diff --git a/src/compiler/nir/nir.h b/src/compiler/nir/nir.h
index f131e0b..b216d29 100644
--- a/src/compiler/nir/nir.h
+++ b/src/compiler/nir/nir.h
@@ -618,7 +618,6 @@ typedef struct nir_variable {
    struct nir_variable_data *members;
 } nir_variable;
 
-
 static inline bool
 _nir_shader_variable_has_mode(nir_variable *var, unsigned modes)
 {
@@ -627,39 +626,43 @@ _nir_shader_variable_has_mode(nir_variable *var, unsigned modes)
    return var->data.mode & modes;
 }
 
-#define nir_foreach_variable(var, var_list) \
+#define nir_foreach_variable_in_list(var, var_list) \
    foreach_list_typed(nir_variable, var, node, var_list)
 
-#define nir_foreach_variable_safe(var, var_list) \
+#define nir_foreach_variable_in_list_safe(var, var_list) \
    foreach_list_typed_safe(nir_variable, var, node, var_list)
 
+#define nir_foreach_variable_in_shader(var, shader) \
+   nir_foreach_variable_in_list(var, &(shader)->variables)
+
+#define nir_foreach_variable_in_shader_safe(var, shader) \
+   nir_foreach_variable_in_list_safe(var, &(shader)->variables)
+
 #define nir_foreach_variable_with_modes(var, shader, modes) \
-   nir_foreach_variable(var, nir_variable_list_for_mode(shader, modes)) \
+   nir_foreach_variable_in_shader(var, shader) \
       if (_nir_shader_variable_has_mode(var, modes))
 
 #define nir_foreach_variable_with_modes_safe(var, shader, modes) \
-   nir_foreach_variable_safe(var, nir_variable_list_for_mode(shader, modes)) \
+   nir_foreach_variable_in_shader_safe(var, shader) \
       if (_nir_shader_variable_has_mode(var, modes))
 
 #define nir_foreach_shader_in_variable(var, shader) \
-   nir_foreach_variable(var, &(shader)->inputs)
+   nir_foreach_variable_with_modes(var, shader, nir_var_shader_in)
 
 #define nir_foreach_shader_in_variable_safe(var, shader) \
-   nir_foreach_variable_safe(var, &(shader)->inputs)
+   nir_foreach_variable_with_modes_safe(var, shader, nir_var_shader_in)
 
 #define nir_foreach_shader_out_variable(var, shader) \
-   nir_foreach_variable(var, &(shader)->outputs)
+   nir_foreach_variable_with_modes(var, shader, nir_var_shader_out)
 
 #define nir_foreach_shader_out_variable_safe(var, shader) \
-   nir_foreach_variable_safe(var, &(shader)->outputs)
+   nir_foreach_variable_with_modes_safe(var, shader, nir_var_shader_out)
 
 #define nir_foreach_uniform_variable(var, shader) \
-   nir_foreach_variable(var, &(shader)->uniforms) \
-      if (var->data.mode == nir_var_uniform)
+   nir_foreach_variable_with_modes(var, shader, nir_var_uniform)
 
 #define nir_foreach_uniform_variable_safe(var, shader) \
-   nir_foreach_variable_safe(var, &(shader)->uniforms) \
-      if (var->data.mode == nir_var_uniform)
+   nir_foreach_variable_with_modes_safe(var, shader, nir_var_uniform)
 
 static inline bool
 nir_variable_is_global(const nir_variable *var)
@@ -3218,16 +3221,7 @@ typedef struct nir_shader_compiler_options {
 
 typedef struct nir_shader {
    /** list of uniforms (nir_variable) */
-   struct exec_list uniforms;
-
-   /** list of inputs (nir_variable) */
-   struct exec_list inputs;
-
-   /** list of outputs (nir_variable) */
-   struct exec_list outputs;
-
-   /** list of shared compute variables (nir_variable) */
-   struct exec_list shared;
+   struct exec_list variables;
 
    /** Set of driver-specific options for the shader.
     *
@@ -3239,12 +3233,6 @@ typedef struct nir_shader {
    /** Various bits of compile-time information about a given shader */
    struct shader_info info;
 
-   /** list of global variables in the shader (nir_variable) */
-   struct exec_list globals;
-
-   /** list of system value variables in the shader (nir_variable) */
-   struct exec_list system_values;
-
    struct exec_list functions; /** < list of nir_function */
 
    /**
@@ -3300,9 +3288,6 @@ nir_register *nir_local_reg_create(nir_function_impl *impl);
 
 void nir_reg_remove(nir_register *reg);
 
-struct exec_list *
-nir_variable_list_for_mode(nir_shader *shader, nir_variable_mode mode);
-
 /** Adds a variable to the appropriate list in nir_shader */
 void nir_shader_add_variable(nir_shader *shader, nir_variable *var);
 
diff --git a/src/compiler/nir/nir_clone.c b/src/compiler/nir/nir_clone.c
index 1aaed56..f57fce5 100644
--- a/src/compiler/nir/nir_clone.c
+++ b/src/compiler/nir/nir_clone.c
@@ -728,12 +728,7 @@ nir_shader_clone(void *mem_ctx, const nir_shader *s)
    nir_shader *ns = nir_shader_create(mem_ctx, s->info.stage, s->options, NULL);
    state.ns = ns;
 
-   clone_var_list(&state, &ns->uniforms, &s->uniforms);
-   clone_var_list(&state, &ns->inputs,   &s->inputs);
-   clone_var_list(&state, &ns->outputs,  &s->outputs);
-   clone_var_list(&state, &ns->shared,   &s->shared);
-   clone_var_list(&state, &ns->globals,  &s->globals);
-   clone_var_list(&state, &ns->system_values, &s->system_values);
+   clone_var_list(&state, &ns->variables, &s->variables);
 
    /* Go through and clone functions */
    foreach_list_typed(nir_function, fxn, node, &s->functions)
@@ -796,12 +791,7 @@ nir_shader_replace(nir_shader *dst, nir_shader *src)
    /* We have to move all the linked lists over separately because we need the
     * pointers in the list elements to point to the lists in dst and not src.
     */
-   exec_list_move_nodes_to(&src->uniforms,      &dst->uniforms);
-   exec_list_move_nodes_to(&src->inputs,        &dst->inputs);
-   exec_list_move_nodes_to(&src->outputs,       &dst->outputs);
-   exec_list_move_nodes_to(&src->shared,        &dst->shared);
-   exec_list_move_nodes_to(&src->globals,       &dst->globals);
-   exec_list_move_nodes_to(&src->system_values, &dst->system_values);
+   exec_list_move_nodes_to(&src->variables, &dst->variables);
 
    /* Now move the functions over.  This takes a tiny bit more work */
    exec_list_move_nodes_to(&src->functions, &dst->functions);
diff --git a/src/compiler/nir/nir_linking_helpers.c b/src/compiler/nir/nir_linking_helpers.c
index 9e60117..e5cd450 100644
--- a/src/compiler/nir/nir_linking_helpers.c
+++ b/src/compiler/nir/nir_linking_helpers.c
@@ -128,9 +128,8 @@ nir_remove_unused_io_vars(nir_shader *shader,
    uint64_t *used;
 
    assert(mode == nir_var_shader_in || mode == nir_var_shader_out);
-   struct exec_list *var_list = nir_variable_list_for_mode(shader, mode);
 
-   nir_foreach_variable_safe(var, var_list) {
+   nir_foreach_variable_with_modes_safe(var, shader, mode) {
       if (var->data.patch)
          used = used_by_other_stage_patches;
       else
@@ -152,9 +151,6 @@ nir_remove_unused_io_vars(nir_shader *shader,
          var->data.location = 0;
          var->data.mode = nir_var_shader_temp;
 
-         exec_node_remove(&var->node);
-         exec_list_push_tail(&shader->globals, &var->node);
-
          progress = true;
       }
    }
@@ -1067,7 +1063,7 @@ nir_link_opt_varyings(nir_shader *producer, nir_shader *consumer)
 static void
 insert_sorted(struct exec_list *var_list, nir_variable *new_var)
 {
-   nir_foreach_variable(var, var_list) {
+   nir_foreach_variable_in_list(var, var_list) {
       if (var->data.location > new_var->data.location) {
          exec_node_insert_node_before(&var->node, &new_var->node);
          return;
@@ -1100,7 +1096,7 @@ nir_assign_io_var_locations(nir_shader *shader, nir_variable_mode mode,
 
    int UNUSED last_loc = 0;
    bool last_partial = false;
-   nir_foreach_variable(var, &io_vars) {
+   nir_foreach_variable_in_list(var, &io_vars) {
       const struct glsl_type *type = var->type;
       if (nir_is_per_vertex_io(var, stage) || var->data.per_view) {
          assert(glsl_type_is_array(type));
@@ -1203,8 +1199,7 @@ nir_assign_io_var_locations(nir_shader *shader, nir_variable_mode mode,
    if (last_partial)
       location++;
 
-   struct exec_list *var_list = nir_variable_list_for_mode(shader, mode);
-   exec_list_append(var_list, &io_vars);
+   exec_list_append(&shader->variables, &io_vars);
    *size = location;
 }
 
diff --git a/src/compiler/nir/nir_lower_amul.c b/src/compiler/nir/nir_lower_amul.c
index 561cf77..52a7d8a 100644
--- a/src/compiler/nir/nir_lower_amul.c
+++ b/src/compiler/nir/nir_lower_amul.c
@@ -235,10 +235,8 @@ nir_lower_amul(nir_shader *shader,
    /* uniforms list actually includes ubo's and ssbo's: */
    int max_slot = 0;
 
-   nir_foreach_variable (var, &shader->uniforms) {
-      if (!(var->data.mode & (nir_var_mem_ubo | nir_var_mem_ssbo)))
-         continue;
-
+   nir_foreach_variable_with_modes (var, shader,
+                                    nir_var_mem_ubo | nir_var_mem_ssbo) {
       int base = var->data.binding;
       int size = MAX2(1, glsl_array_size(var->type));
 
@@ -258,7 +256,7 @@ nir_lower_amul(nir_shader *shader,
    /* Figure out which UBOs or SSBOs are large enough to be
     * disqualified from imul24:
     */
-   nir_foreach_variable(var, &shader->uniforms) {
+   nir_foreach_variable_in_shader (var, shader) {
       if (var->data.mode == nir_var_mem_ubo) {
          if (is_large(&state, var)) {
             state.has_large_ubo = true;
diff --git a/src/compiler/nir/nir_lower_clip.c b/src/compiler/nir/nir_lower_clip.c
index 3a4775f..6e8010c 100644
--- a/src/compiler/nir/nir_lower_clip.c
+++ b/src/compiler/nir/nir_lower_clip.c
@@ -65,12 +65,7 @@ create_clipdist_var(nir_shader *shader,
    } else
       var->type = glsl_vec4_type();
 
-   if (output) {
-      exec_list_push_tail(&shader->outputs, &var->node);
-   }
-   else {
-      exec_list_push_tail(&shader->inputs, &var->node);
-   }
+   nir_shader_add_variable(shader, var);
    return var;
 }
 
@@ -245,9 +240,7 @@ lower_clip_outputs(nir_builder *b, nir_variable *position,
       cv = nir_load_var(b, clipvertex ? clipvertex : position);
 
       if (clipvertex) {
-         exec_node_remove(&clipvertex->node);
          clipvertex->data.mode = nir_var_shader_temp;
-         exec_list_push_tail(&b->shader->globals, &clipvertex->node);
          nir_fixup_deref_modes(b->shader);
       }
    } else {
diff --git a/src/compiler/nir/nir_lower_global_vars_to_local.c b/src/compiler/nir/nir_lower_global_vars_to_local.c
index 563fa39..770c33c 100644
--- a/src/compiler/nir/nir_lower_global_vars_to_local.c
+++ b/src/compiler/nir/nir_lower_global_vars_to_local.c
@@ -83,15 +83,13 @@ nir_lower_global_vars_to_local(nir_shader *shader)
       }
    }
 
-   nir_foreach_variable_safe(var, &shader->globals) {
+   nir_foreach_variable_with_modes_safe(var, shader, nir_var_shader_temp) {
       struct hash_entry *entry = _mesa_hash_table_search(var_func_table, var);
       if (!entry)
          continue;
 
       nir_function_impl *impl = entry->data;
 
-      assert(var->data.mode == nir_var_shader_temp);
-
       if (impl != NULL) {
          exec_node_remove(&var->node);
          var->data.mode = nir_var_function_temp;
diff --git a/src/compiler/nir/nir_lower_io.c b/src/compiler/nir/nir_lower_io.c
index f8c3fe6..58c4597 100644
--- a/src/compiler/nir/nir_lower_io.c
+++ b/src/compiler/nir/nir_lower_io.c
@@ -1470,7 +1470,10 @@ lower_vars_to_explicit(nir_shader *shader,
    default:
       unreachable("Unsupported mode");
    }
-   nir_foreach_variable(var, vars) {
+   nir_foreach_variable_in_list(var, vars) {
+      if (var->data.mode != mode)
+         continue;
+
       unsigned size, align;
       const struct glsl_type *explicit_type =
          glsl_get_explicit_type_for_size_align(var->type, type_info, &size, &align);
@@ -1517,9 +1520,9 @@ nir_lower_vars_to_explicit_types(nir_shader *shader,
    bool progress = false;
 
    if (modes & nir_var_mem_shared)
-      progress |= lower_vars_to_explicit(shader, &shader->shared, nir_var_mem_shared, type_info);
+      progress |= lower_vars_to_explicit(shader, &shader->variables, nir_var_mem_shared, type_info);
    if (modes & nir_var_shader_temp)
-      progress |= lower_vars_to_explicit(shader, &shader->globals, nir_var_shader_temp, type_info);
+      progress |= lower_vars_to_explicit(shader, &shader->variables, nir_var_shader_temp, type_info);
 
    nir_foreach_function(function, shader) {
       if (function->impl) {
diff --git a/src/compiler/nir/nir_lower_io_to_temporaries.c b/src/compiler/nir/nir_lower_io_to_temporaries.c
index ae5f64d..eaeea21 100644
--- a/src/compiler/nir/nir_lower_io_to_temporaries.c
+++ b/src/compiler/nir/nir_lower_io_to_temporaries.c
@@ -312,6 +312,16 @@ create_shadow_temp(struct lower_io_state *state, nir_variable *var)
    return nvar;
 }
 
+static void
+move_variables_to_list(nir_shader *shader, nir_variable_mode mode,
+                       struct exec_list *dst_list)
+{
+   nir_foreach_variable_with_modes_safe(var, shader, mode) {
+      exec_node_remove(&var->node);
+      exec_list_push_tail(dst_list, &var->node);
+   }
+}
+
 void
 nir_lower_io_to_temporaries(nir_shader *shader, nir_function_impl *entrypoint,
                             bool outputs, bool inputs)
@@ -325,15 +335,13 @@ nir_lower_io_to_temporaries(nir_shader *shader, nir_function_impl *entrypoint,
    state.entrypoint = entrypoint;
    state.input_map = _mesa_pointer_hash_table_create(NULL);
 
+   exec_list_make_empty(&state.old_inputs);
    if (inputs)
-      exec_list_move_nodes_to(&shader->inputs, &state.old_inputs);
-   else
-      exec_list_make_empty(&state.old_inputs);
+      move_variables_to_list(shader, nir_var_shader_in, &state.old_inputs);
 
+   exec_list_make_empty(&state.old_outputs);
    if (outputs)
-      exec_list_move_nodes_to(&shader->outputs, &state.old_outputs);
-   else
-      exec_list_make_empty(&state.old_outputs);
+      move_variables_to_list(shader, nir_var_shader_out, &state.old_outputs);
 
    exec_list_make_empty(&state.new_inputs);
    exec_list_make_empty(&state.new_outputs);
@@ -341,13 +349,13 @@ nir_lower_io_to_temporaries(nir_shader *shader, nir_function_impl *entrypoint,
    /* Walk over all of the outputs turn each output into a temporary and
     * make a new variable for the actual output.
     */
-   nir_foreach_variable(var, &state.old_outputs) {
+   nir_foreach_variable_in_list(var, &state.old_outputs) {
       nir_variable *output = create_shadow_temp(&state, var);
       exec_list_push_tail(&state.new_outputs, &output->node);
    }
 
    /* and same for inputs: */
-   nir_foreach_variable(var, &state.old_inputs) {
+   nir_foreach_variable_in_list(var, &state.old_inputs) {
       nir_variable *input = create_shadow_temp(&state, var);
       exec_list_push_tail(&state.new_inputs, &input->node);
       _mesa_hash_table_insert(state.input_map, var, input);
@@ -367,10 +375,10 @@ nir_lower_io_to_temporaries(nir_shader *shader, nir_function_impl *entrypoint,
                                             nir_metadata_dominance);
    }
 
-   exec_list_append(&shader->inputs, &state.new_inputs);
-   exec_list_append(&shader->outputs, &state.new_outputs);
-   exec_list_append(&shader->globals, &state.old_inputs);
-   exec_list_append(&shader->globals, &state.old_outputs);
+   exec_list_append(&shader->variables, &state.old_inputs);
+   exec_list_append(&shader->variables, &state.old_outputs);
+   exec_list_append(&shader->variables, &state.new_inputs);
+   exec_list_append(&shader->variables, &state.new_outputs);
 
    nir_fixup_deref_modes(shader);
 
diff --git a/src/compiler/nir/nir_lower_system_values.c b/src/compiler/nir/nir_lower_system_values.c
index bdc337a..05cf510 100644
--- a/src/compiler/nir/nir_lower_system_values.c
+++ b/src/compiler/nir/nir_lower_system_values.c
@@ -328,7 +328,8 @@ nir_lower_system_values(nir_shader *shader)
    if (progress)
       nir_remove_dead_derefs(shader);
 
-   exec_list_make_empty(&shader->system_values);
+   nir_foreach_variable_with_modes_safe(var, shader, nir_var_system_value)
+      exec_node_remove(&var->node);
 
    return progress;
 }
diff --git a/src/compiler/nir/nir_lower_variable_initializers.c b/src/compiler/nir/nir_lower_variable_initializers.c
index 7b90a19..86252d9 100644
--- a/src/compiler/nir/nir_lower_variable_initializers.c
+++ b/src/compiler/nir/nir_lower_variable_initializers.c
@@ -54,13 +54,17 @@ build_constant_load(nir_builder *b, nir_deref_instr *deref, nir_constant *c)
 }
 
 static bool
-lower_const_initializer(struct nir_builder *b, struct exec_list *var_list)
+lower_const_initializer(struct nir_builder *b, struct exec_list *var_list,
+                        nir_variable_mode modes)
 {
    bool progress = false;
 
    b->cursor = nir_before_cf_list(&b->impl->body);
 
-   nir_foreach_variable(var, var_list) {
+   nir_foreach_variable_in_list(var, var_list) {
+      if (!(var->data.mode & modes))
+         continue;
+
       if (var->constant_initializer) {
          build_constant_load(b, nir_build_deref_var(b, var),
                              var->constant_initializer);
@@ -107,17 +111,17 @@ nir_lower_variable_initializers(nir_shader *shader, nir_variable_mode modes)
       nir_builder builder;
       nir_builder_init(&builder, function->impl);
 
-      if ((modes & nir_var_shader_out) && function->is_entrypoint)
-         impl_progress |= lower_const_initializer(&builder, &shader->outputs);
-
-      if ((modes & nir_var_shader_temp) && function->is_entrypoint)
-         impl_progress |= lower_const_initializer(&builder, &shader->globals);
-
-      if ((modes & nir_var_system_value) && function->is_entrypoint)
-         impl_progress |= lower_const_initializer(&builder, &shader->system_values);
+      if ((modes & ~nir_var_function_temp) && function->is_entrypoint) {
+         impl_progress |= lower_const_initializer(&builder,
+                                                  &shader->variables,
+                                                  modes);
+      }
 
-      if (modes & nir_var_function_temp)
-         impl_progress |= lower_const_initializer(&builder, &function->impl->locals);
+      if (modes & nir_var_function_temp) {
+         impl_progress |= lower_const_initializer(&builder,
+                                                  &function->impl->locals,
+                                                  nir_var_function_temp);
+      }
 
       if (impl_progress) {
          progress = true;
diff --git a/src/compiler/nir/nir_opt_access.c b/src/compiler/nir/nir_opt_access.c
index 99ecdb5..d428e29 100644
--- a/src/compiler/nir/nir_opt_access.c
+++ b/src/compiler/nir/nir_opt_access.c
@@ -322,7 +322,9 @@ nir_opt_access(nir_shader *shader)
       }
    }
 
-   nir_foreach_variable(var, &shader->uniforms)
+   nir_foreach_variable_with_modes(var, shader, nir_var_uniform |
+                                                nir_var_mem_ubo |
+                                                nir_var_mem_ssbo)
       var_progress |= process_variable(&state, var);
 
    nir_foreach_function(func, shader) {
diff --git a/src/compiler/nir/nir_print.c b/src/compiler/nir/nir_print.c
index 801744c..0e24a68 100644
--- a/src/compiler/nir/nir_print.c
+++ b/src/compiler/nir/nir_print.c
@@ -936,27 +936,26 @@ print_intrinsic_instr(nir_intrinsic_instr *instr, print_state *state)
    if (!state->shader)
       return;
 
-   struct exec_list *var_list = NULL;
-
+   nir_variable_mode var_mode;
    switch (instr->intrinsic) {
    case nir_intrinsic_load_uniform:
-      var_list = &state->shader->uniforms;
+      var_mode = nir_var_uniform;
       break;
    case nir_intrinsic_load_input:
    case nir_intrinsic_load_interpolated_input:
    case nir_intrinsic_load_per_vertex_input:
-      var_list = &state->shader->inputs;
+      var_mode = nir_var_shader_in;
       break;
    case nir_intrinsic_load_output:
    case nir_intrinsic_store_output:
    case nir_intrinsic_store_per_vertex_output:
-      var_list = &state->shader->outputs;
+      var_mode = nir_var_shader_out;
       break;
    default:
       return;
    }
 
-   nir_foreach_variable(var, var_list) {
+   nir_foreach_variable_with_modes(var, state->shader, var_mode) {
       if ((var->data.driver_location == nir_intrinsic_base(instr)) &&
           (instr->intrinsic == nir_intrinsic_load_uniform ||
            (nir_intrinsic_component(instr) >= var->data.location_frac  &&
@@ -1519,29 +1518,8 @@ nir_print_shader_annotated(nir_shader *shader, FILE *fp,
    if (shader->scratch_size)
       fprintf(fp, "scratch: %u\n", shader->scratch_size);
 
-   nir_foreach_variable(var, &shader->uniforms) {
+   nir_foreach_variable_in_shader(var, shader)
       print_var_decl(var, &state);
-   }
-
-   nir_foreach_variable(var, &shader->inputs) {
-      print_var_decl(var, &state);
-   }
-
-   nir_foreach_variable(var, &shader->outputs) {
-      print_var_decl(var, &state);
-   }
-
-   nir_foreach_variable(var, &shader->shared) {
-      print_var_decl(var, &state);
-   }
-
-   nir_foreach_variable(var, &shader->globals) {
-      print_var_decl(var, &state);
-   }
-
-   nir_foreach_variable(var, &shader->system_values) {
-      print_var_decl(var, &state);
-   }
 
    foreach_list_typed(nir_function, func, node, &shader->functions) {
       print_function(func, &state);
diff --git a/src/compiler/nir/nir_remove_dead_variables.c b/src/compiler/nir/nir_remove_dead_variables.c
index dfeaa24..5037b86 100644
--- a/src/compiler/nir/nir_remove_dead_variables.c
+++ b/src/compiler/nir/nir_remove_dead_variables.c
@@ -148,7 +148,7 @@ remove_dead_vars(struct exec_list *var_list, nir_variable_mode modes,
 {
    bool progress = false;
 
-   nir_foreach_variable_safe(var, var_list) {
+   nir_foreach_variable_in_list_safe(var, var_list) {
       if (!(var->data.mode & modes))
          continue;
 
@@ -176,34 +176,9 @@ nir_remove_dead_variables(nir_shader *shader, nir_variable_mode modes,
 
    add_var_use_shader(shader, live, modes);
 
-   if (modes & nir_var_uniform) {
-      progress = remove_dead_vars(&shader->uniforms, modes, live, can_remove_var) ||
-         progress;
-   }
-
-   if (modes & nir_var_shader_in) {
-      progress = remove_dead_vars(&shader->inputs, modes, live, can_remove_var) ||
-         progress;
-   }
-
-   if (modes & nir_var_shader_out) {
-      progress = remove_dead_vars(&shader->outputs, modes, live, can_remove_var) ||
-         progress;
-   }
-
-   if (modes & nir_var_shader_temp) {
-      progress = remove_dead_vars(&shader->globals, modes, live, can_remove_var) ||
-         progress;
-   }
-
-   if (modes & nir_var_system_value) {
-      progress = remove_dead_vars(&shader->system_values, modes, live,
-                                  can_remove_var) || progress;
-   }
-
-   if (modes & nir_var_mem_shared) {
-      progress = remove_dead_vars(&shader->shared, modes, live, can_remove_var) ||
-         progress;
+   if (modes & ~nir_var_function_temp) {
+      progress = remove_dead_vars(&shader->variables, modes,
+                                  live, can_remove_var) || progress;
    }
 
    if (modes & nir_var_function_temp) {
diff --git a/src/compiler/nir/nir_serialize.c b/src/compiler/nir/nir_serialize.c
index 6688e9e..8baa735 100644
--- a/src/compiler/nir/nir_serialize.c
+++ b/src/compiler/nir/nir_serialize.c
@@ -2013,12 +2013,7 @@ nir_serialize(struct blob *blob, const nir_shader *nir, bool strip)
    info.name = info.label = NULL;
    blob_write_bytes(blob, (uint8_t *) &info, sizeof(info));
 
-   write_var_list(&ctx, &nir->uniforms);
-   write_var_list(&ctx, &nir->inputs);
-   write_var_list(&ctx, &nir->outputs);
-   write_var_list(&ctx, &nir->shared);
-   write_var_list(&ctx, &nir->globals);
-   write_var_list(&ctx, &nir->system_values);
+   write_var_list(&ctx, &nir->variables);
 
    blob_write_uint32(blob, nir->num_inputs);
    blob_write_uint32(blob, nir->num_uniforms);
@@ -2071,12 +2066,7 @@ nir_deserialize(void *mem_ctx,
 
    ctx.nir->info = info;
 
-   read_var_list(&ctx, &ctx.nir->uniforms);
-   read_var_list(&ctx, &ctx.nir->inputs);
-   read_var_list(&ctx, &ctx.nir->outputs);
-   read_var_list(&ctx, &ctx.nir->shared);
-   read_var_list(&ctx, &ctx.nir->globals);
-   read_var_list(&ctx, &ctx.nir->system_values);
+   read_var_list(&ctx, &ctx.nir->variables);
 
    ctx.nir->num_inputs = blob_read_uint32(blob);
    ctx.nir->num_uniforms = blob_read_uint32(blob);
diff --git a/src/compiler/nir/nir_split_vars.c b/src/compiler/nir/nir_split_vars.c
index db32003..6aee310 100644
--- a/src/compiler/nir/nir_split_vars.c
+++ b/src/compiler/nir/nir_split_vars.c
@@ -173,7 +173,7 @@ split_var_list_structs(nir_shader *shader,
    /* To avoid list confusion (we'll be adding things as we split variables),
     * pull all of the variables we plan to split off of the list
     */
-   nir_foreach_variable_safe(var, vars) {
+   nir_foreach_variable_in_list_safe(var, vars) {
       if (var->data.mode != mode)
          continue;
 
@@ -193,7 +193,7 @@ split_var_list_structs(nir_shader *shader,
       exec_list_push_tail(&split_vars, &var->node);
    }
 
-   nir_foreach_variable(var, &split_vars) {
+   nir_foreach_variable_in_list(var, &split_vars) {
       state.base_var = var;
 
       struct field *root_field = ralloc(mem_ctx, struct field);
@@ -308,7 +308,7 @@ nir_split_struct_vars(nir_shader *shader, nir_variable_mode modes)
    bool has_global_splits = false;
    if (modes & nir_var_shader_temp) {
       has_global_splits = split_var_list_structs(shader, NULL,
-                                                 &shader->globals,
+                                                 &shader->variables,
                                                  nir_var_shader_temp,
                                                  var_field_map,
                                                  &complex_vars,
@@ -382,7 +382,7 @@ init_var_list_array_infos(nir_shader *shader,
 {
    bool has_array = false;
 
-   nir_foreach_variable(var, vars) {
+   nir_foreach_variable_in_list(var, vars) {
       if (var->data.mode != mode)
          continue;
 
@@ -554,7 +554,7 @@ split_var_list_arrays(nir_shader *shader,
    struct exec_list split_vars;
    exec_list_make_empty(&split_vars);
 
-   nir_foreach_variable_safe(var, vars) {
+   nir_foreach_variable_in_list_safe(var, vars) {
       if (var->data.mode != mode)
          continue;
 
@@ -601,7 +601,7 @@ split_var_list_arrays(nir_shader *shader,
       }
    }
 
-   nir_foreach_variable(var, &split_vars) {
+   nir_foreach_variable_in_list(var, &split_vars) {
       struct array_var_info *info = get_array_var_info(var, var_info_map);
       create_split_array_vars(info, 0, &info->root_split, var->name,
                               shader, impl, mem_ctx);
@@ -872,7 +872,7 @@ nir_split_array_vars(nir_shader *shader, nir_variable_mode modes)
    bool has_global_array = false;
    if (modes & nir_var_shader_temp) {
       has_global_array = init_var_list_array_infos(shader,
-                                                   &shader->globals,
+                                                   &shader->variables,
                                                    nir_var_shader_temp,
                                                    var_info_map,
                                                    &complex_vars,
@@ -910,7 +910,7 @@ nir_split_array_vars(nir_shader *shader, nir_variable_mode modes)
    bool has_global_splits = false;
    if (modes & nir_var_shader_temp) {
       has_global_splits = split_var_list_arrays(shader, NULL,
-                                                &shader->globals,
+                                                &shader->variables,
                                                 nir_var_shader_temp,
                                                 var_info_map, mem_ctx);
    }
@@ -1270,7 +1270,7 @@ shrink_vec_var_list(struct exec_list *vars,
     * Also, if we have a copy that to/from something we can't shrink, we need
     * to leave components and array_len of any wildcards alone.
     */
-   nir_foreach_variable(var, vars) {
+   nir_foreach_variable_in_list(var, vars) {
       if (var->data.mode != mode)
          continue;
 
@@ -1306,7 +1306,7 @@ shrink_vec_var_list(struct exec_list *vars,
    bool fp_progress;
    do {
       fp_progress = false;
-      nir_foreach_variable(var, vars) {
+      nir_foreach_variable_in_list(var, vars) {
          if (var->data.mode != mode)
             continue;
 
@@ -1346,7 +1346,7 @@ shrink_vec_var_list(struct exec_list *vars,
    } while (fp_progress);
 
    bool vars_shrunk = false;
-   nir_foreach_variable_safe(var, vars) {
+   nir_foreach_variable_in_list_safe(var, vars) {
       if (var->data.mode != mode)
          continue;
 
@@ -1618,8 +1618,11 @@ function_impl_has_vars_with_modes(nir_function_impl *impl,
 {
    nir_shader *shader = impl->function->shader;
 
-   if ((modes & nir_var_shader_temp) && !exec_list_is_empty(&shader->globals))
-      return true;
+   if (modes & ~nir_var_function_temp) {
+      nir_foreach_variable_with_modes(var, shader,
+                                      modes & ~nir_var_function_temp)
+         return true;
+   }
 
    if ((modes & nir_var_function_temp) && !exec_list_is_empty(&impl->locals))
       return true;
@@ -1669,7 +1672,7 @@ nir_shrink_vec_array_vars(nir_shader *shader, nir_variable_mode modes)
 
    bool globals_shrunk = false;
    if (modes & nir_var_shader_temp) {
-      globals_shrunk = shrink_vec_var_list(&shader->globals,
+      globals_shrunk = shrink_vec_var_list(&shader->variables,
                                            nir_var_shader_temp,
                                            var_usage_map);
    }
diff --git a/src/compiler/nir/nir_sweep.c b/src/compiler/nir/nir_sweep.c
index 56b7a26..e2b70f5 100644
--- a/src/compiler/nir/nir_sweep.c
+++ b/src/compiler/nir/nir_sweep.c
@@ -163,12 +163,7 @@ nir_sweep(nir_shader *nir)
       ralloc_steal(nir, (char *)nir->info.label);
 
    /* Variables and registers are not dead.  Steal them back. */
-   steal_list(nir, nir_variable, &nir->uniforms);
-   steal_list(nir, nir_variable, &nir->inputs);
-   steal_list(nir, nir_variable, &nir->outputs);
-   steal_list(nir, nir_variable, &nir->shared);
-   steal_list(nir, nir_variable, &nir->globals);
-   steal_list(nir, nir_variable, &nir->system_values);
+   steal_list(nir, nir_variable, &nir->variables);
 
    /* Recurse into functions, stealing their contents back. */
    foreach_list_typed(nir_function, func, node, &nir->functions) {
diff --git a/src/compiler/nir/nir_validate.c b/src/compiler/nir/nir_validate.c
index 47beec0..6d498b0 100644
--- a/src/compiler/nir/nir_validate.c
+++ b/src/compiler/nir/nir_validate.c
@@ -1288,38 +1288,19 @@ nir_validate_shader(nir_shader *shader, const char *when)
 
    state.shader = shader;
 
-   exec_list_validate(&shader->uniforms);
-   nir_foreach_variable(var, &shader->uniforms) {
-      validate_var_decl(var, nir_var_uniform |
-                             nir_var_mem_ubo |
-                             nir_var_mem_ssbo,
-                        &state);
-   }
-
-   exec_list_validate(&shader->inputs);
-   nir_foreach_variable(var, &shader->inputs) {
-     validate_var_decl(var, nir_var_shader_in, &state);
-   }
-
-   exec_list_validate(&shader->outputs);
-   nir_foreach_variable(var, &shader->outputs) {
-     validate_var_decl(var, nir_var_shader_out, &state);
-   }
-
-   exec_list_validate(&shader->shared);
-   nir_foreach_variable(var, &shader->shared) {
-      validate_var_decl(var, nir_var_mem_shared, &state);
-   }
-
-   exec_list_validate(&shader->globals);
-   nir_foreach_variable(var, &shader->globals) {
-     validate_var_decl(var, nir_var_shader_temp, &state);
-   }
-
-   exec_list_validate(&shader->system_values);
-   nir_foreach_variable(var, &shader->system_values) {
-     validate_var_decl(var, nir_var_system_value, &state);
-   }
+   nir_variable_mode valid_modes =
+      nir_var_shader_in |
+      nir_var_shader_out |
+      nir_var_shader_temp |
+      nir_var_uniform |
+      nir_var_mem_ubo |
+      nir_var_system_value |
+      nir_var_mem_ssbo |
+      nir_var_mem_shared;
+
+   exec_list_validate(&shader->variables);
+   nir_foreach_variable_in_shader(var, shader)
+     validate_var_decl(var, valid_modes, &state);
 
    exec_list_validate(&shader->functions);
    foreach_list_typed(nir_function, func, node, &shader->functions) {
diff --git a/src/compiler/nir/tests/vars_tests.cpp b/src/compiler/nir/tests/vars_tests.cpp
index 5f17ce0..6d27830 100644
--- a/src/compiler/nir/tests/vars_tests.cpp
+++ b/src/compiler/nir/tests/vars_tests.cpp
@@ -81,7 +81,10 @@ protected:
    }
 
    unsigned count_shader_temp_vars(void) {
-      return exec_list_length(&b->shader->globals);
+      unsigned count = 0;
+      nir_foreach_variable_with_modes(var, b->shader, nir_var_shader_temp)
+         count++;
+      return count;
    }
 
    nir_intrinsic_instr *get_intrinsic(nir_intrinsic_op intrinsic,
diff --git a/src/freedreno/ir3/ir3_nir_lower_tess.c b/src/freedreno/ir3/ir3_nir_lower_tess.c
index 86aaa17..44b2921 100644
--- a/src/freedreno/ir3/ir3_nir_lower_tess.c
+++ b/src/freedreno/ir3/ir3_nir_lower_tess.c
@@ -41,6 +41,7 @@ struct state {
 	nir_variable *vertex_flags_out;
 
 	struct exec_list old_outputs;
+	struct exec_list new_outputs;
 	struct exec_list emit_outputs;
 
 	/* tess ctrl shader on a650 gets the local primitive id at different bits: */
@@ -830,14 +831,19 @@ ir3_nir_lower_gs(nir_shader *shader)
 	 * we copy the emit_outputs to the real outputs, so that we get
 	 * store_output in uniform control flow.
 	 */
-	exec_list_move_nodes_to(&shader->outputs, &state.old_outputs);
+	exec_list_make_empty(&state.old_outputs);
+	nir_foreach_shader_out_variable_safe(var, shader) {
+		exec_node_remove(&var->node);
+		exec_list_push_tail(&state.old_outputs, &var->node);
+	}
+	exec_list_make_empty(&state.new_outputs);
 	exec_list_make_empty(&state.emit_outputs);
-	nir_foreach_variable(var, &state.old_outputs) {
+	nir_foreach_variable_in_list(var, &state.old_outputs) {
 		/* Create a new output var by cloning the original output var and
 		 * stealing the name.
 		 */
 		nir_variable *output = nir_variable_clone(var, shader);
-		exec_list_push_tail(&shader->outputs, &output->node);
+		exec_list_push_tail(&state.new_outputs, &output->node);
 
 		/* Rewrite the original output to be a shadow variable. */
 		var->name = ralloc_asprintf(var, "%s@gs-temp", output->name);
@@ -883,15 +889,16 @@ ir3_nir_lower_gs(nir_shader *shader)
 
 		nir_builder_instr_insert(&b, &discard_if->instr);
 
-		foreach_two_lists(dest_node, &shader->outputs, src_node, &state.emit_outputs) {
+		foreach_two_lists(dest_node, &state.new_outputs, src_node, &state.emit_outputs) {
 			nir_variable *dest = exec_node_data(nir_variable, dest_node, node);
 			nir_variable *src = exec_node_data(nir_variable, src_node, node);
 			nir_copy_var(&b, dest, src);
 		}
 	}
 
-	exec_list_append(&shader->globals, &state.old_outputs);
-	exec_list_append(&shader->globals, &state.emit_outputs);
+	exec_list_append(&shader->variables, &state.old_outputs);
+	exec_list_append(&shader->variables, &state.emit_outputs);
+	exec_list_append(&shader->variables, &state.new_outputs);
 
 	nir_metadata_preserve(impl, 0);
 
diff --git a/src/gallium/drivers/freedreno/ir3/ir3_cmdline.c b/src/gallium/drivers/freedreno/ir3/ir3_cmdline.c
index dafc399..15451af 100644
--- a/src/gallium/drivers/freedreno/ir3/ir3_cmdline.c
+++ b/src/gallium/drivers/freedreno/ir3/ir3_cmdline.c
@@ -67,7 +67,7 @@ static void dump_info(struct ir3_shader_variant *so, const char *str)
 static void
 insert_sorted(struct exec_list *var_list, nir_variable *new_var)
 {
-	nir_foreach_variable(var, var_list) {
+	nir_foreach_variable_in_list(var, var_list) {
 		if (var->data.location > new_var->data.location) {
 			exec_node_insert_node_before(&var->node, &new_var->node);
 			return;
@@ -85,7 +85,7 @@ sort_varyings(nir_shader *nir, nir_variable_mode mode)
 		exec_node_remove(&var->node);
 		insert_sorted(&new_list, var);
 	}
-	exec_list_move_nodes_to(&new_list, var_list);
+	exec_list_append(&nir->variables, &new_list);
 }
 
 static void
diff --git a/src/gallium/drivers/iris/iris_program.c b/src/gallium/drivers/iris/iris_program.c
index 0640828..f6c2ab6 100644
--- a/src/gallium/drivers/iris/iris_program.c
+++ b/src/gallium/drivers/iris/iris_program.c
@@ -286,9 +286,7 @@ iris_fix_edge_flags(nir_shader *nir)
       return false;
    }
 
-   exec_node_remove(&var->node);
    var->data.mode = nir_var_shader_temp;
-   exec_list_push_tail(&nir->globals, &var->node);
    nir->info.outputs_written &= ~VARYING_BIT_EDGE;
    nir->info.inputs_read &= ~VERT_BIT_EDGEFLAG;
    nir_fixup_deref_modes(nir);
diff --git a/src/gallium/drivers/lima/standalone/lima_compiler_cmdline.c b/src/gallium/drivers/lima/standalone/lima_compiler_cmdline.c
index a9d1fba..f510e73 100644
--- a/src/gallium/drivers/lima/standalone/lima_compiler_cmdline.c
+++ b/src/gallium/drivers/lima/standalone/lima_compiler_cmdline.c
@@ -48,7 +48,7 @@ print_usage(void)
 static void
 insert_sorted(struct exec_list *var_list, nir_variable *new_var)
 {
-   nir_foreach_variable(var, var_list) {
+   nir_foreach_variable_in_list(var, var_list) {
       if (var->data.location > new_var->data.location &&
           new_var->data.location >= 0) {
          exec_node_insert_node_before(&var->node, &new_var->node);
@@ -67,7 +67,7 @@ sort_varyings(nir_shader *nir, nir_variable_mode mode)
       exec_node_remove(&var->node);
       insert_sorted(&new_list, var);
    }
-   exec_list_move_nodes_to(&new_list, var_list);
+   exec_list_append(&nir->variables, &new_list);
 }
 
 static void
diff --git a/src/gallium/drivers/r600/sfn/sfn_nir.cpp b/src/gallium/drivers/r600/sfn/sfn_nir.cpp
index 250c9e7..d65a1c9 100644
--- a/src/gallium/drivers/r600/sfn/sfn_nir.cpp
+++ b/src/gallium/drivers/r600/sfn/sfn_nir.cpp
@@ -282,7 +282,9 @@ bool ShaderFromNir::process_declaration()
    }
 
    // scan declarations
-   nir_foreach_variable(variable, &sh->uniforms) {
+   nir_foreach_variable_with_modes(variable, sh, nir_var_uniform |
+                                                 nir_var_mem_ubo |
+                                                 nir_var_mem_ssbo) {
       if (!impl->process_uniforms(variable)) {
          fprintf(stderr, "R600: error parsing outputs varible %s\n", variable->name);
          return false;
diff --git a/src/gallium/drivers/zink/nir_to_spirv/nir_to_spirv.c b/src/gallium/drivers/zink/nir_to_spirv/nir_to_spirv.c
index 9fad2bb..68a5d00 100644
--- a/src/gallium/drivers/zink/nir_to_spirv/nir_to_spirv.c
+++ b/src/gallium/drivers/zink/nir_to_spirv/nir_to_spirv.c
@@ -2313,7 +2313,9 @@ nir_to_spirv(struct nir_shader *s, const struct pipe_stream_output_info *so_info
 
    if (so_info)
       emit_so_info(&ctx, util_last_bit64(s->info.outputs_written), so_info, local_so_info);
-   nir_foreach_variable(var, &s->uniforms)
+   nir_foreach_variable_with_modes(var, s, nir_var_uniform |
+                                           nir_var_mem_ubo |
+                                           nir_var_mem_ssbo)
       emit_uniform(&ctx, var);
 
    if (s->info.stage == MESA_SHADER_FRAGMENT) {
diff --git a/src/gallium/drivers/zink/zink_compiler.c b/src/gallium/drivers/zink/zink_compiler.c
index 865e140..4154f79 100644
--- a/src/gallium/drivers/zink/zink_compiler.c
+++ b/src/gallium/drivers/zink/zink_compiler.c
@@ -242,7 +242,8 @@ zink_compile_nir(struct zink_screen *screen, struct nir_shader *nir,
    }
 
    ret->num_bindings = 0;
-   nir_foreach_variable(var, &nir->uniforms) {
+   nir_foreach_variable_with_modes(var, nir, nir_var_uniform |
+                                             nir_var_mem_ubo) {
       if (var->data.mode == nir_var_mem_ubo) {
          int binding = zink_binding(nir->info.stage,
                                     VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER,
diff --git a/src/intel/blorp/blorp.c b/src/intel/blorp/blorp.c
index c3181a1..681c8f1 100644
--- a/src/intel/blorp/blorp.c
+++ b/src/intel/blorp/blorp.c
@@ -180,7 +180,6 @@ blorp_compile_fs(struct blorp_context *blorp, void *mem_ctx,
 
    memset(wm_prog_data, 0, sizeof(*wm_prog_data));
 
-   assert(exec_list_is_empty(&nir->uniforms));
    wm_prog_data->base.nr_params = 0;
    wm_prog_data->base.param = NULL;
 
diff --git a/src/mesa/state_tracker/st_glsl_to_nir.cpp b/src/mesa/state_tracker/st_glsl_to_nir.cpp
index 8b8a108..2b3e0c9 100644
--- a/src/mesa/state_tracker/st_glsl_to_nir.cpp
+++ b/src/mesa/state_tracker/st_glsl_to_nir.cpp
@@ -117,14 +117,12 @@ st_nir_assign_vs_in_locations(struct nir_shader *nir)
             util_bitcount64(nir->info.inputs_read &
                               BITFIELD64_MASK(var->data.location));
       } else {
-         /* Move unused input variables to the globals list (with no
+         /* Convert unused input variables to shader_temp (with no
           * initialization), to avoid confusing drivers looking through the
           * inputs array and expecting to find inputs with a driver_location
           * set.
           */
-         exec_node_remove(&var->node);
          var->data.mode = nir_var_shader_temp;
-         exec_list_push_tail(&nir->globals, &var->node);
          removed_inputs = true;
       }
    }