list_inithead(®->uses);
list_inithead(®->defs);
- list_inithead(®->if_uses);
reg->num_components = 0;
reg->bit_size = 32;
phi_src = gc_zalloc(gc_get_context(instr), nir_phi_src, 1);
phi_src->pred = pred;
phi_src->src = src;
- phi_src->src.parent_instr = &instr->instr;
+ nir_src_set_parent_instr(&phi_src->src, &instr->instr);
exec_list_push_tail(&instr->srcs, &phi_src->node);
return phi_src;
{
nir_instr *instr = state;
- src->parent_instr = instr;
+ nir_src_set_parent_instr(src, instr);
list_addtail(&src->use_link,
src->is_ssa ? &src->ssa->uses : &src->reg.reg->uses);
continue;
if (parent_instr) {
- src->parent_instr = parent_instr;
- if (src->is_ssa)
- list_addtail(&src->use_link, &src->ssa->uses);
- else
- list_addtail(&src->use_link, &src->reg.reg->uses);
+ nir_src_set_parent_instr(src, parent_instr);
} else {
assert(parent_if);
- src->parent_if = parent_if;
- if (src->is_ssa)
- list_addtail(&src->use_link, &src->ssa->if_uses);
- else
- list_addtail(&src->use_link, &src->reg.reg->if_uses);
+ nir_src_set_parent_if(src, parent_if);
}
+
+ if (src->is_ssa)
+ list_addtail(&src->use_link, &src->ssa->uses);
+ else
+ list_addtail(&src->use_link, &src->reg.reg->uses);
}
}
{
nir_shader *shader = ralloc_parent(if_stmt);
nir_src *src = &if_stmt->condition;
- assert(!src_is_valid(src) || src->parent_if == if_stmt);
+ assert(!src_is_valid(src) || (src->is_if && src->parent_if == if_stmt));
src_remove_all_uses(src);
src_copy(src, &new_src, shader->gctx);
{
def->parent_instr = instr;
list_inithead(&def->uses);
- list_inithead(&def->if_uses);
def->num_components = num_components;
def->bit_size = bit_size;
def->divergent = true; /* This is the safer default */
nir_ssa_def_rewrite_uses(nir_ssa_def *def, nir_ssa_def *new_ssa)
{
assert(def != new_ssa);
- nir_foreach_use_safe(use_src, def)
- nir_instr_rewrite_src_ssa(use_src->parent_instr, use_src, new_ssa);
-
- nir_foreach_if_use_safe(use_src, def)
- nir_if_rewrite_condition_ssa(use_src->parent_if, use_src, new_ssa);
+ nir_foreach_use_including_if_safe(use_src, def) {
+ if (use_src->is_if)
+ nir_if_rewrite_condition_ssa(use_src->parent_if, use_src, new_ssa);
+ else
+ nir_instr_rewrite_src_ssa(use_src->parent_instr, use_src, new_ssa);
+ }
}
void
if (new_src.is_ssa) {
nir_ssa_def_rewrite_uses(def, new_src.ssa);
} else {
- nir_foreach_use_safe(use_src, def)
- nir_instr_rewrite_src(use_src->parent_instr, use_src, new_src);
-
- nir_foreach_if_use_safe(use_src, def)
- nir_if_rewrite_condition(use_src->parent_if, new_src);
+ nir_foreach_use_including_if_safe(use_src, def) {
+ if (use_src->is_if)
+ nir_if_rewrite_condition(use_src->parent_if, new_src);
+ else
+ nir_instr_rewrite_src(use_src->parent_instr, use_src, new_src);
+ }
}
}
if (def == new_ssa)
return;
- nir_foreach_use_safe(use_src, def) {
- assert(use_src->parent_instr != def->parent_instr);
- /* Since def already dominates all of its uses, the only way a use can
- * not be dominated by after_me is if it is between def and after_me in
- * the instruction list.
- */
- if (!is_instr_between(def->parent_instr, after_me, use_src->parent_instr))
- nir_instr_rewrite_src_ssa(use_src->parent_instr, use_src, new_ssa);
- }
-
- nir_foreach_if_use_safe(use_src, def) {
- nir_if_rewrite_condition_ssa(use_src->parent_if,
- &use_src->parent_if->condition,
- new_ssa);
+ nir_foreach_use_including_if_safe(use_src, def) {
+ if (use_src->is_if) {
+ nir_if_rewrite_condition_ssa(use_src->parent_if,
+ &use_src->parent_if->condition,
+ new_ssa);
+ } else {
+ assert(use_src->parent_instr != def->parent_instr);
+ /* Since def already dominates all of its uses, the only way a use can
+ * not be dominated by after_me is if it is between def and after_me in
+ * the instruction list.
+ */
+ if (!is_instr_between(def->parent_instr, after_me, use_src->parent_instr))
+ nir_instr_rewrite_src_ssa(use_src->parent_instr, use_src, new_ssa);
+ }
}
}
{
nir_component_mask_t read_mask = 0;
- if (!list_is_empty(&def->if_uses))
- read_mask |= 1;
+ nir_foreach_use_including_if(use, def) {
+ read_mask |= use->is_if ? 1 : nir_src_components_read(use);
- nir_foreach_use(use, def) {
- read_mask |= nir_src_components_read(use);
if (read_mask == (1 << def->num_components) - 1)
return read_mask;
}
assert(nir_foreach_dest(instr, dest_is_ssa, NULL));
nir_ssa_def *old_def = nir_instr_ssa_def(instr);
- struct list_head old_uses, old_if_uses;
+ struct list_head old_uses;
if (old_def != NULL) {
/* We're about to ask the callback to generate a replacement for instr.
* Save off the uses from instr's SSA def so we know what uses to
list_replace(&old_def->uses, &old_uses);
list_inithead(&old_def->uses);
- list_replace(&old_def->if_uses, &old_if_uses);
- list_inithead(&old_def->if_uses);
}
b.cursor = nir_after_instr(instr);
preserved = nir_metadata_none;
nir_src new_src = nir_src_for_ssa(new_def);
- list_for_each_entry_safe(nir_src, use_src, &old_uses, use_link)
- nir_instr_rewrite_src(use_src->parent_instr, use_src, new_src);
-
- list_for_each_entry_safe(nir_src, use_src, &old_if_uses, use_link)
- nir_if_rewrite_condition(use_src->parent_if, new_src);
+ list_for_each_entry_safe(nir_src, use_src, &old_uses, use_link) {
+ if (use_src->is_if)
+ nir_if_rewrite_condition(use_src->parent_if, new_src);
+ else
+ nir_instr_rewrite_src(use_src->parent_instr, use_src, new_src);
+ }
if (nir_ssa_def_is_unused(old_def)) {
iter = nir_instr_free_and_dce(instr);
progress = true;
} else {
/* We didn't end up lowering after all. Put the uses back */
- if (old_def) {
+ if (old_def)
list_replace(&old_uses, &old_def->uses);
- list_replace(&old_if_uses, &old_def->if_uses);
- }
+
if (new_def == NIR_LOWER_INSTR_PROGRESS_REPLACE) {
/* Only instructions without a return value can be removed like this */
assert(!old_def);
/** set of nir_dests where this register is defined (written to) */
struct list_head defs;
-
- /** set of nir_ifs where this register is used as a condition */
- struct list_head if_uses;
} nir_register;
#define nir_foreach_register(reg, reg_list) \
/** set of nir_instrs where this register is used (read from) */
struct list_head uses;
- /** set of nir_ifs where this register is used as a condition */
- struct list_head if_uses;
-
/** generic SSA definition index. */
unsigned index;
};
bool is_ssa;
+ bool is_if;
} nir_src;
+static inline void
+nir_src_set_parent_instr(nir_src *src, nir_instr *parent_instr)
+{
+ src->is_if = false;
+ src->parent_instr = parent_instr;
+}
+
+static inline void
+nir_src_set_parent_if(nir_src *src, struct nir_if *parent_if)
+{
+ src->is_if = true;
+ src->parent_if = parent_if;
+}
+
static inline nir_src
nir_src_init(void)
{
#define NIR_SRC_INIT nir_src_init()
-#define nir_foreach_use(src, reg_or_ssa_def) \
+#define nir_foreach_use_including_if(src, reg_or_ssa_def) \
list_for_each_entry(nir_src, src, &(reg_or_ssa_def)->uses, use_link)
-#define nir_foreach_use_safe(src, reg_or_ssa_def) \
+#define nir_foreach_use_including_if_safe(src, reg_or_ssa_def) \
list_for_each_entry_safe(nir_src, src, &(reg_or_ssa_def)->uses, use_link)
+#define nir_foreach_use(src, reg_or_ssa_def) \
+ nir_foreach_use_including_if(src, reg_or_ssa_def) \
+ if (!src->is_if)
+
+#define nir_foreach_use_safe(src, reg_or_ssa_def) \
+ nir_foreach_use_including_if_safe(src, reg_or_ssa_def) \
+ if (!src->is_if)
+
#define nir_foreach_if_use(src, reg_or_ssa_def) \
- list_for_each_entry(nir_src, src, &(reg_or_ssa_def)->if_uses, use_link)
+ nir_foreach_use_including_if(src, reg_or_ssa_def) \
+ if (src->is_if)
#define nir_foreach_if_use_safe(src, reg_or_ssa_def) \
- list_for_each_entry_safe(nir_src, src, &(reg_or_ssa_def)->if_uses, use_link)
+ nir_foreach_use_including_if_safe(src, reg_or_ssa_def) \
+ if (src->is_if)
+
+static inline bool
+nir_ssa_def_used_by_if(const nir_ssa_def *def)
+{
+ nir_foreach_if_use(_, def)
+ return true;
+
+ return false;
+}
typedef struct {
union {
nir_instr_rewrite_src_ssa(ASSERTED nir_instr *instr,
nir_src *src, nir_ssa_def *new_ssa)
{
+ assert(!src->is_if);
assert(src->parent_instr == instr);
assert(src->is_ssa && src->ssa);
list_del(&src->use_link);
nir_if_rewrite_condition_ssa(ASSERTED nir_if *if_stmt,
nir_src *src, nir_ssa_def *new_ssa)
{
+ assert(src->is_if);
assert(src->parent_if == if_stmt);
assert(src->is_ssa && src->ssa);
list_del(&src->use_link);
src->ssa = new_ssa;
- list_addtail(&src->use_link, &new_ssa->if_uses);
+ list_addtail(&src->use_link, &new_ssa->uses);
}
void nir_if_rewrite_condition(nir_if *if_stmt, nir_src new_src);
static inline bool
nir_ssa_def_is_unused(nir_ssa_def *ssa)
{
- return list_is_empty(&ssa->uses) && list_is_empty(&ssa->if_uses);
+ return list_is_empty(&ssa->uses);
}
nreg->num_array_elems = reg->num_array_elems;
nreg->index = reg->index;
- /* reconstructing uses/defs/if_uses handled by nir_instr_insert() */
+ /* reconstructing uses/defs handled by nir_instr_insert() */
list_inithead(&nreg->uses);
list_inithead(&nreg->defs);
- list_inithead(&nreg->if_uses);
return nreg;
}
return;
nir_if *if_stmt = nir_cf_node_as_if(node);
+ nir_src_set_parent_if(&if_stmt->condition, if_stmt);
- if_stmt->condition.parent_if = if_stmt;
if (if_stmt->condition.is_ssa) {
list_addtail(&if_stmt->condition.use_link,
- &if_stmt->condition.ssa->if_uses);
+ &if_stmt->condition.ssa->uses);
} else {
list_addtail(&if_stmt->condition.use_link,
- &if_stmt->condition.reg.reg->if_uses);
+ &if_stmt->condition.reg.reg->uses);
}
}
nir_deref_instr_has_complex_use(nir_deref_instr *deref,
nir_deref_instr_has_complex_use_options opts)
{
- nir_foreach_use(use_src, &deref->dest.ssa) {
+ nir_foreach_use_including_if(use_src, &deref->dest.ssa) {
+ if (use_src->is_if)
+ return true;
+
nir_instr *use_instr = use_src->parent_instr;
switch (use_instr->type) {
}
}
- nir_foreach_if_use(use, &deref->dest.ssa)
- return true;
-
return false;
}
assert(cast->dest.is_ssa);
assert(cast->parent.is_ssa);
- nir_foreach_use_safe(use_src, &cast->dest.ssa) {
+ nir_foreach_use_including_if_safe(use_src, &cast->dest.ssa) {
+ assert(!use_src->is_if && "there cannot be if-uses");
+
/* If this isn't a trivial array cast, we can't propagate into
* ptr_as_array derefs.
*/
progress = true;
}
- /* If uses would be a bit crazy */
- assert(list_is_empty(&cast->dest.ssa.if_uses));
-
if (nir_deref_instr_remove_if_unused(cast))
progress = true;
ssa_def_is_local_to_block(nir_ssa_def *def, UNUSED void *state)
{
nir_block *block = def->parent_instr->block;
- nir_foreach_use(use_src, def) {
- if (use_src->parent_instr->block != block ||
+ nir_foreach_use_including_if(use_src, def) {
+ if (use_src->is_if ||
+ use_src->parent_instr->block != block ||
use_src->parent_instr->type == nir_instr_type_phi) {
return false;
}
}
- if (!list_is_empty(&def->if_uses))
- return false;
-
return true;
}
/* Also if the selects condition is only used by the select then
* remove that alu instructons cost from the cost total also.
*/
- if (!list_is_empty(&sel_alu->dest.dest.ssa.if_uses) ||
- !list_is_singular(&sel_alu->dest.dest.ssa.uses))
+ if (!list_is_singular(&sel_alu->dest.dest.ssa.uses) ||
+ nir_ssa_def_used_by_if(&sel_alu->dest.dest.ssa))
return 0;
else
return -1;
* one deref which could break our list walking since we walk the list
* backwards.
*/
- assert(list_is_empty(&deref->dest.ssa.if_uses));
- if (list_is_empty(&deref->dest.ssa.uses)) {
+ if (nir_ssa_def_is_unused(&deref->dest.ssa)) {
nir_instr_remove(&deref->instr);
return;
}
nir_foreach_register_safe(reg, &impl->registers) {
if (state.values[reg->index]) {
assert(list_is_empty(®->uses));
- assert(list_is_empty(®->if_uses));
assert(list_is_empty(®->defs));
exec_node_remove(®->node);
}
if (!(options & nir_lower_float_source_mods))
continue;
- if (!list_is_empty(&alu->dest.dest.ssa.if_uses))
- continue;
-
bool all_children_are_sat = true;
- nir_foreach_use(child_src, &alu->dest.dest.ssa) {
+ nir_foreach_use_including_if(child_src, &alu->dest.dest.ssa) {
+ if (child_src->is_if) {
+ all_children_are_sat = false;
+ break;
+ }
+
assert(child_src->is_ssa);
nir_instr *child = child_src->parent_instr;
if (child->type != nir_instr_type_alu) {
/* If we are going to do a reswizzle, then the vecN operation must be the
* only use of the source value. We also can't have any source modifiers.
*/
- nir_foreach_use(src, vec->src[start_idx].src.ssa) {
+ nir_foreach_use_including_if(src, vec->src[start_idx].src.ssa) {
+ if (src->is_if)
+ return 0;
+
if (src->parent_instr != &vec->instr)
return 0;
return 0;
}
- if (!list_is_empty(&vec->src[start_idx].src.ssa->if_uses))
- return 0;
-
if (vec->src[start_idx].src.ssa->parent_instr->type != nir_instr_type_alu)
return 0;
bool progress = false;
- nir_foreach_use_safe(src, &mov->dest.dest.ssa) {
- if (src->parent_instr->type == nir_instr_type_alu)
+ nir_foreach_use_including_if_safe(src, &mov->dest.dest.ssa) {
+ if (src->is_if)
+ progress |= copy_propagate_if(src, mov);
+ else if (src->parent_instr->type == nir_instr_type_alu)
progress |= copy_propagate_alu(impl, container_of(src, nir_alu_src, src), mov);
else
progress |= copy_propagate(src, mov);
}
- nir_foreach_if_use_safe(src, &mov->dest.dest.ssa)
- progress |= copy_propagate_if(src, mov);
-
if (progress && nir_ssa_def_is_unused(&mov->dest.dest.ssa))
nir_instr_remove(&mov->instr);
nir_block *before = nir_cf_node_as_block(nir_cf_node_prev(node));
nir_block *after = nir_cf_node_as_block(nir_cf_node_next(node));
- nir_foreach_use(use, def) {
+ nir_foreach_use_including_if(use, def) {
+ nir_block *block;
+
+ if (use->is_if)
+ block = nir_cf_node_as_block(nir_cf_node_prev(&use->parent_if->cf_node));
+ else
+ block = use->parent_instr->block;
+
/* Because NIR is structured, we can easily determine whether or not a
* value escapes a CF node by looking at the block indices of its uses
* to see if they lie outside the bounds of the CF node.
* corresponding predecessor is inside the loop or not because the value
* can go through the phi into the outside world and escape the loop.
*/
- if (use->parent_instr->block->index <= before->index ||
- use->parent_instr->block->index >= after->index)
- return false;
- }
-
- /* Same check for if-condition uses */
- nir_foreach_if_use(use, def) {
- nir_block *use_block =
- nir_cf_node_as_block(nir_cf_node_prev(&use->parent_if->cf_node));
-
- if (use_block->index <= before->index ||
- use_block->index >= after->index)
+ if (block->index <= before->index || block->index >= after->index)
return false;
}
if (!is_prev_result_undef && !is_prev_result_const) {
/* check if the only user is a trivial bcsel */
- if (!list_is_empty(&alu->dest.dest.ssa.if_uses) ||
- !list_is_singular(&alu->dest.dest.ssa.uses))
+ if (!list_is_singular(&alu->dest.dest.ssa.uses))
continue;
nir_src *use = list_first_entry(&alu->dest.dest.ssa.uses, nir_src, use_link);
- if (!is_trivial_bcsel(use->parent_instr, true))
+ if (use->is_if || !is_trivial_bcsel(use->parent_instr, true))
continue;
}
*/
static bool
propagate_condition_eval(nir_builder *b, nir_if *nif, nir_src *use_src,
- nir_src *alu_use, nir_alu_instr *alu,
- bool is_if_condition)
+ nir_src *alu_use, nir_alu_instr *alu)
{
bool bool_value;
- b->cursor = nir_before_src(alu_use, is_if_condition);
+ b->cursor = nir_before_src(alu_use, alu_use->is_if);
if (!evaluate_if_condition(nif, b->cursor, &bool_value))
return false;
/* Rewrite use to use new alu instruction */
nir_src new_src = nir_src_for_ssa(nalu);
- if (is_if_condition)
+ if (alu_use->is_if)
nir_if_rewrite_condition(alu_use->parent_if, new_src);
else
nir_instr_rewrite_src(alu_use->parent_instr, alu_use, new_src);
}
static bool
-evaluate_condition_use(nir_builder *b, nir_if *nif, nir_src *use_src,
- bool is_if_condition)
+evaluate_condition_use(nir_builder *b, nir_if *nif, nir_src *use_src)
{
bool progress = false;
- b->cursor = nir_before_src(use_src, is_if_condition);
+ b->cursor = nir_before_src(use_src, use_src->is_if);
bool bool_value;
if (evaluate_if_condition(nif, b->cursor, &bool_value)) {
/* Rewrite use to use const */
nir_src imm_src = nir_src_for_ssa(nir_imm_bool(b, bool_value));
- if (is_if_condition)
+ if (use_src->is_if)
nir_if_rewrite_condition(use_src->parent_if, imm_src);
else
nir_instr_rewrite_src(use_src->parent_instr, use_src, imm_src);
progress = true;
}
- if (!is_if_condition && can_propagate_through_alu(use_src)) {
+ if (!use_src->is_if && can_propagate_through_alu(use_src)) {
nir_alu_instr *alu = nir_instr_as_alu(use_src->parent_instr);
- nir_foreach_use_safe(alu_use, &alu->dest.dest.ssa) {
- progress |= propagate_condition_eval(b, nif, use_src, alu_use, alu,
- false);
- }
-
- nir_foreach_if_use_safe(alu_use, &alu->dest.dest.ssa) {
- progress |= propagate_condition_eval(b, nif, use_src, alu_use, alu,
- true);
- }
+ nir_foreach_use_including_if_safe(alu_use, &alu->dest.dest.ssa)
+ progress |= propagate_condition_eval(b, nif, use_src, alu_use, alu);
}
return progress;
/* Evaluate any uses of the if condition inside the if branches */
assert(nif->condition.is_ssa);
- nir_foreach_use_safe(use_src, nif->condition.ssa) {
- progress |= evaluate_condition_use(b, nif, use_src, false);
- }
-
- nir_foreach_if_use_safe(use_src, nif->condition.ssa) {
- if (use_src->parent_if != nif)
- progress |= evaluate_condition_use(b, nif, use_src, true);
+ nir_foreach_use_including_if_safe(use_src, nif->condition.ssa) {
+ if (!(use_src->is_if && use_src->parent_if == nif))
+ progress |= evaluate_condition_use(b, nif, use_src);
}
return progress;
* uses is reasonable. If we ever want to use this from an if statement,
* we can change it then.
*/
- if (!list_is_empty(&shuffle->dest.ssa.if_uses) ||
- !list_is_singular(&shuffle->dest.ssa.uses))
+ if (!list_is_singular(&shuffle->dest.ssa.uses))
+ return false;
+
+ if (nir_ssa_def_used_by_if(&shuffle->dest.ssa))
return false;
assert(shuffle->src[0].is_ssa);
if (mov->dest.saturate)
return false;
- /* It cannot have any if-uses */
- if (!list_is_empty(&mov->dest.dest.ssa.if_uses))
- return false;
-
/* The only uses of this definition must be phis in the successor */
- nir_foreach_use(use, &mov->dest.dest.ssa) {
- if (use->parent_instr->type != nir_instr_type_phi ||
+ nir_foreach_use_including_if(use, &mov->dest.dest.ssa) {
+ if (use->is_if ||
+ use->parent_instr->type != nir_instr_type_phi ||
use->parent_instr->block != block->successors[0])
return false;
}
/* Are the only uses of the phi conversion instructions, and
* are they all the same conversion?
*/
- nir_foreach_use (use, &phi->dest.ssa) {
+ nir_foreach_use_including_if (use, &phi->dest.ssa) {
+ /* an if use means the phi is used directly in a conditional, ie.
+ * without a conversion
+ */
+ if (use->is_if)
+ return false;
+
op = narrowing_conversion_op(use->parent_instr, op);
/* Not a (compatible) narrowing conversion: */
return false;
}
- /* an if_uses means the phi is used directly in a conditional, ie.
- * without a conversion
- */
- if (!list_is_empty(&phi->dest.ssa.if_uses))
- return false;
-
/* If the phi has no uses, then nothing to do: */
if (op == INVALID_OP)
return false;
nir_intrinsic_instr *intrin = nir_instr_as_intrinsic(instr);
switch (intrin->intrinsic) {
case nir_intrinsic_rq_proceed:
- if (list_length(&intrin->dest.ssa.uses) > 0 ||
- list_length(&intrin->dest.ssa.if_uses) > 0)
+ if (list_length(&intrin->dest.ssa.uses) > 0)
mark_query_read(queries, intrin);
break;
case nir_intrinsic_rq_load:
if (entry)
return false;
- if (intrin->intrinsic == nir_intrinsic_rq_load) {
+ if (intrin->intrinsic == nir_intrinsic_rq_load)
assert(list_is_empty(&intrin->dest.ssa.uses));
- assert(list_is_empty(&intrin->dest.ssa.if_uses));
- }
nir_instr_remove(instr);
* instruction must be duplicated only once in each block because CSE
* cannot be run after this pass.
*/
- nir_foreach_use_safe(use, &alu->dest.dest.ssa) {
- nir_instr *const use_instr = use->parent_instr;
+ nir_foreach_use_including_if_safe(use, &alu->dest.dest.ssa) {
+ if (use->is_if) {
+ nir_if *const if_stmt = use->parent_if;
- /* If the use is in the same block as the def, don't
- * rematerialize.
- */
- if (use_instr->block == alu->instr.block)
- continue;
+ nir_block *const prev_block =
+ nir_cf_node_as_block(nir_cf_node_prev(&if_stmt->cf_node));
- nir_alu_instr *clone = nir_alu_instr_clone(shader, alu);
+ /* If the compare is from the previous block, don't
+ * rematerialize.
+ */
+ if (prev_block == alu->instr.block)
+ continue;
- nir_instr_insert_before(use_instr, &clone->instr);
+ nir_alu_instr *clone = nir_alu_instr_clone(shader, alu);
- nir_alu_instr *const use_alu = nir_instr_as_alu(use_instr);
- for (unsigned i = 0; i < nir_op_infos[use_alu->op].num_inputs; i++) {
- if (use_alu->src[i].src.ssa == &alu->dest.dest.ssa) {
- nir_instr_rewrite_src(&use_alu->instr,
- &use_alu->src[i].src,
+ nir_instr_insert_after_block(prev_block, &clone->instr);
+
+ nir_if_rewrite_condition(if_stmt,
nir_src_for_ssa(&clone->dest.dest.ssa));
- progress = true;
+ progress = true;
+ } else {
+ nir_instr *const use_instr = use->parent_instr;
+
+ /* If the use is in the same block as the def, don't
+ * rematerialize.
+ */
+ if (use_instr->block == alu->instr.block)
+ continue;
+
+ nir_alu_instr *clone = nir_alu_instr_clone(shader, alu);
+
+ nir_instr_insert_before(use_instr, &clone->instr);
+
+ nir_alu_instr *const use_alu = nir_instr_as_alu(use_instr);
+ for (unsigned i = 0; i < nir_op_infos[use_alu->op].num_inputs; i++) {
+ if (use_alu->src[i].src.ssa == &alu->dest.dest.ssa) {
+ nir_instr_rewrite_src(&use_alu->instr,
+ &use_alu->src[i].src,
+ nir_src_for_ssa(&clone->dest.dest.ssa));
+ progress = true;
+ }
}
}
}
-
- nir_foreach_if_use_safe(use, &alu->dest.dest.ssa) {
- nir_if *const if_stmt = use->parent_if;
-
- nir_block *const prev_block =
- nir_cf_node_as_block(nir_cf_node_prev(&if_stmt->cf_node));
-
- /* If the compare is from the previous block, don't
- * rematerialize.
- */
- if (prev_block == alu->instr.block)
- continue;
-
- nir_alu_instr *clone = nir_alu_instr_clone(shader, alu);
-
- nir_instr_insert_after_block(prev_block, &clone->instr);
-
- nir_if_rewrite_condition(if_stmt,
- nir_src_for_ssa(&clone->dest.dest.ssa));
- progress = true;
- }
}
}
{
nir_block *lca = NULL;
- nir_foreach_use(use, def) {
- nir_instr *instr = use->parent_instr;
- nir_block *use_block = instr->block;
-
- /*
- * Kind of an ugly special-case, but phi instructions
- * need to appear first in the block, so by definition
- * we can't move an instruction into a block where it is
- * consumed by a phi instruction. We could conceivably
- * move it into a dominator block.
- */
- if (instr->type == nir_instr_type_phi) {
- nir_phi_instr *phi = nir_instr_as_phi(instr);
- nir_block *phi_lca = NULL;
- nir_foreach_phi_src(src, phi) {
- if (&src->src == use)
- phi_lca = nir_dominance_lca(phi_lca, src->pred);
+ nir_foreach_use_including_if(use, def) {
+ nir_block *use_block;
+
+ if (use->is_if) {
+ use_block =
+ nir_cf_node_as_block(nir_cf_node_prev(&use->parent_if->cf_node));
+ } else {
+ nir_instr *instr = use->parent_instr;
+ use_block = instr->block;
+
+ /*
+ * Kind of an ugly special-case, but phi instructions
+ * need to appear first in the block, so by definition
+ * we can't move an instruction into a block where it is
+ * consumed by a phi instruction. We could conceivably
+ * move it into a dominator block.
+ */
+ if (instr->type == nir_instr_type_phi) {
+ nir_phi_instr *phi = nir_instr_as_phi(instr);
+ nir_block *phi_lca = NULL;
+ nir_foreach_phi_src(src, phi) {
+ if (&src->src == use)
+ phi_lca = nir_dominance_lca(phi_lca, src->pred);
+ }
+ use_block = phi_lca;
}
- use_block = phi_lca;
}
lca = nir_dominance_lca(lca, use_block);
}
- nir_foreach_if_use(use, def) {
- nir_block *use_block =
- nir_cf_node_as_block(nir_cf_node_prev(&use->parent_if->cf_node));
-
- lca = nir_dominance_lca(lca, use_block);
- }
-
/* return in case, we didn't find a reachable user */
if (!lca)
return NULL;
nir_ssa_def old_result = intrin->dest.ssa;
list_replace(&intrin->dest.ssa.uses, &old_result.uses);
- list_replace(&intrin->dest.ssa.if_uses, &old_result.if_uses);
nir_ssa_dest_init(&intrin->instr, &intrin->dest, 1, intrin->dest.ssa.bit_size, NULL);
nir_ssa_def *result = optimize_atomic(b, intrin, return_prev);
struct repair_ssa_state *state = void_state;
bool is_valid = true;
- nir_foreach_use(src, def) {
- if (nir_block_is_unreachable(get_src_block(src)) ||
- !nir_block_dominates(def->parent_instr->block, get_src_block(src))) {
- is_valid = false;
- break;
- }
- }
-
- nir_foreach_if_use(src, def) {
- nir_block *block_before_if =
- nir_cf_node_as_block(nir_cf_node_prev(&src->parent_if->cf_node));
- if (nir_block_is_unreachable(block_before_if) ||
- !nir_block_dominates(def->parent_instr->block, block_before_if)) {
- is_valid = false;
- break;
+ nir_foreach_use_including_if(src, def) {
+ if (src->is_if) {
+ nir_block *block_before_if =
+ nir_cf_node_as_block(nir_cf_node_prev(&src->parent_if->cf_node));
+ if (nir_block_is_unreachable(block_before_if) ||
+ !nir_block_dominates(def->parent_instr->block, block_before_if)) {
+ is_valid = false;
+ break;
+ }
+ } else {
+ if (nir_block_is_unreachable(get_src_block(src)) ||
+ !nir_block_dominates(def->parent_instr->block, get_src_block(src))) {
+ is_valid = false;
+ break;
+ }
}
}
nir_phi_builder_value_set_block_def(val, def->parent_instr->block, def);
- nir_foreach_use_safe(src, def) {
+ nir_foreach_use_including_if_safe(src, def) {
+ if (src->is_if) {
+ nir_block *block_before_if =
+ nir_cf_node_as_block(nir_cf_node_prev(&src->parent_if->cf_node));
+ if (block_before_if == def->parent_instr->block) {
+ assert(nir_phi_builder_value_get_block_def(val, block_before_if) == def);
+ continue;
+ }
+
+ nir_ssa_def *block_def =
+ nir_phi_builder_value_get_block_def(val, block_before_if);
+ if (block_def == def)
+ continue;
+
+ nir_if_rewrite_condition(src->parent_if, nir_src_for_ssa(block_def));
+ continue;
+ }
+
nir_block *src_block = get_src_block(src);
if (src_block == def->parent_instr->block) {
assert(nir_phi_builder_value_get_block_def(val, src_block) == def);
nir_instr_rewrite_src(src->parent_instr, src, nir_src_for_ssa(block_def));
}
- nir_foreach_if_use_safe(src, def) {
- nir_block *block_before_if =
- nir_cf_node_as_block(nir_cf_node_prev(&src->parent_if->cf_node));
- if (block_before_if == def->parent_instr->block) {
- assert(nir_phi_builder_value_get_block_def(val, block_before_if) == def);
- continue;
- }
-
- nir_ssa_def *block_def =
- nir_phi_builder_value_get_block_def(val, block_before_if);
- if (block_def == def)
- continue;
-
- nir_if_rewrite_condition(src->parent_if, nir_src_for_ssa(block_def));
- }
-
return true;
}
static inline bool
is_used_once(const nir_alu_instr *instr)
{
- bool zero_if_use = list_is_empty(&instr->dest.dest.ssa.if_uses);
- bool zero_use = list_is_empty(&instr->dest.dest.ssa.uses);
-
- if (zero_if_use && zero_use)
- return false;
-
- if (!zero_if_use && list_is_singular(&instr->dest.dest.ssa.uses))
- return false;
-
- if (!zero_use && list_is_singular(&instr->dest.dest.ssa.if_uses))
- return false;
-
- if (!list_is_singular(&instr->dest.dest.ssa.if_uses) &&
- !list_is_singular(&instr->dest.dest.ssa.uses))
- return false;
-
- return true;
+ return list_is_singular(&instr->dest.dest.ssa.uses);
}
static inline bool
is_used_by_if(const nir_alu_instr *instr)
{
- return !list_is_empty(&instr->dest.dest.ssa.if_uses);
+ return nir_ssa_def_used_by_if(&instr->dest.dest.ssa);
}
static inline bool
is_not_used_by_if(const nir_alu_instr *instr)
{
- return list_is_empty(&instr->dest.dest.ssa.if_uses);
+ return !is_used_by_if(instr);
}
static inline bool
list_inithead(®->uses);
list_inithead(®->defs);
- list_inithead(®->if_uses);
return reg;
}
return true;
}
- nir_foreach_use(use, def) {
+ nir_foreach_use_including_if(use, def) {
+ if (use->is_if) {
+ if (!is_if_use_inside_loop(use, state->loop))
+ all_uses_inside_loop = false;
+
+ continue;
+ }
+
if (use->parent_instr->type == nir_instr_type_phi &&
use->parent_instr->block == state->block_after_loop) {
continue;
}
}
- nir_foreach_if_use(use, def) {
- if (!is_if_use_inside_loop(use, state->loop)) {
- all_uses_inside_loop = false;
- }
- }
-
/* There where no sources that had defs outside the loop */
if (all_uses_inside_loop)
return true;
/* Run through all uses and rewrite those outside the loop to point to
* the phi instead of pointing to the ssa-def.
*/
- nir_foreach_use_safe(use, def) {
+ nir_foreach_use_including_if_safe(use, def) {
+ if (use->is_if) {
+ if (!is_if_use_inside_loop(use, state->loop))
+ nir_if_rewrite_condition(use->parent_if, nir_src_for_ssa(dest));
+
+ continue;
+ }
+
if (use->parent_instr->type == nir_instr_type_phi &&
state->block_after_loop == use->parent_instr->block) {
continue;
}
}
- nir_foreach_if_use_safe(use, def) {
- if (!is_if_use_inside_loop(use, state->loop)) {
- nir_if_rewrite_condition(use->parent_if, nir_src_for_ssa(dest));
- }
- }
-
state->progress = true;
return true;
}
* equivalent to the uses and defs in nir_register, but built up by the
* validator. At the end, we verify that the sets have the same entries.
*/
- struct set *uses, *if_uses, *defs;
+ struct set *uses, *defs;
nir_function_impl *where_defined; /* NULL for global registers */
} reg_validate_state;
_mesa_set_add(reg_state->uses, src);
} else {
validate_assert(state, state->if_stmt);
- _mesa_set_add(reg_state->if_uses, src);
+ validate_assert(state, src->is_if);
+ _mesa_set_add(reg_state->uses, src);
}
validate_assert(state, reg_state->where_defined == state->impl &&
* our use is seen in a use list.
*/
struct set_entry *entry;
- if (state->instr) {
+ if (!src->is_if && state->instr) {
entry = _mesa_set_search(state->ssa_srcs, src);
} else {
entry = _mesa_set_search(state->ssa_srcs, SET_PTR_BIT(src, 0));
validate_num_components(state, def->num_components);
list_validate(&def->uses);
- nir_foreach_use(src, def) {
+ nir_foreach_use_including_if(src, def) {
validate_assert(state, src->is_ssa);
validate_assert(state, src->ssa == def);
bool already_seen = false;
- _mesa_set_search_and_add(state->ssa_srcs, src, &already_seen);
- /* A nir_src should only appear once and only in one SSA def use list */
- validate_assert(state, !already_seen);
- }
- list_validate(&def->if_uses);
- nir_foreach_if_use(src, def) {
- validate_assert(state, src->is_ssa);
- validate_assert(state, src->ssa == def);
- bool already_seen = false;
- _mesa_set_search_and_add(state->ssa_srcs, SET_PTR_BIT(src, 0),
- &already_seen);
+ nir_src *ssa_src_ptr = src;
+ if (src->is_if)
+ ssa_src_ptr = SET_PTR_BIT(ssa_src_ptr, 0);
+
+ _mesa_set_search_and_add(state->ssa_srcs, ssa_src_ptr, &already_seen);
/* A nir_src should only appear once and only in one SSA def use list */
validate_assert(state, !already_seen);
}
* conditions expect well-formed Booleans. If you want to compare with
* NULL, an explicit comparison operation should be used.
*/
- validate_assert(state, list_is_empty(&instr->dest.ssa.if_uses));
+ validate_assert(state, !nir_ssa_def_used_by_if(&instr->dest.ssa));
/* Certain modes cannot be used as sources for phi instructions because
* way too many passes assume that they can always chase deref chains.
nir_cf_node *next_node = nir_cf_node_next(&if_stmt->cf_node);
validate_assert(state, next_node->type == nir_cf_node_block);
+ validate_assert(state, if_stmt->condition.is_if);
validate_src(&if_stmt->condition, state, 0, 1);
validate_assert(state, !exec_list_is_empty(&if_stmt->then_list));
list_validate(®->uses);
list_validate(®->defs);
- list_validate(®->if_uses);
reg_validate_state *reg_state = ralloc(state->regs, reg_validate_state);
reg_state->uses = _mesa_pointer_set_create(reg_state);
- reg_state->if_uses = _mesa_pointer_set_create(reg_state);
reg_state->defs = _mesa_pointer_set_create(reg_state);
reg_state->where_defined = state->impl;
assume(entry);
reg_validate_state *reg_state = (reg_validate_state *) entry->data;
- nir_foreach_use(src, reg) {
+ nir_foreach_use_including_if(src, reg) {
struct set_entry *entry = _mesa_set_search(reg_state->uses, src);
validate_assert(state, entry);
_mesa_set_remove(reg_state->uses, entry);
}
validate_assert(state, reg_state->uses->entries == 0);
- nir_foreach_if_use(src, reg) {
- struct set_entry *entry = _mesa_set_search(reg_state->if_uses, src);
- validate_assert(state, entry);
- _mesa_set_remove(reg_state->if_uses, entry);
- }
- validate_assert(state, reg_state->if_uses->entries == 0);
-
nir_foreach_def(src, reg) {
struct set_entry *entry = _mesa_set_search(reg_state->defs, src);
validate_assert(state, entry);
static bool
is_used_once(const nir_ssa_def *def)
{
- return list_is_singular(&def->uses) &&
- list_is_empty(&def->if_uses);
+ return list_is_singular(&def->uses);
}
nir_alu_instr *
static bool
all_uses_float(nir_ssa_def *def, bool allow_src2)
{
- nir_foreach_if_use (use, def) {
- return false;
- }
+ nir_foreach_use_including_if (use, def) {
+ if (use->is_if)
+ return false;
- nir_foreach_use (use, def) {
nir_instr *use_instr = use->parent_instr;
if (use_instr->type != nir_instr_type_alu)
return false;
static bool
all_uses_bit(nir_ssa_def *def)
{
- nir_foreach_if_use (use, def) {
- return false;
- }
+ nir_foreach_use_including_if (use, def) {
+ if (use->is_if)
+ return false;
- nir_foreach_use (use, def) {
nir_instr *use_instr = use->parent_instr;
if (use_instr->type != nir_instr_type_alu)
return false;
*/
static bool
ntt_try_store_in_tgsi_output(struct ntt_compile *c, struct ureg_dst *dst,
- struct list_head *uses, struct list_head *if_uses)
+ struct list_head *uses)
{
*dst = ureg_dst_undef();
return false;
}
- if (!list_is_empty(if_uses) || !list_is_singular(uses))
+ if (!list_is_singular(uses))
return false;
nir_src *src = list_first_entry(uses, nir_src, use_link);
+ if (src->is_if)
+ return false;
if (src->parent_instr->type != nir_instr_type_intrinsic)
return false;
if (nir_reg->num_array_elems == 0) {
struct ureg_dst decl;
uint32_t write_mask = BITFIELD_MASK(nir_reg->num_components);
- if (!ntt_try_store_in_tgsi_output(c, &decl, &nir_reg->uses, &nir_reg->if_uses)) {
+ if (!ntt_try_store_in_tgsi_output(c, &decl, &nir_reg->uses)) {
if (nir_reg->bit_size == 64) {
if (nir_reg->num_components > 2) {
fprintf(stderr, "NIR-to-TGSI: error: %d-component NIR r%d\n",
writemask = ntt_64bit_write_mask(writemask);
struct ureg_dst dst;
- if (!ntt_try_store_in_tgsi_output(c, &dst, &ssa->uses, &ssa->if_uses))
+ if (!ntt_try_store_in_tgsi_output(c, &dst, &ssa->uses))
dst = ntt_temp(c);
c->ssa_temp[ssa->index] = ntt_swizzle_for_write_mask(ureg_src(dst), writemask);
nir_ssa_def *ssa = alu->src[i].src.ssa;
/* check that vecN instruction is only user of this */
- bool need_mov = list_length(&ssa->if_uses) != 0;
- nir_foreach_use(use_src, ssa) {
- if (use_src->parent_instr != &alu->instr)
+ bool need_mov = false;
+ nir_foreach_use_including_if(use_src, ssa) {
+ if (use_src->is_if || use_src->parent_instr != &alu->instr)
need_mov = true;
}
if (!dest || !dest->is_ssa)
return dest;
- bool can_bypass_src = !list_length(&dest->ssa.if_uses);
+ bool can_bypass_src = nir_ssa_def_used_by_if(&dest->ssa);
nir_instr *p_instr = dest->ssa.parent_instr;
/* if used by a vecN, the "real" destination becomes the vecN destination
case nir_op_vec2:
case nir_op_vec3:
case nir_op_vec4:
- assert(list_length(&dest->ssa.if_uses) == 0);
+ assert(!nir_ssa_def_used_by_if(&dest->ssa));
nir_foreach_use(use_src, &dest->ssa)
assert(use_src->parent_instr == instr);
default:
continue;
}
- if (list_length(&dest->ssa.if_uses) || list_length(&dest->ssa.uses) > 1)
+ if (nir_ssa_def_used_by_if(&dest->ssa) ||
+ list_length(&dest->ssa.uses) > 1)
continue;
update_swiz_mask(alu, NULL, swiz, mask);
static bool
ntq_src_is_only_ssa_def_user(nir_src *src)
{
- if (!src->is_ssa)
- return false;
-
- if (!list_is_empty(&src->ssa->if_uses))
- return false;
-
- return (src->ssa->uses.next == &src->use_link &&
- src->ssa->uses.next->next == &src->ssa->uses);
+ return src->is_ssa && list_is_singular(&src->ssa->uses);
}
/**
static bool
is_used_in_not_interp_frag_coord(nir_ssa_def *def)
{
- nir_foreach_use(src, def) {
+ nir_foreach_use_including_if(src, def) {
+ if (src->is_if)
+ return true;
+
if (src->parent_instr->type != nir_instr_type_intrinsic)
return true;
return true;
}
- nir_foreach_if_use(src, def)
- return true;
-
return false;
}
static inline bool
are_all_uses_fadd(nir_ssa_def *def)
{
- if (!list_is_empty(&def->if_uses))
- return false;
+ nir_foreach_use_including_if(use_src, def) {
+ if (use_src->is_if)
+ return false;
- nir_foreach_use(use_src, def) {
nir_instr *use_instr = use_src->parent_instr;
-
if (use_instr->type != nir_instr_type_alu)
return false;
nir_load_const_instr *load_const =
nir_instr_as_load_const (srcs[i].src.ssa->parent_instr);
- if (list_is_singular(&load_const->def.uses) &&
- list_is_empty(&load_const->def.if_uses)) {
+ if (list_is_singular(&load_const->def.uses))
return true;
- }
}
}
continue;
nir_intrinsic_instr *intr = nir_instr_as_intrinsic(instr);
if (intr->intrinsic != nir_intrinsic_load_invocation_id ||
- list_length(&intr->dest.ssa.uses) +
- list_length(&intr->dest.ssa.if_uses) <= 1)
+ list_length(&intr->dest.ssa.uses) <= 1)
continue;
- nir_foreach_use_safe(src, &intr->dest.ssa) {
- b.cursor = nir_before_src(src, false);
- nir_instr_rewrite_src_ssa(src->parent_instr, src, nir_load_invocation_id(&b));
- }
- nir_foreach_if_use_safe(src, &intr->dest.ssa) {
- b.cursor = nir_before_src(src, true);
- nir_if_rewrite_condition_ssa(src->parent_if, src, nir_load_invocation_id(&b));
+ nir_foreach_use_including_if_safe(src, &intr->dest.ssa) {
+ b.cursor = nir_before_src(src, src->is_if);
+ nir_ssa_def *id = nir_load_invocation_id(&b);
+
+ if (src->is_if)
+ nir_if_rewrite_condition_ssa(src->parent_if, src, id);
+ else
+ nir_instr_rewrite_src_ssa(src->parent_instr, src, id);
}
nir_instr_remove(instr);
}
- nir_foreach_register_safe
- nir_foreach_use
- nir_foreach_use_safe
+ - nir_foreach_use_including_if
+ - nir_foreach_use_including_if_safe
- nir_foreach_if_use
- nir_foreach_if_use_safe
- nir_foreach_def
if (!intr->dest.is_ssa)
continue;
- if (!list_is_empty(&intr->dest.ssa.if_uses))
- return false;
-
bool valid = true;
- nir_foreach_use(src, &intr->dest.ssa)
- valid &= nir_src_is_f2fmp(src);
+ nir_foreach_use_including_if(src, &intr->dest.ssa)
+ valid &= !src->is_if && nir_src_is_f2fmp(src);
if (!valid)
continue;
return false;
/* Check the uses. We want a single use, with the op `op` */
- if (!list_is_empty(&dest->ssa.if_uses))
- return false;
-
if (!list_is_singular(&dest->ssa.uses))
return false;
nir_src *use = list_first_entry(&dest->ssa.uses, nir_src, use_link);
+ if (use->is_if)
+ return false;
+
nir_instr *parent = use->parent_instr;
/* Check if the op is `op` */
projects / platform / upstream / mesa.git / commitdiff