#include "nir/nir_vla.h"
#include "util/u_debug.h"
-static struct vtn_block *
-vtn_block(struct vtn_builder *b, uint32_t value_id)
-{
- return vtn_value(b, value_id, vtn_value_type_block)->block;
-}
-
static unsigned
glsl_type_count_function_params(const struct glsl_type *type)
{
}
}
-static bool
+bool
vtn_cfg_handle_prepass_instruction(struct vtn_builder *b, SpvOp opcode,
const uint32_t *w, unsigned count)
{
vtn_assert(b->func == NULL);
b->func = rzalloc(b, struct vtn_function);
- b->func->node.type = vtn_cf_node_type_function;
- b->func->node.parent = NULL;
list_inithead(&b->func->body);
b->func->linkage = SpvLinkageTypeMax;
b->func->control = w[3];
+ list_inithead(&b->func->constructs);
UNUSED const struct glsl_type *result_type = vtn_get_type(b, w[1])->type;
struct vtn_value *val = vtn_push_value(b, w[2], vtn_value_type_function);
case SpvOpLabel: {
vtn_assert(b->block == NULL);
b->block = rzalloc(b, struct vtn_block);
- b->block->node.type = vtn_cf_node_type_block;
b->block->label = w;
vtn_push_value(b, w[1], vtn_value_type_block)->block = b->block;
+ b->func->block_count++;
+
if (b->func->start_block == NULL) {
/* This is the first block encountered for this function. In this
* case, we set the start block and add it to the list of
* implemented functions that we'll walk later.
*/
b->func->start_block = b->block;
- list_addtail(&b->func->node.link, &b->functions);
+ list_addtail(&b->func->link, &b->functions);
}
break;
}
return true;
}
-/* This function performs a depth-first search of the cases and puts them
- * in fall-through order.
- */
-static void
-vtn_order_case(struct vtn_switch *swtch, struct vtn_case *cse)
-{
- if (cse->visited)
- return;
-
- cse->visited = true;
-
- list_del(&cse->node.link);
-
- if (cse->fallthrough) {
- vtn_order_case(swtch, cse->fallthrough);
-
- /* If we have a fall-through, place this case right before the case it
- * falls through to. This ensures that fallthroughs come one after
- * the other. These two can never get separated because that would
- * imply something else falling through to the same case. Also, this
- * can't break ordering because the DFS ensures that this case is
- * visited before anything that falls through to it.
- */
- list_addtail(&cse->node.link, &cse->fallthrough->node.link);
- } else {
- list_add(&cse->node.link, &swtch->cases);
- }
-}
-
-static void
-vtn_switch_order_cases(struct vtn_switch *swtch)
-{
- struct list_head cases;
- list_replace(&swtch->cases, &cases);
- list_inithead(&swtch->cases);
- while (!list_is_empty(&cases)) {
- struct vtn_case *cse =
- list_first_entry(&cases, struct vtn_case, node.link);
- vtn_order_case(swtch, cse);
- }
-}
-
-static void
-vtn_block_set_merge_cf_node(struct vtn_builder *b, struct vtn_block *block,
- struct vtn_cf_node *cf_node)
-{
- vtn_fail_if(block->merge_cf_node != NULL,
- "The merge block declared by a header block cannot be a "
- "merge block declared by any other header block.");
-
- block->merge_cf_node = cf_node;
-}
-
-#define VTN_DECL_CF_NODE_FIND(_type) \
-static inline struct vtn_##_type * \
-vtn_cf_node_find_##_type(struct vtn_cf_node *node) \
-{ \
- while (node && node->type != vtn_cf_node_type_##_type) \
- node = node->parent; \
- return (struct vtn_##_type *)node; \
-}
-
-UNUSED VTN_DECL_CF_NODE_FIND(if)
-VTN_DECL_CF_NODE_FIND(loop)
-VTN_DECL_CF_NODE_FIND(case)
-VTN_DECL_CF_NODE_FIND(switch)
-VTN_DECL_CF_NODE_FIND(function)
-
-static enum vtn_branch_type
-vtn_handle_branch(struct vtn_builder *b,
- struct vtn_cf_node *cf_parent,
- struct vtn_block *target_block)
-{
- struct vtn_loop *loop = vtn_cf_node_find_loop(cf_parent);
-
- /* Detect a loop back-edge first. That way none of the code below
- * accidentally operates on a loop back-edge.
- */
- if (loop && target_block == loop->header_block)
- return vtn_branch_type_loop_back_edge;
-
- /* Try to detect fall-through */
- if (target_block->switch_case) {
- /* When it comes to handling switch cases, we can break calls to
- * vtn_handle_branch into two cases: calls from within a case construct
- * and calls for the jump to each case construct. In the second case,
- * cf_parent is the vtn_switch itself and vtn_cf_node_find_case() will
- * return the outer switch case in which this switch is contained. It's
- * fine if the target block is a switch case from an outer switch as
- * long as it is also the switch break for this switch.
- */
- struct vtn_case *switch_case = vtn_cf_node_find_case(cf_parent);
-
- /* This doesn't get called for the OpSwitch */
- vtn_fail_if(switch_case == NULL,
- "A switch case can only be entered through an OpSwitch or "
- "falling through from another switch case.");
-
- /* Because block->switch_case is only set on the entry block for a given
- * switch case, we only ever get here if we're jumping to the start of a
- * switch case. It's possible, however, that a switch case could jump
- * to itself via a back-edge. That *should* get caught by the loop
- * handling case above but if we have a back edge without a loop merge,
- * we could en up here.
- */
- vtn_fail_if(target_block->switch_case == switch_case,
- "A switch cannot fall-through to itself. Likely, there is "
- "a back-edge which is not to a loop header.");
-
- vtn_fail_if(target_block->switch_case->node.parent !=
- switch_case->node.parent,
- "A switch case fall-through must come from the same "
- "OpSwitch construct");
-
- vtn_fail_if(switch_case->fallthrough != NULL &&
- switch_case->fallthrough != target_block->switch_case,
- "Each case construct can have at most one branch to "
- "another case construct");
-
- switch_case->fallthrough = target_block->switch_case;
-
- /* We don't immediately return vtn_branch_type_switch_fallthrough
- * because it may also be a loop or switch break for an inner loop or
- * switch and that takes precedence.
- */
- }
-
- if (loop && target_block == loop->cont_block)
- return vtn_branch_type_loop_continue;
-
- /* We walk blocks as a breadth-first search on the control-flow construct
- * tree where, when we find a construct, we add the vtn_cf_node for that
- * construct and continue iterating at the merge target block (if any).
- * Therefore, we want merges whose with parent == cf_parent to be treated
- * as regular branches. We only want to consider merges if they break out
- * of the current CF construct.
- */
- if (target_block->merge_cf_node != NULL &&
- target_block->merge_cf_node->parent != cf_parent) {
- switch (target_block->merge_cf_node->type) {
- case vtn_cf_node_type_if:
- for (struct vtn_cf_node *node = cf_parent;
- node != target_block->merge_cf_node; node = node->parent) {
- vtn_fail_if(node == NULL || node->type != vtn_cf_node_type_if,
- "Branching to the merge block of a selection "
- "construct can only be used to break out of a "
- "selection construct");
-
- struct vtn_if *if_stmt = vtn_cf_node_as_if(node);
-
- /* This should be guaranteed by our iteration */
- assert(if_stmt->merge_block != target_block);
-
- vtn_fail_if(if_stmt->merge_block != NULL,
- "Branching to the merge block of a selection "
- "construct can only be used to break out of the "
- "inner most nested selection level");
- }
- return vtn_branch_type_if_merge;
-
- case vtn_cf_node_type_loop:
- vtn_fail_if(target_block->merge_cf_node != &loop->node,
- "Loop breaks can only break out of the inner most "
- "nested loop level");
- return vtn_branch_type_loop_break;
-
- case vtn_cf_node_type_switch: {
- struct vtn_switch *swtch = vtn_cf_node_find_switch(cf_parent);
- vtn_fail_if(target_block->merge_cf_node != &swtch->node,
- "Switch breaks can only break out of the inner most "
- "nested switch level");
- return vtn_branch_type_switch_break;
- }
-
- default:
- unreachable("Invalid CF node type for a merge");
- }
- }
-
- if (target_block->switch_case)
- return vtn_branch_type_switch_fallthrough;
-
- return vtn_branch_type_none;
-}
-
-struct vtn_cfg_work_item {
- struct list_head link;
-
- struct vtn_cf_node *cf_parent;
- struct list_head *cf_list;
- struct vtn_block *start_block;
-};
-
-static void
-vtn_add_cfg_work_item(struct vtn_builder *b,
- struct list_head *work_list,
- struct vtn_cf_node *cf_parent,
- struct list_head *cf_list,
- struct vtn_block *start_block)
-{
- struct vtn_cfg_work_item *work = ralloc(b, struct vtn_cfg_work_item);
- work->cf_parent = cf_parent;
- work->cf_list = cf_list;
- work->start_block = start_block;
- list_addtail(&work->link, work_list);
-}
-
/* returns the default block */
-static void
+void
vtn_parse_switch(struct vtn_builder *b,
- struct vtn_switch *swtch,
const uint32_t *branch,
struct list_head *case_list)
{
cse = case_entry->data;
} else {
cse = rzalloc(b, struct vtn_case);
-
- cse->node.type = vtn_cf_node_type_case;
- cse->node.parent = swtch ? &swtch->node : NULL;
cse->block = case_block;
- list_inithead(&cse->body);
+ cse->block->switch_case = cse;
util_dynarray_init(&cse->values, b);
- list_addtail(&cse->node.link, case_list);
+ list_addtail(&cse->link, case_list);
_mesa_hash_table_insert(block_to_case, case_block, cse);
}
_mesa_hash_table_destroy(block_to_case, NULL);
}
-/* Processes a block and returns the next block to process or NULL if we've
- * reached the end of the construct.
- */
-static struct vtn_block *
-vtn_process_block(struct vtn_builder *b,
- struct list_head *work_list,
- struct vtn_cf_node *cf_parent,
- struct list_head *cf_list,
- struct vtn_block *block)
-{
- if (!list_is_empty(cf_list)) {
- /* vtn_process_block() acts like an iterator: it processes the given
- * block and then returns the next block to process. For a given
- * control-flow construct, vtn_build_cfg() calls vtn_process_block()
- * repeatedly until it finally returns NULL. Therefore, we know that
- * the only blocks on which vtn_process_block() can be called are either
- * the first block in a construct or a block that vtn_process_block()
- * returned for the current construct. If cf_list is empty then we know
- * that we're processing the first block in the construct and we have to
- * add it to the list.
- *
- * If cf_list is not empty, then it must be the block returned by the
- * previous call to vtn_process_block(). We know a priori that
- * vtn_process_block only returns either normal branches
- * (vtn_branch_type_none) or merge target blocks.
- */
- switch (vtn_handle_branch(b, cf_parent, block)) {
- case vtn_branch_type_none:
- /* For normal branches, we want to process them and add them to the
- * current construct. Merge target blocks also look like normal
- * branches from the perspective of this construct. See also
- * vtn_handle_branch().
- */
- break;
-
- case vtn_branch_type_loop_continue:
- case vtn_branch_type_switch_fallthrough:
- /* The two cases where we can get early exits from a construct that
- * are not to that construct's merge target are loop continues and
- * switch fall-throughs. In these cases, we need to break out of the
- * current construct by returning NULL.
- */
- return NULL;
-
- default:
- /* The only way we can get here is if something was used as two kinds
- * of merges at the same time and that's illegal.
- */
- vtn_fail("A block was used as a merge target from two or more "
- "structured control-flow constructs");
- }
- }
-
- /* Once a block has been processed, it is placed into and the list link
- * will point to something non-null. If we see a node we've already
- * processed here, it either exists in multiple functions or it's an
- * invalid back-edge.
- */
- if (block->node.parent != NULL) {
- vtn_fail_if(vtn_cf_node_find_function(&block->node) !=
- vtn_cf_node_find_function(cf_parent),
- "A block cannot exist in two functions at the "
- "same time");
-
- vtn_fail("Invalid back or cross-edge in the CFG");
- }
-
- if (block->merge && (*block->merge & SpvOpCodeMask) == SpvOpLoopMerge &&
- block->loop == NULL) {
- vtn_fail_if((*block->branch & SpvOpCodeMask) != SpvOpBranch &&
- (*block->branch & SpvOpCodeMask) != SpvOpBranchConditional,
- "An OpLoopMerge instruction must immediately precede "
- "either an OpBranch or OpBranchConditional instruction.");
-
- struct vtn_loop *loop = rzalloc(b, struct vtn_loop);
-
- loop->node.type = vtn_cf_node_type_loop;
- loop->node.parent = cf_parent;
- list_inithead(&loop->body);
- list_inithead(&loop->cont_body);
- loop->header_block = block;
- loop->break_block = vtn_block(b, block->merge[1]);
- loop->cont_block = vtn_block(b, block->merge[2]);
- loop->control = block->merge[3];
-
- list_addtail(&loop->node.link, cf_list);
- block->loop = loop;
-
- /* Note: The work item for the main loop body will start with the
- * current block as its start block. If we weren't careful, we would
- * get here again and end up in an infinite loop. This is why we set
- * block->loop above and check for it before creating one. This way,
- * we only create the loop once and the second iteration that tries to
- * handle this loop goes to the cases below and gets handled as a
- * regular block.
- */
- vtn_add_cfg_work_item(b, work_list, &loop->node,
- &loop->body, loop->header_block);
-
- /* For continue targets, SPIR-V guarantees the following:
- *
- * - the Continue Target must dominate the back-edge block
- * - the back-edge block must post dominate the Continue Target
- *
- * If the header block is the same as the continue target, this
- * condition is trivially satisfied and there is no real continue
- * section.
- */
- if (loop->cont_block != loop->header_block) {
- vtn_add_cfg_work_item(b, work_list, &loop->node,
- &loop->cont_body, loop->cont_block);
- }
-
- vtn_block_set_merge_cf_node(b, loop->break_block, &loop->node);
-
- return loop->break_block;
- }
-
- /* Add the block to the CF list */
- block->node.parent = cf_parent;
- list_addtail(&block->node.link, cf_list);
-
- switch (*block->branch & SpvOpCodeMask) {
- case SpvOpBranch: {
- struct vtn_block *branch_block = vtn_block(b, block->branch[1]);
-
- block->branch_type = vtn_handle_branch(b, cf_parent, branch_block);
-
- if (block->branch_type == vtn_branch_type_none)
- return branch_block;
- else
- return NULL;
- }
-
- case SpvOpReturn:
- case SpvOpReturnValue:
- block->branch_type = vtn_branch_type_return;
- return NULL;
-
- case SpvOpKill:
- block->branch_type = vtn_branch_type_discard;
- return NULL;
-
- case SpvOpTerminateInvocation:
- block->branch_type = vtn_branch_type_terminate_invocation;
- return NULL;
-
- case SpvOpIgnoreIntersectionKHR:
- block->branch_type = vtn_branch_type_ignore_intersection;
- return NULL;
-
- case SpvOpTerminateRayKHR:
- block->branch_type = vtn_branch_type_terminate_ray;
- return NULL;
-
- case SpvOpEmitMeshTasksEXT:
- block->branch_type = vtn_branch_type_emit_mesh_tasks;
- return NULL;
-
- case SpvOpBranchConditional: {
- struct vtn_value *cond_val = vtn_untyped_value(b, block->branch[1]);
- vtn_fail_if(!cond_val->type ||
- cond_val->type->base_type != vtn_base_type_scalar ||
- cond_val->type->type != glsl_bool_type(),
- "Condition must be a Boolean type scalar");
-
- struct vtn_if *if_stmt = rzalloc(b, struct vtn_if);
-
- if_stmt->node.type = vtn_cf_node_type_if;
- if_stmt->node.parent = cf_parent;
- if_stmt->header_block = block;
- list_inithead(&if_stmt->then_body);
- list_inithead(&if_stmt->else_body);
-
- list_addtail(&if_stmt->node.link, cf_list);
-
- if (block->merge &&
- (*block->merge & SpvOpCodeMask) == SpvOpSelectionMerge) {
- /* We may not always have a merge block and that merge doesn't
- * technically have to be an OpSelectionMerge. We could have a block
- * with an OpLoopMerge which ends in an OpBranchConditional.
- */
- if_stmt->merge_block = vtn_block(b, block->merge[1]);
- vtn_block_set_merge_cf_node(b, if_stmt->merge_block, &if_stmt->node);
-
- if_stmt->control = block->merge[2];
- }
-
- struct vtn_block *then_block = vtn_block(b, block->branch[2]);
- if_stmt->then_type = vtn_handle_branch(b, &if_stmt->node, then_block);
- if (if_stmt->then_type == vtn_branch_type_none) {
- vtn_add_cfg_work_item(b, work_list, &if_stmt->node,
- &if_stmt->then_body, then_block);
- }
-
- struct vtn_block *else_block = vtn_block(b, block->branch[3]);
- if (then_block != else_block) {
- if_stmt->else_type = vtn_handle_branch(b, &if_stmt->node, else_block);
- if (if_stmt->else_type == vtn_branch_type_none) {
- vtn_add_cfg_work_item(b, work_list, &if_stmt->node,
- &if_stmt->else_body, else_block);
- }
- }
-
- return if_stmt->merge_block;
- }
-
- case SpvOpSwitch: {
- struct vtn_switch *swtch = rzalloc(b, struct vtn_switch);
-
- swtch->node.type = vtn_cf_node_type_switch;
- swtch->node.parent = cf_parent;
- swtch->selector = block->branch[1];
- list_inithead(&swtch->cases);
-
- list_addtail(&swtch->node.link, cf_list);
-
- /* We may not always have a merge block */
- if (block->merge) {
- vtn_fail_if((*block->merge & SpvOpCodeMask) != SpvOpSelectionMerge,
- "An OpLoopMerge instruction must immediately precede "
- "either an OpBranch or OpBranchConditional "
- "instruction.");
- swtch->break_block = vtn_block(b, block->merge[1]);
- vtn_block_set_merge_cf_node(b, swtch->break_block, &swtch->node);
- }
-
- /* First, we go through and record all of the cases. */
- vtn_parse_switch(b, swtch, block->branch, &swtch->cases);
-
- /* Gather the branch types for the switch */
- vtn_foreach_cf_node(case_node, &swtch->cases) {
- struct vtn_case *cse = vtn_cf_node_as_case(case_node);
-
- cse->type = vtn_handle_branch(b, &swtch->node, cse->block);
- switch (cse->type) {
- case vtn_branch_type_none:
- /* This is a "real" cases which has stuff in it */
- vtn_fail_if(cse->block->switch_case != NULL,
- "OpSwitch has a case which is also in another "
- "OpSwitch construct");
- cse->block->switch_case = cse;
- vtn_add_cfg_work_item(b, work_list, &cse->node,
- &cse->body, cse->block);
- break;
-
- case vtn_branch_type_switch_break:
- case vtn_branch_type_loop_break:
- case vtn_branch_type_loop_continue:
- /* Switch breaks as well as loop breaks and continues can be
- * used to break out of a switch construct or as direct targets
- * of the OpSwitch.
- */
- break;
-
- default:
- vtn_fail("Target of OpSwitch is not a valid structured exit "
- "from the switch construct.");
- }
- }
-
- return swtch->break_block;
- }
-
- case SpvOpUnreachable:
- return NULL;
-
- default:
- vtn_fail("Block did not end with a valid branch instruction");
- }
-}
-
void
vtn_build_cfg(struct vtn_builder *b, const uint32_t *words, const uint32_t *end)
{
if (b->shader->info.stage == MESA_SHADER_KERNEL)
return;
- vtn_foreach_cf_node(func_node, &b->functions) {
- struct vtn_function *func = vtn_cf_node_as_function(func_node);
-
- /* We build the CFG for each function by doing a breadth-first search on
- * the control-flow graph. We keep track of our state using a worklist.
- * Doing a BFS ensures that we visit each structured control-flow
- * construct and its merge node before we visit the stuff inside the
- * construct.
- */
- struct list_head work_list;
- list_inithead(&work_list);
- vtn_add_cfg_work_item(b, &work_list, &func->node, &func->body,
- func->start_block);
-
- while (!list_is_empty(&work_list)) {
- struct vtn_cfg_work_item *work =
- list_first_entry(&work_list, struct vtn_cfg_work_item, link);
- list_del(&work->link);
-
- for (struct vtn_block *block = work->start_block; block; ) {
- block = vtn_process_block(b, &work_list, work->cf_parent,
- work->cf_list, block);
- }
- }
- }
+ vtn_build_structured_cfg(b, words, end);
}
-static bool
+bool
vtn_handle_phis_first_pass(struct vtn_builder *b, SpvOp opcode,
const uint32_t *w, unsigned count)
{
return true;
}
-static void
+void
vtn_emit_ret_store(struct vtn_builder *b, const struct vtn_block *block)
{
if ((*block->branch & SpvOpCodeMask) != SpvOpReturnValue)
vtn_local_store(b, src, ret_deref, 0);
}
-static void
-vtn_emit_branch(struct vtn_builder *b, enum vtn_branch_type branch_type,
- const struct vtn_block *block,
- nir_variable *switch_fall_var, bool *has_switch_break)
-{
- switch (branch_type) {
- case vtn_branch_type_if_merge:
- break; /* Nothing to do */
- case vtn_branch_type_switch_break:
- nir_store_var(&b->nb, switch_fall_var, nir_imm_false(&b->nb), 1);
- *has_switch_break = true;
- break;
- case vtn_branch_type_switch_fallthrough:
- break; /* Nothing to do */
- case vtn_branch_type_loop_break:
- nir_jump(&b->nb, nir_jump_break);
- break;
- case vtn_branch_type_loop_continue:
- nir_jump(&b->nb, nir_jump_continue);
- break;
- case vtn_branch_type_loop_back_edge:
- break;
- case vtn_branch_type_return:
- vtn_assert(block);
- vtn_emit_ret_store(b, block);
- nir_jump(&b->nb, nir_jump_return);
- break;
- case vtn_branch_type_discard:
- if (b->convert_discard_to_demote)
- nir_demote(&b->nb);
- else
- nir_discard(&b->nb);
- break;
- case vtn_branch_type_terminate_invocation:
- nir_terminate(&b->nb);
- break;
- case vtn_branch_type_ignore_intersection:
- nir_ignore_ray_intersection(&b->nb);
- nir_jump(&b->nb, nir_jump_halt);
- break;
- case vtn_branch_type_terminate_ray:
- nir_terminate_ray(&b->nb);
- nir_jump(&b->nb, nir_jump_halt);
- break;
- case vtn_branch_type_emit_mesh_tasks: {
- assert(block);
- assert(block->branch);
-
- const uint32_t *w = block->branch;
- vtn_assert((w[0] & SpvOpCodeMask) == SpvOpEmitMeshTasksEXT);
-
- /* Launches mesh shader workgroups from the task shader.
- * Arguments are: vec(x, y, z), payload pointer
- */
- nir_ssa_def *dimensions =
- nir_vec3(&b->nb, vtn_get_nir_ssa(b, w[1]),
- vtn_get_nir_ssa(b, w[2]),
- vtn_get_nir_ssa(b, w[3]));
-
- /* The payload variable is optional.
- * We don't have a NULL deref in NIR, so just emit the explicit
- * intrinsic when there is no payload.
- */
- const unsigned count = w[0] >> SpvWordCountShift;
- if (count == 4)
- nir_launch_mesh_workgroups(&b->nb, dimensions);
- else if (count == 5)
- nir_launch_mesh_workgroups_with_payload_deref(&b->nb, dimensions,
- vtn_get_nir_ssa(b, w[4]));
- else
- vtn_fail("Invalid EmitMeshTasksEXT.");
-
- nir_jump(&b->nb, nir_jump_halt);
- break;
- }
- default:
- vtn_fail("Invalid branch type");
- }
-}
-
-static nir_ssa_def *
-vtn_switch_case_condition(struct vtn_builder *b, struct vtn_switch *swtch,
- nir_ssa_def *sel, struct vtn_case *cse)
-{
- if (cse->is_default) {
- nir_ssa_def *any = nir_imm_false(&b->nb);
- vtn_foreach_cf_node(other_node, &swtch->cases) {
- struct vtn_case *other = vtn_cf_node_as_case(other_node);
- if (other->is_default)
- continue;
-
- any = nir_ior(&b->nb, any,
- vtn_switch_case_condition(b, swtch, sel, other));
- }
- return nir_inot(&b->nb, any);
- } else {
- nir_ssa_def *cond = nir_imm_false(&b->nb);
- util_dynarray_foreach(&cse->values, uint64_t, val)
- cond = nir_ior(&b->nb, cond, nir_ieq_imm(&b->nb, sel, *val));
- return cond;
- }
-}
-
-static nir_loop_control
-vtn_loop_control(struct vtn_builder *b, struct vtn_loop *vtn_loop)
-{
- if (vtn_loop->control == SpvLoopControlMaskNone)
- return nir_loop_control_none;
- else if (vtn_loop->control & SpvLoopControlDontUnrollMask)
- return nir_loop_control_dont_unroll;
- else if (vtn_loop->control & SpvLoopControlUnrollMask)
- return nir_loop_control_unroll;
- else if (vtn_loop->control & SpvLoopControlDependencyInfiniteMask ||
- vtn_loop->control & SpvLoopControlDependencyLengthMask ||
- vtn_loop->control & SpvLoopControlMinIterationsMask ||
- vtn_loop->control & SpvLoopControlMaxIterationsMask ||
- vtn_loop->control & SpvLoopControlIterationMultipleMask ||
- vtn_loop->control & SpvLoopControlPeelCountMask ||
- vtn_loop->control & SpvLoopControlPartialCountMask) {
- /* We do not do anything special with these yet. */
- return nir_loop_control_none;
- } else {
- vtn_fail("Invalid loop control");
- }
-}
-
-static nir_selection_control
-vtn_selection_control(struct vtn_builder *b, struct vtn_if *vtn_if)
-{
- if (vtn_if->control == SpvSelectionControlMaskNone)
- return nir_selection_control_none;
- else if (vtn_if->control & SpvSelectionControlDontFlattenMask)
- return nir_selection_control_dont_flatten;
- else if (vtn_if->control & SpvSelectionControlFlattenMask)
- return nir_selection_control_flatten;
- else
- vtn_fail("Invalid selection control");
-}
-
-static void
-vtn_emit_cf_list_structured(struct vtn_builder *b, struct list_head *cf_list,
- nir_variable *switch_fall_var,
- bool *has_switch_break,
- vtn_instruction_handler handler)
-{
- vtn_foreach_cf_node(node, cf_list) {
- switch (node->type) {
- case vtn_cf_node_type_block: {
- struct vtn_block *block = vtn_cf_node_as_block(node);
-
- const uint32_t *block_start = block->label;
- const uint32_t *block_end = block->merge ? block->merge :
- block->branch;
-
- block_start = vtn_foreach_instruction(b, block_start, block_end,
- vtn_handle_phis_first_pass);
-
- vtn_foreach_instruction(b, block_start, block_end, handler);
-
- block->end_nop = nir_nop(&b->nb);
-
- if (block->branch_type != vtn_branch_type_none) {
- vtn_emit_branch(b, block->branch_type, block,
- switch_fall_var, has_switch_break);
- return;
- }
-
- break;
- }
-
- case vtn_cf_node_type_if: {
- struct vtn_if *vtn_if = vtn_cf_node_as_if(node);
- const uint32_t *branch = vtn_if->header_block->branch;
- vtn_assert((branch[0] & SpvOpCodeMask) == SpvOpBranchConditional);
-
- bool sw_break = false;
- /* If both branches are the same, just emit the first block, which is
- * the only one we filled when building the CFG.
- */
- if (branch[2] == branch[3]) {
- if (vtn_if->then_type == vtn_branch_type_none) {
- vtn_emit_cf_list_structured(b, &vtn_if->then_body,
- switch_fall_var, &sw_break, handler);
- } else {
- vtn_emit_branch(b, vtn_if->then_type, NULL, switch_fall_var, &sw_break);
- }
- break;
- }
-
- nir_if *nif =
- nir_push_if(&b->nb, vtn_get_nir_ssa(b, branch[1]));
-
- nif->control = vtn_selection_control(b, vtn_if);
-
- if (vtn_if->then_type == vtn_branch_type_none) {
- vtn_emit_cf_list_structured(b, &vtn_if->then_body,
- switch_fall_var, &sw_break, handler);
- } else {
- vtn_emit_branch(b, vtn_if->then_type, NULL, switch_fall_var, &sw_break);
- }
-
- nir_push_else(&b->nb, nif);
- if (vtn_if->else_type == vtn_branch_type_none) {
- vtn_emit_cf_list_structured(b, &vtn_if->else_body,
- switch_fall_var, &sw_break, handler);
- } else {
- vtn_emit_branch(b, vtn_if->else_type, NULL, switch_fall_var, &sw_break);
- }
-
- nir_pop_if(&b->nb, nif);
-
- /* If we encountered a switch break somewhere inside of the if,
- * then it would have been handled correctly by calling
- * emit_cf_list or emit_branch for the interrior. However, we
- * need to predicate everything following on wether or not we're
- * still going.
- */
- if (sw_break) {
- *has_switch_break = true;
- nir_push_if(&b->nb, nir_load_var(&b->nb, switch_fall_var));
- }
- break;
- }
-
- case vtn_cf_node_type_loop: {
- struct vtn_loop *vtn_loop = vtn_cf_node_as_loop(node);
-
- nir_loop *loop = nir_push_loop(&b->nb);
- loop->control = vtn_loop_control(b, vtn_loop);
- vtn_emit_cf_list_structured(b, &vtn_loop->body, NULL, NULL, handler);
-
- nir_push_continue(&b->nb, loop);
- vtn_emit_cf_list_structured(b, &vtn_loop->cont_body, NULL, NULL, handler);
-
- nir_pop_loop(&b->nb, loop);
- break;
- }
-
- case vtn_cf_node_type_switch: {
- struct vtn_switch *vtn_switch = vtn_cf_node_as_switch(node);
-
- /* Before we can emit anything, we need to sort the list of cases in
- * fall-through order.
- */
- vtn_switch_order_cases(vtn_switch);
-
- /* First, we create a variable to keep track of whether or not the
- * switch is still going at any given point. Any switch breaks
- * will set this variable to false.
- */
- nir_variable *fall_var =
- nir_local_variable_create(b->nb.impl, glsl_bool_type(), "fall");
- nir_store_var(&b->nb, fall_var, nir_imm_false(&b->nb), 1);
-
- nir_ssa_def *sel = vtn_get_nir_ssa(b, vtn_switch->selector);
-
- /* Now we can walk the list of cases and actually emit code */
- vtn_foreach_cf_node(case_node, &vtn_switch->cases) {
- struct vtn_case *cse = vtn_cf_node_as_case(case_node);
-
- /* If this case jumps directly to the break block, we don't have
- * to handle the case as the body is empty and doesn't fall
- * through.
- */
- if (cse->block == vtn_switch->break_block)
- continue;
-
- /* Figure out the condition */
- nir_ssa_def *cond =
- vtn_switch_case_condition(b, vtn_switch, sel, cse);
- /* Take fallthrough into account */
- cond = nir_ior(&b->nb, cond, nir_load_var(&b->nb, fall_var));
-
- nir_if *case_if = nir_push_if(&b->nb, cond);
-
- bool has_break = false;
- nir_store_var(&b->nb, fall_var, nir_imm_true(&b->nb), 1);
- vtn_emit_cf_list_structured(b, &cse->body, fall_var, &has_break,
- handler);
- (void)has_break; /* We don't care */
-
- nir_pop_if(&b->nb, case_if);
- }
-
- break;
- }
-
- default:
- vtn_fail("Invalid CF node type");
- }
- }
-}
-
static struct nir_block *
vtn_new_unstructured_block(struct vtn_builder *b, struct vtn_function *func)
{
{
if (!block->block) {
block->block = vtn_new_unstructured_block(b, func);
- list_addtail(&block->node.link, work_list);
+ list_addtail(&block->link, work_list);
}
}
list_inithead(&work_list);
func->start_block->block = nir_start_block(func->nir_func->impl);
- list_addtail(&func->start_block->node.link, &work_list);
+ list_addtail(&func->start_block->link, &work_list);
while (!list_is_empty(&work_list)) {
struct vtn_block *block =
- list_first_entry(&work_list, struct vtn_block, node.link);
- list_del(&block->node.link);
+ list_first_entry(&work_list, struct vtn_block, link);
+ list_del(&block->link);
vtn_assert(block->block);
case SpvOpSwitch: {
struct list_head cases;
list_inithead(&cases);
- vtn_parse_switch(b, NULL, block->branch, &cases);
+ vtn_parse_switch(b, block->branch, &cases);
nir_ssa_def *sel = vtn_get_nir_ssa(b, block->branch[1]);
struct vtn_case *def = NULL;
- vtn_foreach_cf_node(case_node, &cases) {
- struct vtn_case *cse = vtn_cf_node_as_case(case_node);
+ vtn_foreach_case(cse, &cases) {
if (cse->is_default) {
assert(def == NULL);
def = cse;
impl->structured = false;
vtn_emit_cf_func_unstructured(b, func, instruction_handler);
} else {
- vtn_emit_cf_list_structured(b, &func->body, NULL, NULL,
- instruction_handler);
+ vtn_emit_cf_func_structured(b, func, instruction_handler);
}
vtn_foreach_instruction(b, func->start_block->label, func->end,
--- /dev/null
+/*
+ * Copyright © 2015-2023 Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ */
+
+#include "vtn_private.h"
+#include "spirv_info.h"
+#include "util/u_math.h"
+
+/* Handle SPIR-V structured control flow, mapping SPIR-V constructs into
+ * equivalent NIR constructs.
+ *
+ * Because SPIR-V can represent more complex control flow than NIR, some
+ * constructs are mapped into a combination of nir_if and nir_loop nodes. For
+ * example, an selection construct with an "if-break" (an early branch into
+ * the end of the construct) will be mapped into NIR as a loop (to allow the
+ * break) with a nested if (to handle the actual selection).
+ *
+ * Note that using NIR loops this way requires us to propagate breaks and
+ * continues that are meant to outer constructs when a nir_loop is used for a
+ * SPIR-V construct other than Loop.
+ *
+ * The process of identifying and ordering the blocks before the NIR
+ * translation is similar to what's done in Tint, using the "reverse
+ * structured post-order traversal". See also the file comments
+ * src/reader/spirv/function.cc in the Tint repository.
+ */
+
+enum vtn_construct_type {
+ /* Not formally a SPIR-V construct but used to represent the entire
+ * function.
+ */
+ vtn_construct_type_function,
+
+ /* Selection construct uses a nir_if and optionally a nir_loop to handle
+ * if-breaks.
+ */
+ vtn_construct_type_selection,
+
+ /* Loop construct uses a nir_loop and optionally a nir_if to handle an
+ * OpBranchConditional as part of the head of the loop.
+ */
+ vtn_construct_type_loop,
+
+ /* Continue construct maps to the NIR continue construct of the corresponding
+ * loop. For convenience, unlike in SPIR-V, the parent of this construct is
+ * always the loop construct. Continue construct is omitted for single-block
+ * loops.
+ */
+ vtn_construct_type_continue,
+
+ /* Switch construct is not directly mapped into any NIR structure, the work
+ * is handled by the case constructs. It does keep a nir_variable for
+ * handling case fallback logic.
+ */
+ vtn_construct_type_switch,
+
+ /* Case construct uses a nir_if and optionally a nir_loop to handle early
+ * breaks. Note switch_breaks are handled by each case.
+ */
+ vtn_construct_type_case,
+};
+
+static const char *
+vtn_construct_type_to_string(enum vtn_construct_type t)
+{
+#define CASE(typ) case vtn_construct_type_##typ: return #typ
+ switch (t) {
+ CASE(function);
+ CASE(selection);
+ CASE(loop);
+ CASE(continue);
+ CASE(switch);
+ CASE(case);
+ }
+#undef CASE
+ unreachable("invalid construct type");
+ return "";
+}
+
+struct vtn_construct {
+ enum vtn_construct_type type;
+
+ bool needs_nloop;
+ bool needs_break_propagation;
+ bool needs_continue_propagation;
+ bool needs_fallthrough;
+
+ struct vtn_construct *parent;
+
+ struct vtn_construct *innermost_loop;
+ struct vtn_construct *innermost_switch;
+ struct vtn_construct *innermost_case;
+
+ unsigned start_pos;
+ unsigned end_pos;
+
+ /* Usually the same as end_pos, but may be different in case of an "early
+ * merge" after divergence caused by an OpBranchConditional. This can
+ * happen in selection and loop constructs.
+ */
+ unsigned merge_pos;
+
+ /* Valid when not zero, indicates the block that starts the then and else
+ * paths in a condition. This may be used by selection constructs.
+ */
+ unsigned then_pos;
+ unsigned else_pos;
+
+ /* Indicates where the continue block is, marking the end of the body of
+ * the loop. Note the block ordering will always give us first the loop
+ * body blocks then the continue block. Used by loop construct.
+ */
+ unsigned continue_pos;
+
+ /* For the list of all constructs in vtn_function. */
+ struct list_head link;
+
+ /* NIR nodes that are associated with this construct. See
+ * vtn_construct_type for an overview.
+ */
+ nir_loop *nloop;
+ nir_if *nif;
+
+ /* This variable will be set by an inner construct to indicate that a break
+ * is necessary. We need to use variables here for situations when the
+ * inner construct has a loop of its own for other reasons.
+ */
+ nir_variable *break_var;
+
+ /* Same logic but for continue. */
+ nir_variable *continue_var;
+
+ /* This is used by each case to force entering in the case regardless of
+ * the condition. We always set it when handling a branch that is a
+ * switch_break or a switch_fallthrough.
+ */
+ nir_variable *fallthrough_var;
+
+ unsigned index;
+};
+
+enum vtn_branch_type {
+ vtn_branch_type_none,
+ vtn_branch_type_forward,
+ vtn_branch_type_if_break,
+ vtn_branch_type_switch_break,
+ vtn_branch_type_switch_fallthrough,
+ vtn_branch_type_loop_break,
+ vtn_branch_type_loop_continue,
+ vtn_branch_type_loop_back_edge,
+ vtn_branch_type_discard,
+ vtn_branch_type_terminate_invocation,
+ vtn_branch_type_ignore_intersection,
+ vtn_branch_type_terminate_ray,
+ vtn_branch_type_emit_mesh_tasks,
+ vtn_branch_type_return,
+};
+
+static const char *
+vtn_branch_type_to_string(enum vtn_branch_type t)
+{
+#define CASE(typ) case vtn_branch_type_##typ: return #typ
+ switch (t) {
+ CASE(none);
+ CASE(forward);
+ CASE(if_break);
+ CASE(switch_break);
+ CASE(switch_fallthrough);
+ CASE(loop_break);
+ CASE(loop_continue);
+ CASE(loop_back_edge);
+ CASE(discard);
+ CASE(terminate_invocation);
+ CASE(ignore_intersection);
+ CASE(terminate_ray);
+ CASE(emit_mesh_tasks);
+ CASE(return);
+ }
+#undef CASE
+ unreachable("unknown branch type");
+ return "";
+}
+
+struct vtn_successor {
+ struct vtn_block *block;
+ enum vtn_branch_type branch_type;
+};
+
+static bool
+vtn_is_single_block_loop(const struct vtn_construct *c)
+{
+ return c->type == vtn_construct_type_loop &&
+ c->start_pos == c->continue_pos;
+}
+
+static struct vtn_construct *
+vtn_find_innermost(enum vtn_construct_type type, struct vtn_construct *c)
+{
+ while (c && c->type != type)
+ c = c->parent;
+ return c;
+}
+
+static void
+print_ordered_blocks(const struct vtn_function *func)
+{
+ for (unsigned i = 0; i < func->ordered_blocks_count; i++) {
+ struct vtn_block *block = func->ordered_blocks[i];
+ printf("[id=%-6u] %4u", block->label[1], block->pos);
+ if (block->successors_count > 0) {
+ printf(" ->");
+ for (unsigned j = 0; j < block->successors_count; j++) {
+ printf(" ");
+ if (block->successors[j].block)
+ printf("%u/", block->successors[j].block->pos);
+ printf("%s", vtn_branch_type_to_string(block->successors[j].branch_type));
+ }
+ }
+ if (!block->visited)
+ printf(" NOT VISITED");
+ printf("\n");
+ }
+}
+
+static struct vtn_case *
+vtn_find_fallthrough_target(struct vtn_builder *b, const uint32_t *switch_merge,
+ struct vtn_block *source_block, struct vtn_block *block)
+{
+ if (block->visited)
+ return NULL;
+
+ if (block->label[1] == switch_merge[1])
+ return NULL;
+
+ /* Don't consider the initial source block a fallthrough target of itself. */
+ if (block->switch_case && block != source_block)
+ return block->switch_case;
+
+ if (block->merge)
+ return vtn_find_fallthrough_target(b, switch_merge, source_block,
+ vtn_block(b, block->merge[1]));
+
+ const uint32_t *branch = block->branch;
+ vtn_assert(branch);
+
+ switch (branch[0] & SpvOpCodeMask) {
+ case SpvOpBranch:
+ return vtn_find_fallthrough_target(b, switch_merge, source_block,
+ vtn_block(b, branch[1]));
+ case SpvOpBranchConditional: {
+ struct vtn_case *target =
+ vtn_find_fallthrough_target(b, switch_merge, source_block,
+ vtn_block(b, branch[2]));
+ if (!target)
+ target = vtn_find_fallthrough_target(b, switch_merge, source_block,
+ vtn_block(b, branch[3]));
+ return target;
+ }
+ default:
+ return NULL;
+ }
+}
+
+static void
+structured_post_order_traversal(struct vtn_builder *b, struct vtn_block *block)
+{
+ if (block->visited)
+ return;
+
+ block->visited = true;
+
+ if (block->merge) {
+ structured_post_order_traversal(b, vtn_block(b, block->merge[1]));
+
+ SpvOp merge_op = block->merge[0] & SpvOpCodeMask;
+ if (merge_op == SpvOpLoopMerge) {
+ struct vtn_block *continue_block = vtn_block(b, block->merge[2]);
+ structured_post_order_traversal(b, continue_block);
+ }
+ }
+
+ const uint32_t *branch = block->branch;
+ vtn_assert(branch);
+
+ switch (branch[0] & SpvOpCodeMask) {
+ case SpvOpBranch:
+ block->successors_count = 1;
+ block->successors = rzalloc(b, struct vtn_successor);
+ block->successors[0].block = vtn_block(b, branch[1]);
+ structured_post_order_traversal(b, block->successors[0].block);
+ break;
+
+ case SpvOpBranchConditional:
+ block->successors_count = 2;
+ block->successors = rzalloc_array(b, struct vtn_successor, 2);
+ block->successors[0].block = vtn_block(b, branch[2]);
+ block->successors[1].block = vtn_block(b, branch[3]);
+
+ /* The result of the traversal will be reversed, so to provide a
+ * more natural order, with THEN blocks appearing before ELSE blocks,
+ * we need to traverse them in the reversed order.
+ */
+ int order[] = { 1, 0 };
+
+ /* There's a catch when traversing case fallthroughs: we want to avoid
+ * walking part of a case construct, then the fallthrough -- possibly
+ * visiting another entire case construct, and back to the other part
+ * of that original case construct. So if the THEN path is a fallthrough,
+ * swap the visit order.
+ */
+ if (block->successors[0].block->switch_case) {
+ order[0] = !order[0];
+ order[1] = !order[1];
+ }
+
+ structured_post_order_traversal(b, block->successors[order[0]].block);
+ structured_post_order_traversal(b, block->successors[order[1]].block);
+ break;
+
+ case SpvOpSwitch: {
+ /* TODO: Save this to use during Switch construct creation. */
+ struct list_head cases;
+ list_inithead(&cases);
+ vtn_parse_switch(b, block->branch, &cases);
+
+ block->successors_count = list_length(&cases);
+ block->successors = rzalloc_array(b, struct vtn_successor, block->successors_count);
+
+ /* The 'Rules for Structured Control-flow constructs' already guarantee
+ * that the labels of the targets are ordered in a way that if
+ * there is a fallthrough, they will appear consecutively. The only
+ * exception is Default, which is always the first in the list.
+ *
+ * Because we are doing a DFS from the end of the cases, the
+ * traversal already handle a Case falling through Default.
+ *
+ * The scenario that needs fixing is when no case falls to Default, but
+ * Default falls to another case. For that scenario we move the Default
+ * right before the case it falls to.
+ */
+
+ struct vtn_case *default_case = list_first_entry(&cases, struct vtn_case, link);
+ vtn_assert(default_case && default_case->is_default);
+
+ struct vtn_case *fall_target =
+ vtn_find_fallthrough_target(b, block->merge, default_case->block,
+ default_case->block);
+ if (fall_target)
+ list_move_to(&default_case->link, &fall_target->link);
+
+ /* Because the result of the traversal will be reversed, loop backwards
+ * in the case list.
+ */
+ unsigned i = 0;
+ list_for_each_entry_rev(struct vtn_case, cse, &cases, link) {
+ structured_post_order_traversal(b, cse->block);
+ block->successors[i].block = cse->block;
+ i++;
+ }
+
+ break;
+ }
+
+ case SpvOpKill:
+ case SpvOpTerminateInvocation:
+ case SpvOpIgnoreIntersectionKHR:
+ case SpvOpTerminateRayKHR:
+ case SpvOpReturn:
+ case SpvOpReturnValue:
+ case SpvOpEmitMeshTasksEXT:
+ case SpvOpUnreachable:
+ block->successors_count = 1;
+ block->successors = rzalloc(b, struct vtn_successor);
+ break;
+
+ default:
+ unreachable("invalid branch opcode");
+ }
+
+ b->func->ordered_blocks[b->func->ordered_blocks_count++] = block;
+}
+
+static void
+sort_blocks(struct vtn_builder *b)
+{
+ struct vtn_block **ordered_blocks =
+ rzalloc_array(b, struct vtn_block *, b->func->block_count);
+
+ b->func->ordered_blocks = ordered_blocks;
+
+ structured_post_order_traversal(b, b->func->start_block);
+
+ /* Reverse it, so that blocks appear before their successors. */
+ unsigned count = b->func->ordered_blocks_count;
+ for (unsigned i = 0; i < (count / 2); i++) {
+ unsigned j = count - i - 1;
+ struct vtn_block *tmp = ordered_blocks[i];
+ ordered_blocks[i] = ordered_blocks[j];
+ ordered_blocks[j] = tmp;
+ }
+
+ for (unsigned i = 0; i < count; i++)
+ ordered_blocks[i]->pos = i;
+}
+
+static void
+print_construct(const struct vtn_function *func,
+ const struct vtn_construct *c)
+{
+ for (const struct vtn_construct *p = c->parent; p; p = p->parent)
+ printf(" ");
+ printf("C%u/%s ", c->index, vtn_construct_type_to_string(c->type));
+ printf(" %u->%u", c->start_pos, c->end_pos);
+ if (c->merge_pos)
+ printf(" merge=%u", c->merge_pos);
+ if (c->then_pos)
+ printf(" then=%u", c->then_pos);
+ if (c->else_pos)
+ printf(" else=%u", c->else_pos);
+ if (c->needs_nloop)
+ printf(" nloop");
+ if (c->needs_break_propagation)
+ printf(" break_prop");
+ if (c->needs_continue_propagation)
+ printf(" continue_prop");
+ if (c->type == vtn_construct_type_loop) {
+ if (vtn_is_single_block_loop(c))
+ printf(" single_block_loop");
+ else
+ printf(" cont=%u", c->continue_pos);
+ }
+ if (c->type == vtn_construct_type_case) {
+ struct vtn_block *block = func->ordered_blocks[c->start_pos];
+ if (block->switch_case->is_default) {
+ printf(" [default]");
+ } else {
+ printf(" [values:");
+ util_dynarray_foreach(&block->switch_case->values, uint64_t, val)
+ printf(" %" PRIu64, *val);
+ printf("]");
+ }
+ }
+ printf("\n");
+}
+
+static void
+print_constructs(struct vtn_function *func)
+{
+ list_for_each_entry(struct vtn_construct, c, &func->constructs, link)
+ print_construct(func, c);
+}
+
+struct vtn_construct_stack {
+ /* Array of `struct vtn_construct *`. */
+ struct util_dynarray data;
+};
+
+static inline void
+init_construct_stack(struct vtn_construct_stack *stack, void *mem_ctx)
+{
+ assert(mem_ctx);
+ util_dynarray_init(&stack->data, mem_ctx);
+}
+
+static inline unsigned
+count_construct_stack(struct vtn_construct_stack *stack)
+{
+ return util_dynarray_num_elements(&stack->data, struct vtn_construct *);
+}
+
+static inline struct vtn_construct *
+top_construct(struct vtn_construct_stack *stack)
+{
+ assert(count_construct_stack(stack) > 0);
+ return util_dynarray_top(&stack->data, struct vtn_construct *);
+}
+
+static inline void
+pop_construct(struct vtn_construct_stack *stack)
+{
+ assert(count_construct_stack(stack) > 0);
+ (void)util_dynarray_pop(&stack->data, struct vtn_construct *);
+}
+
+static inline void
+push_construct(struct vtn_construct_stack *stack, struct vtn_construct *c)
+{
+ util_dynarray_append(&stack->data, struct vtn_construct *, c);
+}
+
+static int
+cmp_succ_block_pos(const void *pa, const void *pb)
+{
+ const struct vtn_successor *sa = pa;
+ const struct vtn_successor *sb = pb;
+ const unsigned a = sa->block->pos;
+ const unsigned b = sb->block->pos;
+ if (a < b)
+ return -1;
+ if (a > b)
+ return 1;
+ return 0;
+}
+
+static void
+create_constructs(struct vtn_builder *b)
+{
+ struct vtn_construct *func_construct = rzalloc(b, struct vtn_construct);
+ func_construct->type = vtn_construct_type_function;
+ func_construct->start_pos = 0;
+ func_construct->end_pos = b->func->ordered_blocks_count;
+
+ for (unsigned i = 0; i < b->func->ordered_blocks_count; i++) {
+ struct vtn_block *block = b->func->ordered_blocks[i];
+
+ if (block->merge) {
+ SpvOp merge_op = block->merge[0] & SpvOpCodeMask;
+ SpvOp branch_op = block->branch[0] & SpvOpCodeMask;
+
+ const unsigned end_pos = vtn_block(b, block->merge[1])->pos;
+
+ if (merge_op == SpvOpLoopMerge) {
+ struct vtn_construct *loop = rzalloc(b, struct vtn_construct);
+ loop->type = vtn_construct_type_loop;
+ loop->start_pos = block->pos;
+ loop->end_pos = end_pos;
+
+ loop->parent = block->parent;
+ block->parent = loop;
+
+ struct vtn_block *continue_block = vtn_block(b, block->merge[2]);
+ loop->continue_pos = continue_block->pos;
+
+ if (!vtn_is_single_block_loop(loop)) {
+ struct vtn_construct *cont = rzalloc(b, struct vtn_construct);
+ cont->type = vtn_construct_type_continue;
+ cont->parent = loop;
+ cont->start_pos = loop->continue_pos;
+ cont->end_pos = end_pos;
+
+ cont->parent = loop;
+ continue_block->parent = cont;
+ }
+
+ /* Not all combinations of OpLoopMerge and OpBranchConditional are valid,
+ * workaround for invalid combinations by injecting an extra selection.
+ *
+ * Old versions of dxil-spirv generated this.
+ */
+ if (branch_op == SpvOpBranchConditional) {
+ vtn_assert(block->successors_count == 2);
+ const unsigned then_pos = block->successors[0].block ?
+ block->successors[0].block->pos : 0;
+ const unsigned else_pos = block->successors[1].block ?
+ block->successors[1].block->pos : 0;
+
+ if (then_pos > loop->start_pos && then_pos < loop->continue_pos &&
+ else_pos > loop->start_pos && else_pos < loop->continue_pos) {
+ vtn_warn("An OpSelectionMerge instruction is required to precede "
+ "an OpBranchConditional instruction that has different "
+ "True Label and False Label operands where neither are "
+ "declared merge blocks or Continue Targets.");
+ struct vtn_construct *sel = rzalloc(b, struct vtn_construct);
+ sel->type = vtn_construct_type_selection;
+ sel->start_pos = loop->start_pos;
+ sel->end_pos = loop->continue_pos;
+ sel->then_pos = then_pos;
+ sel->else_pos = else_pos;
+ sel->parent = loop;
+ block->parent = sel;
+ }
+ }
+
+ } else if (branch_op == SpvOpSwitch) {
+ vtn_assert(merge_op == SpvOpSelectionMerge);
+
+ struct vtn_construct *swtch = rzalloc(b, struct vtn_construct);
+ swtch->type = vtn_construct_type_switch;
+ swtch->start_pos = block->pos;
+ swtch->end_pos = end_pos;
+
+ swtch->parent = block->parent;
+ block->parent = swtch;
+
+ struct list_head cases;
+ list_inithead(&cases);
+ vtn_parse_switch(b, block->branch, &cases);
+
+ vtn_foreach_case_safe(cse, &cases) {
+ if (cse->block->pos < end_pos) {
+ struct vtn_block *case_block = cse->block;
+ struct vtn_construct *c = rzalloc(b, struct vtn_construct);
+ c->type = vtn_construct_type_case;
+ c->parent = swtch;
+ c->start_pos = case_block->pos;
+
+ /* Upper bound, will be updated right after. */
+ c->end_pos = swtch->end_pos;
+
+ vtn_assert(case_block->parent == NULL || case_block->parent == swtch);
+ case_block->parent = c;
+ } else {
+ /* A target in OpSwitch must point either to one of the case
+ * constructs or to the Merge block. No outer break/continue
+ * is allowed.
+ */
+ vtn_assert(cse->block->pos == end_pos);
+ }
+ list_delinit(&cse->link);
+ }
+
+ /* Case constructs don't overlap, so they end as the next one
+ * begins.
+ */
+ qsort(block->successors, block->successors_count,
+ sizeof(struct vtn_successor), cmp_succ_block_pos);
+ for (unsigned succ_idx = 1; succ_idx < block->successors_count; succ_idx++) {
+ unsigned succ_pos = block->successors[succ_idx].block->pos;
+ /* The successors are ordered, so once we see a successor point
+ * to the merge block, we are done fixing the cases.
+ */
+ if (succ_pos >= swtch->end_pos)
+ break;
+ struct vtn_construct *prev_cse =
+ vtn_find_innermost(vtn_construct_type_case,
+ block->successors[succ_idx - 1].block->parent);
+ vtn_assert(prev_cse);
+ prev_cse->end_pos = succ_pos;
+ }
+
+ } else {
+ vtn_assert(merge_op == SpvOpSelectionMerge);
+ vtn_assert(branch_op == SpvOpBranchConditional);
+
+ struct vtn_construct *sel = rzalloc(b, struct vtn_construct);
+ sel->type = vtn_construct_type_selection;
+ sel->start_pos = block->pos;
+ sel->end_pos = end_pos;
+ sel->parent = block->parent;
+ block->parent = sel;
+
+ vtn_assert(block->successors_count == 2);
+ struct vtn_block *then_block = block->successors[0].block;
+ struct vtn_block *else_block = block->successors[1].block;
+
+ sel->then_pos = then_block ? then_block->pos : 0;
+ sel->else_pos = else_block ? else_block->pos : 0;
+ }
+ }
+ }
+
+ /* Link the constructs with their parents and with the remaining blocks
+ * that do not start one. This will also build the ordered list of
+ * constructs.
+ */
+ struct vtn_construct_stack stack;
+ init_construct_stack(&stack, b);
+ push_construct(&stack, func_construct);
+ list_addtail(&func_construct->link, &b->func->constructs);
+
+ for (unsigned i = 0; i < b->func->ordered_blocks_count; i++) {
+ struct vtn_block *block = b->func->ordered_blocks[i];
+
+ while (block->pos == top_construct(&stack)->end_pos)
+ pop_construct(&stack);
+
+ /* Identify the start of a continue construct. */
+ if (top_construct(&stack)->type == vtn_construct_type_loop &&
+ !vtn_is_single_block_loop(top_construct(&stack)) &&
+ top_construct(&stack)->continue_pos == block->pos) {
+ struct vtn_construct *c = vtn_find_innermost(vtn_construct_type_continue, block->parent);
+ vtn_assert(c);
+ vtn_assert(c->parent == top_construct(&stack));
+
+ list_addtail(&c->link, &b->func->constructs);
+ push_construct(&stack, c);
+ }
+
+ if (top_construct(&stack)->type == vtn_construct_type_switch) {
+ struct vtn_block *header = b->func->ordered_blocks[top_construct(&stack)->start_pos];
+ for (unsigned succ_idx = 0; succ_idx < header->successors_count; succ_idx++) {
+ struct vtn_successor *succ = &header->successors[succ_idx];
+ if (block == succ->block) {
+ struct vtn_construct *c = vtn_find_innermost(vtn_construct_type_case, succ->block->parent);
+ if (c) {
+ vtn_assert(c->parent == top_construct(&stack));
+
+ list_addtail(&c->link, &b->func->constructs);
+ push_construct(&stack, c);
+ }
+ break;
+ }
+ }
+ }
+
+ if (block->merge) {
+ switch (block->merge[0] & SpvOpCodeMask) {
+ case SpvOpSelectionMerge: {
+ struct vtn_construct *c = block->parent;
+ vtn_assert(c->type == vtn_construct_type_selection ||
+ c->type == vtn_construct_type_switch);
+
+ c->parent = top_construct(&stack);
+
+ list_addtail(&c->link, &b->func->constructs);
+ push_construct(&stack, c);
+ break;
+ }
+
+ case SpvOpLoopMerge: {
+ struct vtn_construct *c = block->parent;
+ struct vtn_construct *loop = c;
+
+ /* A loop might have an extra selection injected, skip it. */
+ if (c->type == vtn_construct_type_selection)
+ loop = c->parent;
+
+ vtn_assert(loop->type == vtn_construct_type_loop);
+ loop->parent = top_construct(&stack);
+
+ list_addtail(&loop->link, &b->func->constructs);
+ push_construct(&stack, loop);
+
+ if (loop != c) {
+ /* Make sure we also "enter" the extra construct. */
+ list_addtail(&c->link, &b->func->constructs);
+ push_construct(&stack, c);
+ }
+ break;
+ }
+
+ default:
+ unreachable("invalid merge opcode");
+ }
+ }
+
+ block->parent = top_construct(&stack);
+ }
+
+ vtn_assert(count_construct_stack(&stack) == 1);
+ vtn_assert(top_construct(&stack)->type == vtn_construct_type_function);
+
+ unsigned index = 0;
+ list_for_each_entry(struct vtn_construct, c, &b->func->constructs, link)
+ c->index = index++;
+}
+
+static void
+validate_constructs(struct vtn_builder *b)
+{
+ list_for_each_entry(struct vtn_construct, c, &b->func->constructs, link) {
+ if (c->type == vtn_construct_type_function)
+ vtn_assert(c->parent == NULL);
+ else
+ vtn_assert(c->parent);
+
+ switch (c->type) {
+ case vtn_construct_type_continue:
+ vtn_assert(c->parent->type == vtn_construct_type_loop);
+ break;
+ case vtn_construct_type_case:
+ vtn_assert(c->parent->type == vtn_construct_type_switch);
+ break;
+ default:
+ /* Nothing to do. */
+ break;
+ }
+ }
+}
+
+static void
+find_innermost_constructs(struct vtn_builder *b)
+{
+ list_for_each_entry(struct vtn_construct, c, &b->func->constructs, link) {
+ if (c->type == vtn_construct_type_function) {
+ c->innermost_loop = NULL;
+ c->innermost_switch = NULL;
+ c->innermost_case = NULL;
+ continue;
+ }
+
+ if (c->type == vtn_construct_type_loop)
+ c->innermost_loop = c;
+ else
+ c->innermost_loop = c->parent->innermost_loop;
+
+ if (c->type == vtn_construct_type_switch)
+ c->innermost_switch = c;
+ else
+ c->innermost_switch = c->parent->innermost_switch;
+
+ if (c->type == vtn_construct_type_case)
+ c->innermost_case = c;
+ else
+ c->innermost_case = c->parent->innermost_case;
+ }
+
+ list_for_each_entry(struct vtn_construct, c, &b->func->constructs, link) {
+ vtn_assert(vtn_find_innermost(vtn_construct_type_loop, c) == c->innermost_loop);
+ vtn_assert(vtn_find_innermost(vtn_construct_type_switch, c) == c->innermost_switch);
+ vtn_assert(vtn_find_innermost(vtn_construct_type_case, c) == c->innermost_case);
+ }
+}
+
+static void
+set_needs_continue_propagation(struct vtn_construct *c)
+{
+ for (; c != c->innermost_loop; c = c->parent)
+ c->needs_continue_propagation = true;
+}
+
+static void
+set_needs_break_propagation(struct vtn_construct *c,
+ struct vtn_construct *to_break)
+{
+ for (; c != to_break; c = c->parent)
+ c->needs_break_propagation = true;
+}
+
+static enum vtn_branch_type
+branch_type_for_successor(struct vtn_builder *b, struct vtn_block *block,
+ struct vtn_successor *succ)
+{
+ unsigned pos = block->pos;
+ unsigned succ_pos = succ->block->pos;
+
+ struct vtn_construct *inner = block->parent;
+ vtn_assert(inner);
+
+ /* Identify the types of branches, applying the "Rules for Structured
+ * Control-flow Constructs" from SPIR-V spec.
+ */
+
+ struct vtn_construct *innermost_loop = inner->innermost_loop;
+ if (innermost_loop) {
+ /* Entering the innermost loop’s continue construct. */
+ if (!vtn_is_single_block_loop(innermost_loop) &&
+ succ_pos == innermost_loop->continue_pos) {
+ set_needs_continue_propagation(inner);
+ return vtn_branch_type_loop_continue;
+ }
+
+ /* Breaking from the innermost loop (and branching from back-edge block
+ * to loop merge).
+ */
+ if (succ_pos == innermost_loop->end_pos) {
+ set_needs_break_propagation(inner, innermost_loop);
+ return vtn_branch_type_loop_break;
+ }
+
+ /* Next loop iteration. There can be only a single loop back-edge
+ * for each loop construct.
+ */
+ if (succ_pos == innermost_loop->start_pos) {
+ vtn_assert(inner->type == vtn_construct_type_continue ||
+ vtn_is_single_block_loop(innermost_loop));
+ return vtn_branch_type_loop_back_edge;
+ }
+ }
+
+ struct vtn_construct *innermost_switch = inner->innermost_switch;
+ if (innermost_switch) {
+ struct vtn_construct *innermost_cse = inner->innermost_case;
+
+ /* Breaking from the innermost switch construct. */
+ if (succ_pos == innermost_switch->end_pos) {
+ /* Use a nloop if this is not a natural exit from a case construct. */
+ if (innermost_cse && pos != innermost_cse->end_pos - 1) {
+ innermost_cse->needs_nloop = true;
+ set_needs_break_propagation(inner, innermost_cse);
+ }
+ return vtn_branch_type_switch_break;
+ }
+
+ /* Branching from one case construct to another. */
+ if (inner != innermost_switch) {
+ vtn_assert(innermost_cse);
+ vtn_assert(innermost_cse->parent == innermost_switch);
+
+ if (succ->block->switch_case) {
+ /* Both cases should be from the same Switch construct. */
+ struct vtn_construct *target_cse = succ->block->parent->innermost_case;
+ vtn_assert(target_cse->parent == innermost_switch);
+ target_cse->needs_fallthrough = true;
+ return vtn_branch_type_switch_fallthrough;
+ }
+ }
+ }
+
+ if (inner->type == vtn_construct_type_selection) {
+ /* Branches from the header block that were not categorized above will
+ * follow to the then/else paths or to the merge block, and are handled
+ * by the nir_if node.
+ */
+ if (block->merge)
+ return vtn_branch_type_forward;
+
+ /* Breaking from a selection construct. */
+ if (succ_pos == inner->end_pos) {
+ /* Identify cases where the break would be a natural flow in the NIR
+ * construct. We don't need the extra loop in such cases.
+ *
+ * Because then/else are not ordered, we need to find which one happens
+ * later. For non early merges, the branch from the block right before
+ * the second side of the if starts will also jumps naturally to the
+ * end of the if.
+ */
+ const bool has_early_merge = inner->merge_pos != inner->end_pos;
+ const unsigned second_pos = MAX2(inner->then_pos, inner->else_pos);
+
+ const bool natural_exit_from_if =
+ pos + 1 == inner->end_pos ||
+ (!has_early_merge && (pos + 1 == second_pos));
+
+ inner->needs_nloop = !natural_exit_from_if;
+ return vtn_branch_type_if_break;
+ }
+ }
+
+ if (succ_pos < inner->end_pos)
+ return vtn_branch_type_forward;
+
+ const enum nir_spirv_debug_level level = NIR_SPIRV_DEBUG_LEVEL_ERROR;
+ const size_t offset = 0;
+
+ vtn_logf(b, level, offset,
+ "SPIR-V parsing FAILED:\n"
+ " Unrecognized branch from block pos %u (id=%u) "
+ "to block pos %u (id=%u)",
+ block->pos, block->label[1],
+ succ->block->pos, succ->block->label[1]);
+
+ vtn_logf(b, level, offset,
+ " Inner construct '%s': %u -> %u (merge=%u then=%u else=%u)",
+ vtn_construct_type_to_string(inner->type),
+ inner->start_pos, inner->end_pos, inner->merge_pos, inner->then_pos, inner->else_pos);
+
+ struct vtn_construct *outer = inner->parent;
+ if (outer) {
+ vtn_logf(b, level, offset,
+ " Outer construct '%s': %u -> %u (merge=%u then=%u else=%u)",
+ vtn_construct_type_to_string(outer->type),
+ outer->start_pos, outer->end_pos, outer->merge_pos, outer->then_pos, outer->else_pos);
+ }
+
+ vtn_fail("Unable to identify branch type");
+ return vtn_branch_type_none;
+}
+
+static enum vtn_branch_type
+branch_type_for_terminator(struct vtn_builder *b, struct vtn_block *block)
+{
+ vtn_assert(block->successors_count == 1);
+ vtn_assert(block->successors[0].block == NULL);
+
+ switch (block->branch[0] & SpvOpCodeMask) {
+ case SpvOpKill:
+ return vtn_branch_type_discard;
+ case SpvOpTerminateInvocation:
+ return vtn_branch_type_terminate_invocation;
+ case SpvOpIgnoreIntersectionKHR:
+ return vtn_branch_type_ignore_intersection;
+ case SpvOpTerminateRayKHR:
+ return vtn_branch_type_terminate_ray;
+ case SpvOpEmitMeshTasksEXT:
+ return vtn_branch_type_emit_mesh_tasks;
+ case SpvOpReturn:
+ case SpvOpReturnValue:
+ case SpvOpUnreachable:
+ return vtn_branch_type_return;
+ default:
+ unreachable("unexpected terminator operation");
+ return vtn_branch_type_none;
+ }
+}
+
+static void
+set_branch_types(struct vtn_builder *b)
+{
+ for (unsigned i = 0; i < b->func->ordered_blocks_count; i++) {
+ struct vtn_block *block = b->func->ordered_blocks[i];
+ for (unsigned j = 0; j < block->successors_count; j++) {
+ struct vtn_successor *succ = &block->successors[j];
+
+ if (succ->block)
+ succ->branch_type = branch_type_for_successor(b, block, succ);
+ else
+ succ->branch_type = branch_type_for_terminator(b, block);
+
+ vtn_assert(succ->branch_type != vtn_branch_type_none);
+ }
+ }
+}
+
+static void
+find_merge_pos(struct vtn_builder *b)
+{
+ /* Merges are at the end of the construct by construction... */
+ list_for_each_entry(struct vtn_construct, c, &b->func->constructs, link)
+ c->merge_pos = c->end_pos;
+
+ /* ...except when we have an "early merge", i.e. a branch that converges
+ * before the declared merge point. For these cases the actual merge is
+ * stored in merge_pos.
+ *
+ * Look at all header blocks for constructs that may have such early
+ * merge, and check whether they fit
+ */
+ for (unsigned i = 0; i < b->func->ordered_blocks_count; i++) {
+ if (!b->func->ordered_blocks[i]->merge)
+ continue;
+
+ struct vtn_block *header = b->func->ordered_blocks[i];
+ if (header->successors_count != 2)
+ continue;
+
+ /* Ignore single-block loops (i.e. header thats in a continue
+ * construct). Because the loop has no body, no block will
+ * be identified in the then/else sides, the vtn_emit_branch
+ * calls will be enough.
+ */
+
+ struct vtn_construct *c = header->parent;
+ if (c->type != vtn_construct_type_selection)
+ continue;
+
+ const unsigned first_pos = MIN2(c->then_pos, c->else_pos);
+ const unsigned second_pos = MAX2(c->then_pos, c->else_pos);
+
+ /* The first side ends where the second starts. The second side ends
+ * either the continue position (that is guaranteed to appear after the
+ * body of a loop) or the actual end of the construct.
+ *
+ * Because of the way we ordered the blocks, if there's an early merge,
+ * the first side of the if will have a branch inside the second side.
+ */
+ const unsigned first_end = second_pos;
+ const unsigned second_end = c->end_pos;
+
+ unsigned early_merge_pos = 0;
+ for (unsigned pos = first_pos; pos < first_end; pos++) {
+ /* For each block in first... */
+ struct vtn_block *block = b->func->ordered_blocks[pos];
+ for (unsigned s = 0; s < block->successors_count; s++) {
+ if (block->successors[s].block) {
+ /* ...see if one of its successors branches to the second side. */
+ const unsigned succ_pos = block->successors[s].block->pos;
+ if (succ_pos >= second_pos && succ_pos < second_end) {
+ vtn_fail_if(early_merge_pos,
+ "A single selection construct cannot "
+ "have multiple early merges");
+ early_merge_pos = succ_pos;
+ }
+ }
+ }
+
+ if (early_merge_pos) {
+ c->merge_pos = early_merge_pos;
+ break;
+ }
+ }
+ }
+}
+
+void
+vtn_build_structured_cfg(struct vtn_builder *b, const uint32_t *words, const uint32_t *end)
+{
+ vtn_foreach_function(func, &b->functions) {
+ b->func = func;
+
+ sort_blocks(b);
+
+ create_constructs(b);
+
+ validate_constructs(b);
+
+ find_innermost_constructs(b);
+
+ find_merge_pos(b);
+
+ set_branch_types(b);
+
+ if (MESA_SPIRV_DEBUG(STRUCTURED)) {
+ printf("\nBLOCKS (%u):\n", func->ordered_blocks_count);
+ print_ordered_blocks(func);
+ printf("\nCONSTRUCTS (%u):\n", list_length(&func->constructs));
+ print_constructs(func);
+ printf("\n");
+ }
+ }
+}
+
+static int
+vtn_set_break_vars_between(struct vtn_builder *b,
+ struct vtn_construct *from,
+ struct vtn_construct *to)
+{
+ vtn_assert(from);
+ vtn_assert(to);
+
+ int count = 0;
+ for (struct vtn_construct *c = from; c != to; c = c->parent) {
+ if (c->break_var) {
+ vtn_assert(c->nloop);
+ count++;
+
+ /* There's no need to set break_var for the from block an actual break will be emitted
+ * by the callers.
+ */
+ if (c != from)
+ nir_store_var(&b->nb, c->break_var, nir_imm_true(&b->nb), 1);
+ } else {
+ /* There's a 1:1 correspondence between break_vars and nloops. */
+ vtn_assert(!c->nloop);
+ }
+ }
+
+ return count;
+}
+
+static void
+vtn_emit_break_for_construct(struct vtn_builder *b,
+ const struct vtn_block *block,
+ struct vtn_construct *to_break)
+{
+ vtn_assert(to_break);
+ vtn_assert(to_break->nloop);
+
+ bool has_intermediate = vtn_set_break_vars_between(b, block->parent, to_break);
+ if (has_intermediate)
+ nir_store_var(&b->nb, to_break->break_var, nir_imm_true(&b->nb), 1);
+
+ nir_jump(&b->nb, nir_jump_break);
+}
+
+static void
+vtn_emit_continue_for_construct(struct vtn_builder *b,
+ const struct vtn_block *block,
+ struct vtn_construct *to_continue)
+{
+ vtn_assert(to_continue);
+ vtn_assert(to_continue->type == vtn_construct_type_loop);
+ vtn_assert(to_continue->nloop);
+
+ bool has_intermediate = vtn_set_break_vars_between(b, block->parent, to_continue);
+ if (has_intermediate) {
+ nir_store_var(&b->nb, to_continue->continue_var, nir_imm_true(&b->nb), 1);
+ nir_jump(&b->nb, nir_jump_break);
+ } else {
+ nir_jump(&b->nb, nir_jump_continue);
+ }
+}
+
+static void
+vtn_emit_branch(struct vtn_builder *b, const struct vtn_block *block,
+ const struct vtn_successor *succ)
+{
+ switch (succ->branch_type) {
+ case vtn_branch_type_none:
+ vtn_assert(!"invalid branch type");
+ break;
+
+ case vtn_branch_type_forward:
+ /* Nothing to do. */
+ break;
+
+ case vtn_branch_type_if_break: {
+ struct vtn_construct *inner_if = block->parent;
+ vtn_assert(inner_if->type == vtn_construct_type_selection);
+ if (inner_if->nloop) {
+ vtn_emit_break_for_construct(b, block, inner_if);
+ } else {
+ /* Nothing to do. This is a natural exit from an if construct. */
+ }
+ break;
+ }
+
+ case vtn_branch_type_switch_break: {
+ struct vtn_construct *swtch = block->parent->innermost_switch;
+ vtn_assert(swtch);
+
+ struct vtn_construct *cse = block->parent->innermost_case;
+ if (cse && cse->parent == swtch && cse->nloop) {
+ vtn_emit_break_for_construct(b, block, cse);
+ } else {
+ /* Nothing to do. This case doesn't have a loop, so this is a
+ * natural break from a case.
+ */
+ }
+ break;
+ }
+
+ case vtn_branch_type_switch_fallthrough: {
+ struct vtn_construct *cse = block->parent->innermost_case;
+ vtn_assert(cse);
+
+ struct vtn_construct *swtch = cse->parent;
+ vtn_assert(swtch->type == vtn_construct_type_switch);
+
+ /* Successor is the start of another case construct with the same parent
+ * switch construct.
+ */
+ vtn_assert(succ->block->switch_case != NULL);
+ struct vtn_construct *target = succ->block->parent->innermost_case;
+ vtn_assert(target != NULL && target->type == vtn_construct_type_case);
+ vtn_assert(target->parent == swtch);
+ vtn_assert(target->fallthrough_var);
+
+ nir_store_var(&b->nb, target->fallthrough_var, nir_imm_true(&b->nb), 1);
+ if (cse->nloop)
+ vtn_emit_break_for_construct(b, block, cse);
+ break;
+ }
+
+ case vtn_branch_type_loop_break: {
+ struct vtn_construct *loop = block->parent->innermost_loop;
+ vtn_assert(loop);
+ vtn_emit_break_for_construct(b, block, loop);
+ break;
+ }
+
+ case vtn_branch_type_loop_continue: {
+ struct vtn_construct *loop = block->parent->innermost_loop;
+ vtn_assert(loop);
+ vtn_emit_continue_for_construct(b, block, loop);
+ break;
+ }
+
+ case vtn_branch_type_loop_back_edge:
+ /* Nothing to do: naturally handled by NIR loop node. */
+ break;
+
+ case vtn_branch_type_return:
+ vtn_assert(block);
+ vtn_emit_ret_store(b, block);
+ nir_jump(&b->nb, nir_jump_return);
+ break;
+
+ case vtn_branch_type_discard:
+ if (b->convert_discard_to_demote)
+ nir_demote(&b->nb);
+ else
+ nir_discard(&b->nb);
+ break;
+
+ case vtn_branch_type_terminate_invocation:
+ nir_terminate(&b->nb);
+ break;
+
+ case vtn_branch_type_ignore_intersection:
+ nir_ignore_ray_intersection(&b->nb);
+ nir_jump(&b->nb, nir_jump_halt);
+ break;
+
+ case vtn_branch_type_terminate_ray:
+ nir_terminate_ray(&b->nb);
+ nir_jump(&b->nb, nir_jump_halt);
+ break;
+
+ case vtn_branch_type_emit_mesh_tasks: {
+ vtn_assert(block);
+ vtn_assert(block->branch);
+
+ const uint32_t *w = block->branch;
+ vtn_assert((w[0] & SpvOpCodeMask) == SpvOpEmitMeshTasksEXT);
+
+ /* Launches mesh shader workgroups from the task shader.
+ * Arguments are: vec(x, y, z), payload pointer
+ */
+ nir_ssa_def *dimensions =
+ nir_vec3(&b->nb, vtn_get_nir_ssa(b, w[1]),
+ vtn_get_nir_ssa(b, w[2]),
+ vtn_get_nir_ssa(b, w[3]));
+
+ /* The payload variable is optional.
+ * We don't have a NULL deref in NIR, so just emit the explicit
+ * intrinsic when there is no payload.
+ */
+ const unsigned count = w[0] >> SpvWordCountShift;
+ if (count == 4)
+ nir_launch_mesh_workgroups(&b->nb, dimensions);
+ else if (count == 5)
+ nir_launch_mesh_workgroups_with_payload_deref(&b->nb, dimensions,
+ vtn_get_nir_ssa(b, w[4]));
+ else
+ vtn_fail("Invalid EmitMeshTasksEXT.");
+
+ nir_jump(&b->nb, nir_jump_halt);
+ break;
+ }
+
+ default:
+ vtn_fail("Invalid branch type");
+ }
+}
+
+static nir_selection_control
+vtn_selection_control(struct vtn_builder *b, SpvSelectionControlMask control)
+{
+ if (control == SpvSelectionControlMaskNone)
+ return nir_selection_control_none;
+ else if (control & SpvSelectionControlDontFlattenMask)
+ return nir_selection_control_dont_flatten;
+ else if (control & SpvSelectionControlFlattenMask)
+ return nir_selection_control_flatten;
+ else
+ vtn_fail("Invalid selection control");
+}
+
+static void
+vtn_emit_block(struct vtn_builder *b, struct vtn_block *block,
+ vtn_instruction_handler handler)
+{
+ const uint32_t *block_start = block->label;
+ const uint32_t *block_end = block->merge ? block->merge :
+ block->branch;
+
+ block_start = vtn_foreach_instruction(b, block_start, block_end,
+ vtn_handle_phis_first_pass);
+
+ vtn_foreach_instruction(b, block_start, block_end, handler);
+
+ block->end_nop = nir_nop(&b->nb);
+
+ if (block->parent->type == vtn_construct_type_switch) {
+ /* Switch is handled as a sequence of NIR if for each of the cases. */
+
+ } else if (block->successors_count == 1) {
+ vtn_assert(block->successors[0].branch_type != vtn_branch_type_none);
+ vtn_emit_branch(b, block, &block->successors[0]);
+
+ } else if (block->successors_count == 2) {
+ struct vtn_successor *then_succ = &block->successors[0];
+ struct vtn_successor *else_succ = &block->successors[1];
+ struct vtn_construct *c = block->parent;
+
+ nir_ssa_def *cond = vtn_get_nir_ssa(b, block->branch[1]);
+ if (then_succ->block == else_succ->block)
+ cond = nir_imm_true(&b->nb);
+
+ /* The branches will already be emitted here, so for paths that
+ * doesn't have blocks inside the construct, e.g. that are an
+ * exit from the construct, nothing else is needed.
+ */
+ nir_if *sel = nir_push_if(&b->nb, cond);
+ vtn_emit_branch(b, block, then_succ);
+ if (then_succ->block != else_succ->block) {
+ nir_push_else(&b->nb, NULL);
+ vtn_emit_branch(b, block, else_succ);
+ }
+ nir_pop_if(&b->nb, NULL);
+
+ if (c->type == vtn_construct_type_selection &&
+ block->pos == c->start_pos) {
+ /* This is the start of a selection construct. Record the nir_if in
+ * the construct so we can close it properly and handle the then and
+ * else cases in block iteration.
+ */
+ vtn_assert(c->nif == NULL);
+ c->nif = sel;
+
+ vtn_assert(block->merge != NULL);
+
+ SpvOp merge_op = block->merge[0] & SpvOpCodeMask;
+ if (merge_op == SpvOpSelectionMerge)
+ sel->control = vtn_selection_control(b, block->merge[2]);
+
+ /* In most cases, vtn_emit_cf_func_structured() will place the cursor
+ * in the correct side of the nir_if. However, in the case where the
+ * selection construct is empty, we need to ensure that the cursor is
+ * at least inside the nir_if or NIR will assert when we try to close
+ * it with nir_pop_if().
+ */
+ b->nb.cursor = nir_before_cf_list(&sel->then_list);
+ } else {
+ vtn_fail_if(then_succ->branch_type == vtn_branch_type_forward &&
+ else_succ->branch_type == vtn_branch_type_forward &&
+ then_succ->block != else_succ->block,
+ "An OpSelectionMerge instruction is required to precede "
+ "an OpBranchConditional instruction that has different "
+ "True Label and False Label operands where neither are "
+ "declared merge blocks or Continue Targets.");
+
+ if (then_succ->branch_type == vtn_branch_type_forward) {
+ b->nb.cursor = nir_before_cf_list(&sel->then_list);
+ } else if (else_succ->branch_type == vtn_branch_type_forward) {
+ b->nb.cursor = nir_before_cf_list(&sel->else_list);
+ } else {
+ /* Leave it alone */
+ }
+ }
+ }
+}
+
+static nir_ssa_def *
+vtn_switch_case_condition(struct vtn_builder *b, struct vtn_construct *swtch,
+ nir_ssa_def *sel, struct vtn_case *cse)
+{
+ vtn_assert(swtch->type == vtn_construct_type_switch);
+
+ if (cse->is_default) {
+ nir_ssa_def *any = nir_imm_false(&b->nb);
+
+ struct vtn_block *header = b->func->ordered_blocks[swtch->start_pos];
+
+ for (unsigned j = 0; j < header->successors_count; j++) {
+ struct vtn_successor *succ = &header->successors[j];
+ struct vtn_case *other = succ->block->switch_case;
+
+ if (other->is_default)
+ continue;
+ any = nir_ior(&b->nb, any,
+ vtn_switch_case_condition(b, swtch, sel, other));
+ }
+
+ return nir_inot(&b->nb, any);
+ } else {
+ nir_ssa_def *cond = nir_imm_false(&b->nb);
+ util_dynarray_foreach(&cse->values, uint64_t, val)
+ cond = nir_ior(&b->nb, cond, nir_ieq_imm(&b->nb, sel, *val));
+ return cond;
+ }
+}
+
+static nir_loop_control
+vtn_loop_control(struct vtn_builder *b, SpvLoopControlMask control)
+{
+ if (control == SpvLoopControlMaskNone)
+ return nir_loop_control_none;
+ else if (control & SpvLoopControlDontUnrollMask)
+ return nir_loop_control_dont_unroll;
+ else if (control & SpvLoopControlUnrollMask)
+ return nir_loop_control_unroll;
+ else if ((control & SpvLoopControlDependencyInfiniteMask) ||
+ (control & SpvLoopControlDependencyLengthMask) ||
+ (control & SpvLoopControlMinIterationsMask) ||
+ (control & SpvLoopControlMaxIterationsMask) ||
+ (control & SpvLoopControlIterationMultipleMask) ||
+ (control & SpvLoopControlPeelCountMask) ||
+ (control & SpvLoopControlPartialCountMask)) {
+ /* We do not do anything special with these yet. */
+ return nir_loop_control_none;
+ } else {
+ vtn_fail("Invalid loop control");
+ }
+}
+
+static void
+vtn_emit_control_flow_propagation(struct vtn_builder *b,
+ struct vtn_construct *top)
+{
+ if (top->type == vtn_construct_type_function ||
+ top->type == vtn_construct_type_continue ||
+ top->type == vtn_construct_type_switch)
+ return;
+
+ /* Find the innermost parent with a NIR loop. */
+ struct vtn_construct *parent_with_nloop = NULL;
+ for (struct vtn_construct *c = top->parent; c; c = c->parent) {
+ if (c->nloop) {
+ parent_with_nloop = c;
+ break;
+ }
+ }
+ if (parent_with_nloop == NULL)
+ return;
+
+ /* If there's another nloop in the parent chain, decide whether we need
+ * to emit conditional continue/break after top construct is closed.
+ */
+
+ if (top->needs_continue_propagation &&
+ parent_with_nloop == top->innermost_loop) {
+ struct vtn_construct *loop = top->innermost_loop;
+ vtn_assert(loop);
+ vtn_assert(loop != top);
+
+ nir_push_if(&b->nb, nir_load_var(&b->nb, loop->continue_var));
+ nir_jump(&b->nb, nir_jump_continue);
+ nir_pop_if(&b->nb, NULL);
+ }
+
+ if (top->needs_break_propagation) {
+ vtn_assert(parent_with_nloop->break_var);
+ nir_push_if(&b->nb, nir_load_var(&b->nb, parent_with_nloop->break_var));
+ nir_jump(&b->nb, nir_jump_break);
+ nir_pop_if(&b->nb, NULL);
+ }
+}
+
+static inline nir_variable *
+vtn_create_local_bool(struct vtn_builder *b, const char *name)
+{
+ return nir_local_variable_create(b->nb.impl, glsl_bool_type(), name);
+}
+
+void
+vtn_emit_cf_func_structured(struct vtn_builder *b, struct vtn_function *func,
+ vtn_instruction_handler handler)
+{
+ struct vtn_construct *current =
+ list_first_entry(&func->constructs, struct vtn_construct, link);
+ vtn_assert(current->type == vtn_construct_type_function);
+
+ /* Walk the blocks in order keeping track of the constructs that started
+ * but haven't ended yet. When constructs start and end, add extra code to
+ * setup the NIR control flow (different for each construct), also add
+ * extra code for propagating certain branch types.
+ */
+
+ struct vtn_construct_stack stack;
+ init_construct_stack(&stack, b);
+ push_construct(&stack, current);
+
+ for (unsigned i = 0; i < func->ordered_blocks_count; i++) {
+ struct vtn_block *block = func->ordered_blocks[i];
+ struct vtn_construct *top = top_construct(&stack);
+
+ /* Close out any past constructs and make sure the cursor is at the
+ * right place to start this block. For each block, there are three
+ * cases we care about here:
+ *
+ * 1. It is the block at the end (in our reverse structured post-order
+ * traversal) of one or more constructs and closes them.
+ *
+ * 2. It is an early merge of a selection construct.
+ *
+ * 3. It is the start of the then or else case of a selection construct
+ * and we may have previously been emitting code in the other side.
+ */
+
+ /* Close (or early merge) any constructs that end at this block. */
+ bool merged_any_constructs = false;
+ while (top->end_pos == block->pos || top->merge_pos == block->pos) {
+ merged_any_constructs = true;
+ if (top->nif) {
+ const bool has_early_merge = top->merge_pos != top->end_pos;
+
+ if (!has_early_merge) {
+ nir_pop_if(&b->nb, top->nif);
+ } else if (block->pos == top->merge_pos) {
+ /* This is an early merge. */
+
+ nir_pop_if(&b->nb, top->nif);
+
+ /* The extra dummy "if (true)" for the merged part avoids
+ * generating multiple jumps in sequence and upsetting
+ * NIR rules. We'll pop it in the case below when we reach
+ * the end_pos block.
+ */
+ nir_push_if(&b->nb, nir_imm_true(&b->nb));
+
+ /* Stop since this construct still has more blocks. */
+ break;
+ } else {
+ /* Pop the dummy if added for the blocks after the early merge. */
+ vtn_assert(block->pos == top->end_pos);
+ nir_pop_if(&b->nb, NULL);
+ }
+ }
+
+ if (top->nloop) {
+ /* For constructs that are not SPIR-V loop, a NIR loop may be used
+ * to provide richer control flow. So we add a nir break to cause
+ * the loop stop at the first iteration, unless there's already a
+ * jump at the end of the last block.
+ */
+ if (top->type != vtn_construct_type_loop) {
+ nir_block *last = nir_loop_last_block(top->nloop);
+ if (!nir_block_ends_in_jump(last)) {
+ b->nb.cursor = nir_after_block(last);
+ nir_jump(&b->nb, nir_jump_break);
+ }
+ }
+
+ nir_pop_loop(&b->nb, top->nloop);
+ }
+
+ vtn_emit_control_flow_propagation(b, top);
+
+ pop_construct(&stack);
+ top = top_construct(&stack);
+ }
+
+ /* We are fully inside the current top. */
+ vtn_assert(block->pos < top->end_pos);
+
+ /* Move the cursor to the right side of a selection construct.
+ *
+ * If we merged any constructs, we don't need to move because
+ * either: this is an early merge and we already set the cursor above;
+ * or a construct ended, and this is a 'merge block' for that
+ * construct, so it can't also be a 'Target' for an outer conditional.
+ */
+ if (!merged_any_constructs && top->type == vtn_construct_type_selection &&
+ (block->pos == top->then_pos || block->pos == top->else_pos)) {
+ vtn_assert(top->nif);
+
+ struct vtn_block *header = func->ordered_blocks[top->start_pos];
+ vtn_assert(header->successors_count == 2);
+
+ if (block->pos == top->then_pos)
+ b->nb.cursor = nir_before_cf_list(&top->nif->then_list);
+ else
+ b->nb.cursor = nir_before_cf_list(&top->nif->else_list);
+ }
+
+ /* Open any constructs which start at this block.
+ *
+ * Constructs which are designated by Op*Merge are considered to start
+ * at the block which contains the merge instruction. This means that
+ * loops constructs start at the first block inside the loop while
+ * selection and switch constructs start at the block containing the
+ * OpBranchConditional or OpSwitch.
+ */
+ while (current->link.next != &func->constructs) {
+ struct vtn_construct *next =
+ list_entry(current->link.next, struct vtn_construct, link);
+
+ /* Stop once we find a construct that doesn't start in this block. */
+ if (next->start_pos != block->pos)
+ break;
+
+ switch (next->type) {
+ case vtn_construct_type_function:
+ unreachable("should've already entered function construct");
+ break;
+
+ case vtn_construct_type_selection: {
+ /* Add the wrapper loop now and the nir_if, along the contents of
+ * this entire block, will get added inside the loop as part of
+ * vtn_emit_block() below.
+ */
+ if (next->needs_nloop) {
+ next->break_var = vtn_create_local_bool(b, "if_break");
+ nir_store_var(&b->nb, next->break_var, nir_imm_false(&b->nb), 1);
+ next->nloop = nir_push_loop(&b->nb);
+ }
+ break;
+ }
+
+ case vtn_construct_type_loop: {
+ next->break_var = vtn_create_local_bool(b, "loop_break");
+ next->continue_var = vtn_create_local_bool(b, "loop_continue");
+
+ nir_store_var(&b->nb, next->break_var, nir_imm_false(&b->nb), 1);
+ next->nloop = nir_push_loop(&b->nb);
+ nir_store_var(&b->nb, next->continue_var, nir_imm_false(&b->nb), 1);
+
+ next->nloop->control = vtn_loop_control(b, block->merge[3]);
+
+ break;
+ }
+
+ case vtn_construct_type_continue: {
+ struct vtn_construct *loop = next->parent;
+ assert(loop->type == vtn_construct_type_loop);
+ assert(!vtn_is_single_block_loop(loop));
+
+ nir_push_continue(&b->nb, loop->nloop);
+
+ break;
+ }
+
+ case vtn_construct_type_switch: {
+ /* Switch is not translated to any NIR node, all is handled by
+ * each individual case construct.
+ */
+ for (unsigned j = 0; j < block->successors_count; j++) {
+ struct vtn_successor *s = &block->successors[j];
+ if (s->block && s->block->pos < next->end_pos) {
+ struct vtn_construct *c = s->block->parent->innermost_case;
+ vtn_assert(c->type == vtn_construct_type_case);
+ if (c->needs_fallthrough) {
+ c->fallthrough_var = vtn_create_local_bool(b, "fallthrough");
+ nir_store_var(&b->nb, c->fallthrough_var, nir_imm_false(&b->nb), 1);
+ }
+ }
+ }
+ break;
+ }
+
+ case vtn_construct_type_case: {
+ struct vtn_construct *swtch = next->parent;
+ struct vtn_block *header = func->ordered_blocks[swtch->start_pos];
+
+ nir_ssa_def *sel = vtn_get_nir_ssa(b, header->branch[1]);
+ nir_ssa_def *case_condition =
+ vtn_switch_case_condition(b, swtch, sel, block->switch_case);
+ if (next->fallthrough_var) {
+ case_condition =
+ nir_ior(&b->nb, case_condition,
+ nir_load_var(&b->nb, next->fallthrough_var));
+ }
+
+ if (next->needs_nloop) {
+ next->break_var = vtn_create_local_bool(b, "case_break");
+ nir_store_var(&b->nb, next->break_var, nir_imm_false(&b->nb), 1);
+ next->nloop = nir_push_loop(&b->nb);
+ }
+
+ next->nif = nir_push_if(&b->nb, case_condition);
+
+ break;
+ }
+ }
+
+ current = next;
+ push_construct(&stack, next);
+ }
+
+ vtn_emit_block(b, block, handler);
+ }
+
+ vtn_assert(count_construct_stack(&stack) == 1);
+}
projects / platform / upstream / mesa.git / commitdiff