current_branch->next_block = endif_block;
}
-/* Invoked after a branch is exited. */
-static void ac_branch_exited(struct ac_llvm_context *ctx)
-{
- if (ctx->flow->depth == 0 && ctx->conditional_demote_seen) {
- /* The previous conditional branch contained demote. Kill threads
- * after all conditional blocks because amdgcn.wqm.vote doesn't
- * return usable values inside the blocks.
- *
- * This is an optional optimization that only kills whole inactive quads.
- */
- LLVMValueRef cond = LLVMBuildLoad2(ctx->builder, ctx->i1, ctx->postponed_kill, "");
- ac_build_kill_if_false(ctx, ac_build_wqm_vote(ctx, cond));
- ctx->conditional_demote_seen = false;
- }
-}
-
void ac_build_endif(struct ac_llvm_context *ctx, int label_id)
{
struct ac_llvm_flow *current_branch = get_current_flow(ctx);
set_basicblock_name(current_branch->next_block, "endif", label_id);
ctx->flow->depth--;
- ac_branch_exited(ctx);
}
void ac_build_endloop(struct ac_llvm_context *ctx, int label_id)
LLVMPositionBuilderAtEnd(ctx->builder, current_loop->next_block);
set_basicblock_name(current_loop->next_block, "endloop", label_id);
ctx->flow->depth--;
- ac_branch_exited(ctx);
}
void ac_build_ifcc(struct ac_llvm_context *ctx, LLVMValueRef cond, int label_id)
LLVMValueRef ac_build_load_helper_invocation(struct ac_llvm_context *ctx)
{
- LLVMValueRef result;
+ LLVMValueRef result = ac_build_intrinsic(ctx, "llvm.amdgcn.live.mask", ctx->i1, NULL, 0, 0);
- if (LLVM_VERSION_MAJOR >= 13) {
- result = ac_build_intrinsic(ctx, "llvm.amdgcn.live.mask", ctx->i1, NULL, 0, 0);
- } else {
- result = ac_build_intrinsic(ctx, "llvm.amdgcn.ps.live", ctx->i1, NULL, 0, 0);
- }
return LLVMBuildNot(ctx->builder, result, "");
}
-LLVMValueRef ac_build_is_helper_invocation(struct ac_llvm_context *ctx)
-{
- if (!ctx->postponed_kill)
- return ac_build_load_helper_invocation(ctx);
-
- /* postponed_kill should be NULL on LLVM 13+ */
- assert(LLVM_VERSION_MAJOR < 13);
-
- /* !(exact && postponed) */
- LLVMValueRef exact =
- ac_build_intrinsic(ctx, "llvm.amdgcn.ps.live", ctx->i1, NULL, 0, 0);
-
- LLVMValueRef postponed = LLVMBuildLoad2(ctx->builder, ctx->i1, ctx->postponed_kill, "");
- return LLVMBuildNot(ctx->builder, LLVMBuildAnd(ctx->builder, exact, postponed, ""), "");
-}
-
LLVMValueRef ac_build_call(struct ac_llvm_context *ctx, LLVMTypeRef fn_type, LLVMValueRef func, LLVMValueRef *args,
unsigned num_args)
{
static void visit_store_ssbo(struct ac_nir_context *ctx, nir_intrinsic_instr *instr)
{
- if (ctx->ac.postponed_kill) {
- LLVMValueRef cond = LLVMBuildLoad2(ctx->ac.builder, ctx->ac.i1, ctx->ac.postponed_kill, "");
- ac_build_ifcc(&ctx->ac, cond, 7000);
- }
-
LLVMValueRef src_data = get_src(ctx, instr->src[0]);
int elem_size_bytes = ac_get_elem_bits(&ctx->ac, LLVMTypeOf(src_data)) / 8;
unsigned writemask = nir_intrinsic_write_mask(instr);
}
exit_waterfall(ctx, &wctx, NULL);
-
- if (ctx->ac.postponed_kill)
- ac_build_endif(&ctx->ac, 7000);
}
static LLVMValueRef emit_ssbo_comp_swap_64(struct ac_nir_context *ctx, LLVMValueRef descriptor,
static LLVMValueRef visit_atomic_ssbo(struct ac_nir_context *ctx, nir_intrinsic_instr *instr)
{
- if (ctx->ac.postponed_kill) {
- LLVMValueRef cond = LLVMBuildLoad2(ctx->ac.builder, ctx->ac.i1, ctx->ac.postponed_kill, "");
- ac_build_ifcc(&ctx->ac, cond, 7001);
- }
-
nir_atomic_op nir_op = nir_intrinsic_atomic_op(instr);
const char *op = translate_atomic_op_str(nir_op);
bool is_float = nir_atomic_op_type(nir_op) == nir_type_float;
}
}
- result = exit_waterfall(ctx, &wctx, result);
- if (ctx->ac.postponed_kill)
- ac_build_endif(&ctx->ac, 7001);
- return result;
+ return exit_waterfall(ctx, &wctx, result);
}
static LLVMValueRef visit_load_buffer(struct ac_nir_context *ctx, nir_intrinsic_instr *instr)
static void visit_store_global(struct ac_nir_context *ctx,
nir_intrinsic_instr *instr)
{
- if (ctx->ac.postponed_kill) {
- LLVMValueRef cond = LLVMBuildLoad2(ctx->ac.builder, ctx->ac.i1, ctx->ac.postponed_kill, "");
- ac_build_ifcc(&ctx->ac, cond, 7002);
- }
-
LLVMValueRef data = get_src(ctx, instr->src[0]);
LLVMTypeRef type = LLVMTypeOf(data);
LLVMValueRef addr = get_global_address(ctx, instr, type);
LLVMSetOrdering(val, LLVMAtomicOrderingMonotonic);
LLVMSetAlignment(val, ac_get_type_size(type));
}
-
- if (ctx->ac.postponed_kill)
- ac_build_endif(&ctx->ac, 7002);
}
static LLVMValueRef visit_global_atomic(struct ac_nir_context *ctx,
nir_intrinsic_instr *instr)
{
- if (ctx->ac.postponed_kill) {
- LLVMValueRef cond = LLVMBuildLoad2(ctx->ac.builder, ctx->ac.i1, ctx->ac.postponed_kill, "");
- ac_build_ifcc(&ctx->ac, cond, 7002);
- }
-
LLVMValueRef data = get_src(ctx, instr->src[1]);
LLVMAtomicRMWBinOp op;
LLVMValueRef result;
result = ac_build_atomic_rmw(&ctx->ac, op, addr, ac_to_integer(&ctx->ac, data), sync_scope);
}
- if (ctx->ac.postponed_kill)
- ac_build_endif(&ctx->ac, 7002);
-
return result;
}
static void visit_store_output(struct ac_nir_context *ctx, nir_intrinsic_instr *instr)
{
- if (ctx->ac.postponed_kill) {
- LLVMValueRef cond = LLVMBuildLoad2(ctx->ac.builder, ctx->ac.i1, ctx->ac.postponed_kill, "");
- ac_build_ifcc(&ctx->ac, cond, 7002);
- }
-
unsigned base = nir_intrinsic_base(instr);
unsigned writemask = nir_intrinsic_write_mask(instr);
unsigned component = nir_intrinsic_component(instr);
}
LLVMBuildStore(ctx->ac.builder, value, output_addr);
}
-
- if (ctx->ac.postponed_kill)
- ac_build_endif(&ctx->ac, 7002);
}
static int image_type_to_components_count(enum glsl_sampler_dim dim, bool array)
static void visit_image_store(struct ac_nir_context *ctx, const nir_intrinsic_instr *instr)
{
- if (ctx->ac.postponed_kill) {
- LLVMValueRef cond = LLVMBuildLoad2(ctx->ac.builder, ctx->ac.i1, ctx->ac.postponed_kill, "");
- ac_build_ifcc(&ctx->ac, cond, 7003);
- }
-
enum glsl_sampler_dim dim = nir_intrinsic_image_dim(instr);
bool is_array = nir_intrinsic_image_array(instr);
}
exit_waterfall(ctx, &wctx, NULL);
- if (ctx->ac.postponed_kill)
- ac_build_endif(&ctx->ac, 7003);
}
static LLVMValueRef visit_image_atomic(struct ac_nir_context *ctx, const nir_intrinsic_instr *instr)
{
- if (ctx->ac.postponed_kill) {
- LLVMValueRef cond = LLVMBuildLoad2(ctx->ac.builder, ctx->ac.i1, ctx->ac.postponed_kill, "");
- ac_build_ifcc(&ctx->ac, cond, 7004);
- }
-
LLVMValueRef params[7];
int param_count = 0;
result = ac_build_image_opcode(&ctx->ac, &args);
}
- result = exit_waterfall(ctx, &wctx, result);
- if (ctx->ac.postponed_kill)
- ac_build_endif(&ctx->ac, 7004);
- return result;
+ return exit_waterfall(ctx, &wctx, result);
}
static void emit_discard(struct ac_nir_context *ctx, const nir_intrinsic_instr *instr)
cond = ctx->ac.i1false;
}
- if (LLVM_VERSION_MAJOR >= 13) {
- /* This demotes the pixel if the condition is false. */
- ac_build_intrinsic(&ctx->ac, "llvm.amdgcn.wqm.demote", ctx->ac.voidt, &cond, 1, 0);
- return;
- }
-
- LLVMValueRef mask = LLVMBuildLoad2(ctx->ac.builder, ctx->ac.i1, ctx->ac.postponed_kill, "");
- mask = LLVMBuildAnd(ctx->ac.builder, mask, cond, "");
- LLVMBuildStore(ctx->ac.builder, mask, ctx->ac.postponed_kill);
-
- if (!ctx->info->fs.needs_all_helper_invocations) {
- /* This is an optional optimization that only kills whole inactive quads.
- * It's not used when subgroup operations can possibly use all helper
- * invocations.
- */
- if (ctx->ac.flow->depth == 0) {
- ac_build_kill_if_false(&ctx->ac, ac_build_wqm_vote(&ctx->ac, cond));
- } else {
- /* amdgcn.wqm.vote doesn't work inside conditional blocks. Here's why.
- *
- * The problem is that kill(wqm.vote(0)) kills all active threads within
- * the block, which breaks the whole quad mode outside the block if
- * the conditional block has partially active quads (2x2 pixel blocks).
- * E.g. threads 0-3 are active outside the block, but only thread 0 is
- * active inside the block. Thread 0 shouldn't be killed by demote,
- * because threads 1-3 are still active outside the block.
- *
- * The fix for amdgcn.wqm.vote would be to return S_WQM((live & ~exec) | cond)
- * instead of S_WQM(cond).
- *
- * The less efficient workaround we do here is to save the kill condition
- * to a temporary (postponed_kill) and do kill(wqm.vote(cond)) after we
- * exit the conditional block.
- */
- ctx->ac.conditional_demote_seen = true;
- }
- }
+ /* This demotes the pixel if the condition is false. */
+ ac_build_intrinsic(&ctx->ac, "llvm.amdgcn.wqm.demote", ctx->ac.voidt, &cond, 1, 0);
}
static LLVMValueRef visit_load_subgroup_id(struct ac_nir_context *ctx)
static LLVMValueRef visit_var_atomic(struct ac_nir_context *ctx, const nir_intrinsic_instr *instr,
LLVMValueRef ptr, int src_idx)
{
- if (ctx->ac.postponed_kill) {
- LLVMValueRef cond = LLVMBuildLoad2(ctx->ac.builder, ctx->ac.i1, ctx->ac.postponed_kill, "");
- ac_build_ifcc(&ctx->ac, cond, 7005);
- }
-
LLVMValueRef result;
LLVMValueRef src = get_src(ctx, instr->src[src_idx]);
nir_atomic_op nir_op = nir_intrinsic_atomic_op(instr);
}
}
- if (ctx->ac.postponed_kill)
- ac_build_endif(&ctx->ac, 7005);
return result;
}
result = emit_i2b(&ctx->ac, ac_get_arg(&ctx->ac, ctx->args->front_face));
break;
case nir_intrinsic_load_helper_invocation:
- result = ac_build_load_helper_invocation(&ctx->ac);
- break;
case nir_intrinsic_is_helper_invocation:
- result = ac_build_is_helper_invocation(&ctx->ac);
+ result = ac_build_load_helper_invocation(&ctx->ac);
break;
case nir_intrinsic_load_user_data_amd:
assert(LLVMTypeOf(ctx->abi->user_data) == ctx->ac.v4i32);
if (gl_shader_stage_is_compute(nir->info.stage))
setup_shared(&ctx, nir);
- if (nir->info.stage == MESA_SHADER_FRAGMENT && nir->info.fs.uses_demote &&
- LLVM_VERSION_MAJOR < 13) {
- /* true = don't kill. */
- ctx.ac.postponed_kill = ac_build_alloca_init(&ctx.ac, ctx.ac.i1true, "");
- }
-
if (!visit_cf_list(&ctx, &func->impl->body))
return false;
phi_post_pass(&ctx);
- if (ctx.ac.postponed_kill)
- ac_build_kill_if_false(&ctx.ac, LLVMBuildLoad2(ctx.ac.builder, ctx.ac.i1, ctx.ac.postponed_kill, ""));
-
free(ctx.ssa_defs);
ralloc_free(ctx.defs);
ralloc_free(ctx.phis);