return false;
}
-static bool
-export_fs_mrt_z(isel_context* ctx)
-{
- Builder bld(ctx->program, ctx->block);
- unsigned enabled_channels = 0;
- bool compr = false;
- Operand values[4];
-
- for (unsigned i = 0; i < 4; ++i) {
- values[i] = Operand(v1);
- }
-
- bool writes_mrt0_alpha = ctx->program->info.ps.alpha_to_coverage_via_mrtz &&
- (ctx->outputs.mask[FRAG_RESULT_DATA0] & 0x8);
-
- /* Both stencil and sample mask only need 16-bits. */
- if (!ctx->program->info.ps.writes_z && !writes_mrt0_alpha &&
- (ctx->program->info.ps.writes_stencil || ctx->program->info.ps.writes_sample_mask)) {
- compr = ctx->program->gfx_level < GFX11; /* COMPR flag */
-
- if (ctx->program->info.ps.writes_stencil) {
- /* Stencil should be in X[23:16]. */
- values[0] = Operand(ctx->outputs.temps[FRAG_RESULT_STENCIL * 4u]);
- values[0] = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand::c32(16u), values[0]);
- enabled_channels |= ctx->program->gfx_level >= GFX11 ? 0x1 : 0x3;
- }
-
- if (ctx->program->info.ps.writes_sample_mask) {
- /* SampleMask should be in Y[15:0]. */
- values[1] = Operand(ctx->outputs.temps[FRAG_RESULT_SAMPLE_MASK * 4u]);
- enabled_channels |= ctx->program->gfx_level >= GFX11 ? 0x2 : 0xc;
- }
- } else {
- if (ctx->program->info.ps.writes_z) {
- values[0] = Operand(ctx->outputs.temps[FRAG_RESULT_DEPTH * 4u]);
- enabled_channels |= 0x1;
- }
-
- if (ctx->program->info.ps.writes_stencil) {
- values[1] = Operand(ctx->outputs.temps[FRAG_RESULT_STENCIL * 4u]);
- enabled_channels |= 0x2;
- }
-
- if (ctx->program->info.ps.writes_sample_mask) {
- values[2] = Operand(ctx->outputs.temps[FRAG_RESULT_SAMPLE_MASK * 4u]);
- enabled_channels |= 0x4;
- }
-
- if (writes_mrt0_alpha) {
- assert(ctx->program->gfx_level >= GFX11);
- values[3] = Operand(ctx->outputs.temps[FRAG_RESULT_DATA0 * 4u + 3u]);
- enabled_channels |= 0x8;
- }
- }
-
- /* GFX6 (except OLAND and HAINAN) has a bug that it only looks at the X
- * writemask component.
- */
- if (ctx->options->gfx_level == GFX6 && ctx->options->family != CHIP_OLAND &&
- ctx->options->family != CHIP_HAINAN) {
- enabled_channels |= 0x1;
- }
-
- bld.exp(aco_opcode::exp, values[0], values[1], values[2], values[3], enabled_channels,
- V_008DFC_SQ_EXP_MRTZ, compr);
-
- return true;
-}
-
struct mrt_color_export {
int slot;
unsigned write_mask;
ctx->block->instructions.emplace_back(std::move(jump));
}
-static void
-create_fs_exports(isel_context* ctx)
-{
- Builder bld(ctx->program, ctx->block);
- bool exported = false;
-
- /* Export depth, stencil and sample mask. */
- if (ctx->outputs.mask[FRAG_RESULT_DEPTH] || ctx->outputs.mask[FRAG_RESULT_STENCIL] ||
- ctx->outputs.mask[FRAG_RESULT_SAMPLE_MASK])
- exported |= export_fs_mrt_z(ctx);
-
- if (ctx->program->info.ps.has_epilog) {
- create_fs_jump_to_epilog(ctx);
-
- /* FS epilogs always have at least one color/null export. */
- ctx->program->has_color_exports = true;
- } else {
- struct aco_export_mrt mrts[8];
- unsigned compacted_mrt_index = 0;
-
- /* MRT compaction doesn't work with dual-source blending. Dual-source blending seems to
- * require MRT0 to be written. Just copy MRT1 into MRT0. Skipping MRT1 exports seems to be
- * fine.
- */
- if (ctx->program->info.ps.epilog.mrt0_is_dual_src && !ctx->outputs.mask[FRAG_RESULT_DATA0] &&
- ctx->outputs.mask[FRAG_RESULT_DATA1]) {
- u_foreach_bit (j, ctx->outputs.mask[FRAG_RESULT_DATA1]) {
- ctx->outputs.temps[FRAG_RESULT_DATA0 * 4u + j] =
- ctx->outputs.temps[FRAG_RESULT_DATA1 * 4u + j];
- }
- ctx->outputs.mask[FRAG_RESULT_DATA0] = ctx->outputs.mask[FRAG_RESULT_DATA1];
- }
-
- /* Export all color render targets. */
- for (unsigned i = FRAG_RESULT_DATA0; i < FRAG_RESULT_DATA7 + 1; ++i) {
- unsigned idx = i - FRAG_RESULT_DATA0;
-
- mrts[idx].enabled_channels = 0;
-
- if (!ctx->outputs.mask[i])
- continue;
-
- struct mrt_color_export out = {0};
-
- out.slot = compacted_mrt_index;
- out.write_mask = ctx->outputs.mask[i];
- out.col_format = (ctx->program->info.ps.epilog.spi_shader_col_format >> (4 * idx)) & 0xf;
- out.is_int8 = (ctx->program->info.ps.epilog.color_is_int8 >> idx) & 1;
- out.is_int10 = (ctx->program->info.ps.epilog.color_is_int10 >> idx) & 1;
- out.enable_mrt_output_nan_fixup =
- (ctx->options->enable_mrt_output_nan_fixup >> idx) & 1;
-
- for (unsigned c = 0; c < 4; ++c) {
- if (out.write_mask & (1 << c)) {
- out.values[c] = Operand(ctx->outputs.temps[i * 4u + c]);
- } else {
- out.values[c] = Operand(v1);
- }
- }
-
- if (export_fs_mrt_color(ctx, &out, &mrts[compacted_mrt_index])) {
- compacted_mrt_index++;
- exported = true;
- }
- }
-
- if (exported) {
- if (ctx->options->gfx_level >= GFX11 && ctx->program->info.ps.epilog.mrt0_is_dual_src) {
- struct aco_export_mrt* mrt0 = mrts[0].enabled_channels ? &mrts[0] : NULL;
- struct aco_export_mrt* mrt1 = mrts[1].enabled_channels ? &mrts[1] : NULL;
- create_fs_dual_src_export_gfx11(ctx, mrt0, mrt1);
- } else {
- for (unsigned i = 0; i < compacted_mrt_index; i++) {
- export_mrt(ctx, &mrts[i]);
- }
- }
- } else {
- create_fs_null_export(ctx);
- }
- }
-
- ctx->block->kind |= block_kind_export_end;
-}
-
Pseudo_instruction*
add_startpgm(struct isel_context* ctx)
{
sendmsg_gs_done(false, false, 0));
}
- if (ctx.stage == fragment_fs) {
- create_fs_exports(&ctx);
+ if (ctx.stage == fragment_fs && ctx.program->info.ps.has_epilog) {
+ create_fs_jump_to_epilog(&ctx);
+
+ /* FS epilogs always have at least one color/null export. */
+ ctx.program->has_color_exports = true;
+
+ ctx.block->kind |= block_kind_export_end;
}
if (endif_merged_wave_info) {
ctx->output_mask |= mask_attribs;
}
-/* Initialize arguments for the shader export intrinsic */
-static void
-si_llvm_init_export_args(struct radv_shader_context *ctx, LLVMValueRef *values,
- unsigned enabled_channels, unsigned target, unsigned index,
- struct ac_export_args *args)
-{
- /* Specify the channels that are enabled. */
- args->enabled_channels = enabled_channels;
-
- /* Specify whether the EXEC mask represents the valid mask */
- args->valid_mask = 0;
-
- /* Specify whether this is the last export */
- args->done = 0;
-
- /* Specify the target we are exporting */
- args->target = target;
-
- args->compr = false;
- args->out[0] = LLVMGetUndef(ctx->ac.f32);
- args->out[1] = LLVMGetUndef(ctx->ac.f32);
- args->out[2] = LLVMGetUndef(ctx->ac.f32);
- args->out[3] = LLVMGetUndef(ctx->ac.f32);
-
- if (!values)
- return;
-
- bool is_16bit = ac_get_type_size(LLVMTypeOf(values[0])) == 2;
- if (ctx->stage == MESA_SHADER_FRAGMENT) {
- unsigned col_format =
- (ctx->options->key.ps.epilog.spi_shader_col_format >> (4 * index)) & 0xf;
- bool is_int8 = (ctx->options->key.ps.epilog.color_is_int8 >> index) & 1;
- bool is_int10 = (ctx->options->key.ps.epilog.color_is_int10 >> index) & 1;
- bool enable_mrt_output_nan_fixup = (ctx->options->enable_mrt_output_nan_fixup >> index) & 1;
-
- LLVMValueRef (*packf)(struct ac_llvm_context * ctx, LLVMValueRef args[2]) = NULL;
- LLVMValueRef (*packi)(struct ac_llvm_context * ctx, LLVMValueRef args[2], unsigned bits,
- bool hi) = NULL;
-
- switch (col_format) {
- case V_028714_SPI_SHADER_ZERO:
- args->enabled_channels = 0; /* writemask */
- args->target = V_008DFC_SQ_EXP_NULL;
- break;
-
- case V_028714_SPI_SHADER_32_R:
- args->enabled_channels = 1;
- args->out[0] = values[0];
- break;
-
- case V_028714_SPI_SHADER_32_GR:
- args->enabled_channels = 0x3;
- args->out[0] = values[0];
- args->out[1] = values[1];
- break;
-
- case V_028714_SPI_SHADER_32_AR:
- if (ctx->ac.gfx_level >= GFX10) {
- args->enabled_channels = 0x3;
- args->out[0] = values[0];
- args->out[1] = values[3];
- } else {
- args->enabled_channels = 0x9;
- args->out[0] = values[0];
- args->out[3] = values[3];
- }
- break;
-
- case V_028714_SPI_SHADER_FP16_ABGR:
- args->enabled_channels = 0xf;
- packf = ac_build_cvt_pkrtz_f16;
- if (is_16bit) {
- for (unsigned chan = 0; chan < 4; chan++)
- values[chan] = LLVMBuildFPExt(ctx->ac.builder, values[chan], ctx->ac.f32, "");
- }
- break;
-
- case V_028714_SPI_SHADER_UNORM16_ABGR:
- args->enabled_channels = 0xf;
- packf = ac_build_cvt_pknorm_u16;
- break;
-
- case V_028714_SPI_SHADER_SNORM16_ABGR:
- args->enabled_channels = 0xf;
- packf = ac_build_cvt_pknorm_i16;
- break;
-
- case V_028714_SPI_SHADER_UINT16_ABGR:
- args->enabled_channels = 0xf;
- packi = ac_build_cvt_pk_u16;
- if (is_16bit) {
- for (unsigned chan = 0; chan < 4; chan++)
- values[chan] = LLVMBuildZExt(ctx->ac.builder, ac_to_integer(&ctx->ac, values[chan]),
- ctx->ac.i32, "");
- }
- break;
-
- case V_028714_SPI_SHADER_SINT16_ABGR:
- args->enabled_channels = 0xf;
- packi = ac_build_cvt_pk_i16;
- if (is_16bit) {
- for (unsigned chan = 0; chan < 4; chan++)
- values[chan] = LLVMBuildSExt(ctx->ac.builder, ac_to_integer(&ctx->ac, values[chan]),
- ctx->ac.i32, "");
- }
- break;
-
- default:
- case V_028714_SPI_SHADER_32_ABGR:
- memcpy(&args->out[0], values, sizeof(values[0]) * 4);
- break;
- }
-
- /* Replace NaN by zero (for 32-bit float formats) to fix game bugs if requested. */
- if (enable_mrt_output_nan_fixup && !is_16bit) {
- for (unsigned i = 0; i < 4; i++) {
- LLVMValueRef class_args[2] = {values[i],
- LLVMConstInt(ctx->ac.i32, S_NAN | Q_NAN, false)};
- LLVMValueRef isnan = ac_build_intrinsic(&ctx->ac, "llvm.amdgcn.class.f32", ctx->ac.i1,
- class_args, 2, 0);
- values[i] = LLVMBuildSelect(ctx->ac.builder, isnan, ctx->ac.f32_0, values[i], "");
- }
- }
-
- /* Pack f16 or norm_i16/u16. */
- if (packf) {
- for (unsigned chan = 0; chan < 2; chan++) {
- LLVMValueRef pack_args[2] = {values[2 * chan], values[2 * chan + 1]};
- LLVMValueRef packed;
-
- packed = packf(&ctx->ac, pack_args);
- args->out[chan] = ac_to_float(&ctx->ac, packed);
- }
- }
-
- /* Pack i16/u16. */
- if (packi) {
- for (unsigned chan = 0; chan < 2; chan++) {
- LLVMValueRef pack_args[2] = {ac_to_integer(&ctx->ac, values[2 * chan]),
- ac_to_integer(&ctx->ac, values[2 * chan + 1])};
- LLVMValueRef packed;
-
- packed = packi(&ctx->ac, pack_args, is_int8 ? 8 : is_int10 ? 10 : 16, chan == 1);
- args->out[chan] = ac_to_float(&ctx->ac, packed);
- }
- }
-
- if (packf || packi) {
- if (ctx->options->gfx_level >= GFX11) {
- args->enabled_channels = 0x3;
- } else {
- args->compr = 1; /* COMPR flag */
- }
- }
-
- return;
- }
-
- if (is_16bit) {
- for (unsigned chan = 0; chan < 4; chan++) {
- values[chan] = LLVMBuildBitCast(ctx->ac.builder, values[chan], ctx->ac.i16, "");
- args->out[chan] = LLVMBuildZExt(ctx->ac.builder, values[chan], ctx->ac.i32, "");
- }
- } else
- memcpy(&args->out[0], values, sizeof(values[0]) * 4);
-
- for (unsigned i = 0; i < 4; ++i)
- args->out[i] = ac_to_float(&ctx->ac, args->out[i]);
-}
-
static LLVMValueRef
radv_load_output(struct radv_shader_context *ctx, unsigned index, unsigned chan)
{
return LLVMBuildLoad2(ctx->ac.builder, type, output, "");
}
-static bool
-si_export_mrt_color(struct radv_shader_context *ctx, LLVMValueRef *color, unsigned target,
- unsigned index, struct ac_export_args *args)
-{
- unsigned mrt_target = V_008DFC_SQ_EXP_MRT + target;
-
- if (ctx->options->gfx_level >= GFX11 && ctx->options->key.ps.epilog.mrt0_is_dual_src &&
- (target == 0 || target == 1)) {
- mrt_target += 21;
- }
-
- si_llvm_init_export_args(ctx, color, 0xf, mrt_target, index, args);
- if (!args->enabled_channels)
- return false; /* unnecessary NULL export */
-
- return true;
-}
-
-static void
-radv_export_mrt_z(struct radv_shader_context *ctx, LLVMValueRef depth, LLVMValueRef stencil,
- LLVMValueRef samplemask)
-{
- struct ac_export_args args;
-
- ac_export_mrt_z(&ctx->ac, depth, stencil, samplemask, NULL, true, &args);
-
- ac_build_export(&ctx->ac, &args);
-}
-
-static void
-handle_fs_outputs_post(struct radv_shader_context *ctx)
-{
- unsigned index = 0;
- LLVMValueRef depth = NULL, stencil = NULL, samplemask = NULL;
- struct ac_export_args color_args[8];
-
- for (unsigned i = 0; i < AC_LLVM_MAX_OUTPUTS; ++i) {
- LLVMValueRef values[4];
-
- if (!(ctx->output_mask & (1ull << i)))
- continue;
-
- if (i < FRAG_RESULT_DATA0)
- continue;
-
- for (unsigned j = 0; j < 4; j++)
- values[j] = ac_to_float(&ctx->ac, radv_load_output(ctx, i, j));
-
- bool ret = si_export_mrt_color(ctx, values, index, i - FRAG_RESULT_DATA0, &color_args[index]);
- if (ret)
- index++;
- }
-
- /* Process depth, stencil, samplemask. */
- if (ctx->shader_info->ps.writes_z) {
- depth = ac_to_float(&ctx->ac, radv_load_output(ctx, FRAG_RESULT_DEPTH, 0));
- }
- if (ctx->shader_info->ps.writes_stencil) {
- stencil = ac_to_float(&ctx->ac, radv_load_output(ctx, FRAG_RESULT_STENCIL, 0));
- }
- if (ctx->shader_info->ps.writes_sample_mask) {
- samplemask = ac_to_float(&ctx->ac, radv_load_output(ctx, FRAG_RESULT_SAMPLE_MASK, 0));
- }
-
- /* Set the DONE bit on last non-null color export only if Z isn't
- * exported.
- */
- if (index > 0 && !ctx->shader_info->ps.writes_z &&
- !ctx->shader_info->ps.writes_stencil &&
- !ctx->shader_info->ps.writes_sample_mask) {
- unsigned last = index - 1;
-
- color_args[last].valid_mask = 1; /* whether the EXEC mask is valid */
- color_args[last].done = 1; /* DONE bit */
-
- if (ctx->options->gfx_level >= GFX11 && ctx->options->key.ps.epilog.mrt0_is_dual_src) {
- ac_build_dual_src_blend_swizzle(&ctx->ac, &color_args[0], &color_args[1]);
- }
- }
-
- /* Export PS outputs. */
- for (unsigned i = 0; i < index; i++)
- ac_build_export(&ctx->ac, &color_args[i]);
-
- if (depth || stencil || samplemask)
- radv_export_mrt_z(ctx, depth, stencil, samplemask);
- else if (!index)
- ac_build_export_null(&ctx->ac, true);
-}
-
static void
emit_gs_epilogue(struct radv_shader_context *ctx)
{
case MESA_SHADER_VERTEX:
case MESA_SHADER_TESS_CTRL:
case MESA_SHADER_TESS_EVAL:
- break; /* Lowered in NIR */
case MESA_SHADER_FRAGMENT:
- handle_fs_outputs_post(ctx);
- break;
+ break; /* Lowered in NIR */
case MESA_SHADER_GEOMETRY:
if (ctx->shader_info->is_ngg)
break; /* Lowered in NIR */
/* Lower I/O intrinsics to memory instructions. */
bool io_to_mem = radv_nir_lower_io_to_mem(device, stage);
bool lowered_ngg = stage->info.is_ngg && stage->stage == last_vgt_api_stage;
- if (lowered_ngg)
+ if (lowered_ngg) {
radv_lower_ngg(device, stage, pipeline_key);
-
- if (stage->stage == last_vgt_api_stage && !lowered_ngg) {
+ } else if (stage->stage == last_vgt_api_stage) {
if (stage->stage != MESA_SHADER_GEOMETRY) {
NIR_PASS_V(stage->nir, ac_nir_lower_legacy_vs, gfx_level,
stage->info.outinfo.clip_dist_mask | stage->info.outinfo.cull_dist_mask,
};
NIR_PASS_V(stage->nir, ac_nir_lower_legacy_gs, false, false, &gs_out_info);
}
+ } else if (stage->stage == MESA_SHADER_FRAGMENT) {
+ ac_nir_lower_ps_options options = {
+ .gfx_level = gfx_level,
+ .family = device->physical_device->rad_info.family,
+ .use_aco = !radv_use_llvm_for_stage(device, stage->stage),
+ .uses_discard = true,
+ /* Compared to radv_pipeline_key.ps.alpha_to_coverage_via_mrtz,
+ * radv_shader_info.ps.writes_mrt0_alpha need any depth/stencil/sample_mask exist.
+ * ac_nir_lower_ps() require this field to reflect whether alpha via mrtz is really
+ * present.
+ */
+ .alpha_to_coverage_via_mrtz = stage->info.ps.writes_mrt0_alpha,
+ .dual_src_blend_swizzle =
+ pipeline_key->ps.epilog.mrt0_is_dual_src && gfx_level >= GFX11,
+ /* Need to filter out unwritten color slots. */
+ .spi_shader_col_format =
+ pipeline_key->ps.epilog.spi_shader_col_format & stage->info.ps.colors_written,
+ .color_is_int8 = pipeline_key->ps.epilog.color_is_int8,
+ .color_is_int10 = pipeline_key->ps.epilog.color_is_int10,
+ .alpha_func = PIPE_FUNC_ALWAYS,
+
+ .enable_mrt_output_nan_fixup = pipeline_key->ps.epilog.enable_mrt_output_nan_fixup,
+ .no_color_export = stage->info.ps.has_epilog,
+ };
+
+ NIR_PASS_V(stage->nir, ac_nir_lower_ps, &options);
}
NIR_PASS(_, stage->nir, nir_opt_idiv_const, 8);
projects / platform / upstream / mesa.git / commitdiff