From 290c3d360e5a6f5226c062d6a9267629adb1060e Mon Sep 17 00:00:00 2001 From: Qiang Yu Date: Thu, 30 Mar 2023 09:32:58 +0800 Subject: [PATCH] aco,radv: lower outputs to exports when nir for monolithic ps MIME-Version: 1.0 Content-Type: text/plain; charset=utf8 Content-Transfer-Encoding: 8bit Remove the compiler backend code for outputs to exports. Reviewed-by: Timur Kristóf Signed-off-by: Qiang Yu Part-of: --- src/amd/compiler/aco_instruction_selection.cpp | 162 +-------------- src/amd/vulkan/radv_nir_to_llvm.c | 264 +------------------------ src/amd/vulkan/radv_pipeline.c | 31 ++- 3 files changed, 36 insertions(+), 421 deletions(-) diff --git a/src/amd/compiler/aco_instruction_selection.cpp b/src/amd/compiler/aco_instruction_selection.cpp index 9fffd3e..3cabec8 100644 --- a/src/amd/compiler/aco_instruction_selection.cpp +++ b/src/amd/compiler/aco_instruction_selection.cpp @@ -10821,75 +10821,6 @@ visit_cf_list(isel_context* ctx, struct exec_list* list) return false; } -static bool -export_fs_mrt_z(isel_context* ctx) -{ - Builder bld(ctx->program, ctx->block); - unsigned enabled_channels = 0; - bool compr = false; - Operand values[4]; - - for (unsigned i = 0; i < 4; ++i) { - values[i] = Operand(v1); - } - - bool writes_mrt0_alpha = ctx->program->info.ps.alpha_to_coverage_via_mrtz && - (ctx->outputs.mask[FRAG_RESULT_DATA0] & 0x8); - - /* Both stencil and sample mask only need 16-bits. */ - if (!ctx->program->info.ps.writes_z && !writes_mrt0_alpha && - (ctx->program->info.ps.writes_stencil || ctx->program->info.ps.writes_sample_mask)) { - compr = ctx->program->gfx_level < GFX11; /* COMPR flag */ - - if (ctx->program->info.ps.writes_stencil) { - /* Stencil should be in X[23:16]. */ - values[0] = Operand(ctx->outputs.temps[FRAG_RESULT_STENCIL * 4u]); - values[0] = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand::c32(16u), values[0]); - enabled_channels |= ctx->program->gfx_level >= GFX11 ? 0x1 : 0x3; - } - - if (ctx->program->info.ps.writes_sample_mask) { - /* SampleMask should be in Y[15:0]. */ - values[1] = Operand(ctx->outputs.temps[FRAG_RESULT_SAMPLE_MASK * 4u]); - enabled_channels |= ctx->program->gfx_level >= GFX11 ? 0x2 : 0xc; - } - } else { - if (ctx->program->info.ps.writes_z) { - values[0] = Operand(ctx->outputs.temps[FRAG_RESULT_DEPTH * 4u]); - enabled_channels |= 0x1; - } - - if (ctx->program->info.ps.writes_stencil) { - values[1] = Operand(ctx->outputs.temps[FRAG_RESULT_STENCIL * 4u]); - enabled_channels |= 0x2; - } - - if (ctx->program->info.ps.writes_sample_mask) { - values[2] = Operand(ctx->outputs.temps[FRAG_RESULT_SAMPLE_MASK * 4u]); - enabled_channels |= 0x4; - } - - if (writes_mrt0_alpha) { - assert(ctx->program->gfx_level >= GFX11); - values[3] = Operand(ctx->outputs.temps[FRAG_RESULT_DATA0 * 4u + 3u]); - enabled_channels |= 0x8; - } - } - - /* GFX6 (except OLAND and HAINAN) has a bug that it only looks at the X - * writemask component. - */ - if (ctx->options->gfx_level == GFX6 && ctx->options->family != CHIP_OLAND && - ctx->options->family != CHIP_HAINAN) { - enabled_channels |= 0x1; - } - - bld.exp(aco_opcode::exp, values[0], values[1], values[2], values[3], enabled_channels, - V_008DFC_SQ_EXP_MRTZ, compr); - - return true; -} - struct mrt_color_export { int slot; unsigned write_mask; @@ -11159,90 +11090,6 @@ create_fs_jump_to_epilog(isel_context* ctx) ctx->block->instructions.emplace_back(std::move(jump)); } -static void -create_fs_exports(isel_context* ctx) -{ - Builder bld(ctx->program, ctx->block); - bool exported = false; - - /* Export depth, stencil and sample mask. */ - if (ctx->outputs.mask[FRAG_RESULT_DEPTH] || ctx->outputs.mask[FRAG_RESULT_STENCIL] || - ctx->outputs.mask[FRAG_RESULT_SAMPLE_MASK]) - exported |= export_fs_mrt_z(ctx); - - if (ctx->program->info.ps.has_epilog) { - create_fs_jump_to_epilog(ctx); - - /* FS epilogs always have at least one color/null export. */ - ctx->program->has_color_exports = true; - } else { - struct aco_export_mrt mrts[8]; - unsigned compacted_mrt_index = 0; - - /* MRT compaction doesn't work with dual-source blending. Dual-source blending seems to - * require MRT0 to be written. Just copy MRT1 into MRT0. Skipping MRT1 exports seems to be - * fine. - */ - if (ctx->program->info.ps.epilog.mrt0_is_dual_src && !ctx->outputs.mask[FRAG_RESULT_DATA0] && - ctx->outputs.mask[FRAG_RESULT_DATA1]) { - u_foreach_bit (j, ctx->outputs.mask[FRAG_RESULT_DATA1]) { - ctx->outputs.temps[FRAG_RESULT_DATA0 * 4u + j] = - ctx->outputs.temps[FRAG_RESULT_DATA1 * 4u + j]; - } - ctx->outputs.mask[FRAG_RESULT_DATA0] = ctx->outputs.mask[FRAG_RESULT_DATA1]; - } - - /* Export all color render targets. */ - for (unsigned i = FRAG_RESULT_DATA0; i < FRAG_RESULT_DATA7 + 1; ++i) { - unsigned idx = i - FRAG_RESULT_DATA0; - - mrts[idx].enabled_channels = 0; - - if (!ctx->outputs.mask[i]) - continue; - - struct mrt_color_export out = {0}; - - out.slot = compacted_mrt_index; - out.write_mask = ctx->outputs.mask[i]; - out.col_format = (ctx->program->info.ps.epilog.spi_shader_col_format >> (4 * idx)) & 0xf; - out.is_int8 = (ctx->program->info.ps.epilog.color_is_int8 >> idx) & 1; - out.is_int10 = (ctx->program->info.ps.epilog.color_is_int10 >> idx) & 1; - out.enable_mrt_output_nan_fixup = - (ctx->options->enable_mrt_output_nan_fixup >> idx) & 1; - - for (unsigned c = 0; c < 4; ++c) { - if (out.write_mask & (1 << c)) { - out.values[c] = Operand(ctx->outputs.temps[i * 4u + c]); - } else { - out.values[c] = Operand(v1); - } - } - - if (export_fs_mrt_color(ctx, &out, &mrts[compacted_mrt_index])) { - compacted_mrt_index++; - exported = true; - } - } - - if (exported) { - if (ctx->options->gfx_level >= GFX11 && ctx->program->info.ps.epilog.mrt0_is_dual_src) { - struct aco_export_mrt* mrt0 = mrts[0].enabled_channels ? &mrts[0] : NULL; - struct aco_export_mrt* mrt1 = mrts[1].enabled_channels ? &mrts[1] : NULL; - create_fs_dual_src_export_gfx11(ctx, mrt0, mrt1); - } else { - for (unsigned i = 0; i < compacted_mrt_index; i++) { - export_mrt(ctx, &mrts[i]); - } - } - } else { - create_fs_null_export(ctx); - } - } - - ctx->block->kind |= block_kind_export_end; -} - Pseudo_instruction* add_startpgm(struct isel_context* ctx) { @@ -11610,8 +11457,13 @@ select_program(Program* program, unsigned shader_count, struct nir_shader* const sendmsg_gs_done(false, false, 0)); } - if (ctx.stage == fragment_fs) { - create_fs_exports(&ctx); + if (ctx.stage == fragment_fs && ctx.program->info.ps.has_epilog) { + create_fs_jump_to_epilog(&ctx); + + /* FS epilogs always have at least one color/null export. */ + ctx.program->has_color_exports = true; + + ctx.block->kind |= block_kind_export_end; } if (endif_merged_wave_info) { diff --git a/src/amd/vulkan/radv_nir_to_llvm.c b/src/amd/vulkan/radv_nir_to_llvm.c index 0cde5b1..7ae308e 100644 --- a/src/amd/vulkan/radv_nir_to_llvm.c +++ b/src/amd/vulkan/radv_nir_to_llvm.c @@ -303,176 +303,6 @@ scan_shader_output_decl(struct radv_shader_context *ctx, struct nir_variable *va ctx->output_mask |= mask_attribs; } -/* Initialize arguments for the shader export intrinsic */ -static void -si_llvm_init_export_args(struct radv_shader_context *ctx, LLVMValueRef *values, - unsigned enabled_channels, unsigned target, unsigned index, - struct ac_export_args *args) -{ - /* Specify the channels that are enabled. */ - args->enabled_channels = enabled_channels; - - /* Specify whether the EXEC mask represents the valid mask */ - args->valid_mask = 0; - - /* Specify whether this is the last export */ - args->done = 0; - - /* Specify the target we are exporting */ - args->target = target; - - args->compr = false; - args->out[0] = LLVMGetUndef(ctx->ac.f32); - args->out[1] = LLVMGetUndef(ctx->ac.f32); - args->out[2] = LLVMGetUndef(ctx->ac.f32); - args->out[3] = LLVMGetUndef(ctx->ac.f32); - - if (!values) - return; - - bool is_16bit = ac_get_type_size(LLVMTypeOf(values[0])) == 2; - if (ctx->stage == MESA_SHADER_FRAGMENT) { - unsigned col_format = - (ctx->options->key.ps.epilog.spi_shader_col_format >> (4 * index)) & 0xf; - bool is_int8 = (ctx->options->key.ps.epilog.color_is_int8 >> index) & 1; - bool is_int10 = (ctx->options->key.ps.epilog.color_is_int10 >> index) & 1; - bool enable_mrt_output_nan_fixup = (ctx->options->enable_mrt_output_nan_fixup >> index) & 1; - - LLVMValueRef (*packf)(struct ac_llvm_context * ctx, LLVMValueRef args[2]) = NULL; - LLVMValueRef (*packi)(struct ac_llvm_context * ctx, LLVMValueRef args[2], unsigned bits, - bool hi) = NULL; - - switch (col_format) { - case V_028714_SPI_SHADER_ZERO: - args->enabled_channels = 0; /* writemask */ - args->target = V_008DFC_SQ_EXP_NULL; - break; - - case V_028714_SPI_SHADER_32_R: - args->enabled_channels = 1; - args->out[0] = values[0]; - break; - - case V_028714_SPI_SHADER_32_GR: - args->enabled_channels = 0x3; - args->out[0] = values[0]; - args->out[1] = values[1]; - break; - - case V_028714_SPI_SHADER_32_AR: - if (ctx->ac.gfx_level >= GFX10) { - args->enabled_channels = 0x3; - args->out[0] = values[0]; - args->out[1] = values[3]; - } else { - args->enabled_channels = 0x9; - args->out[0] = values[0]; - args->out[3] = values[3]; - } - break; - - case V_028714_SPI_SHADER_FP16_ABGR: - args->enabled_channels = 0xf; - packf = ac_build_cvt_pkrtz_f16; - if (is_16bit) { - for (unsigned chan = 0; chan < 4; chan++) - values[chan] = LLVMBuildFPExt(ctx->ac.builder, values[chan], ctx->ac.f32, ""); - } - break; - - case V_028714_SPI_SHADER_UNORM16_ABGR: - args->enabled_channels = 0xf; - packf = ac_build_cvt_pknorm_u16; - break; - - case V_028714_SPI_SHADER_SNORM16_ABGR: - args->enabled_channels = 0xf; - packf = ac_build_cvt_pknorm_i16; - break; - - case V_028714_SPI_SHADER_UINT16_ABGR: - args->enabled_channels = 0xf; - packi = ac_build_cvt_pk_u16; - if (is_16bit) { - for (unsigned chan = 0; chan < 4; chan++) - values[chan] = LLVMBuildZExt(ctx->ac.builder, ac_to_integer(&ctx->ac, values[chan]), - ctx->ac.i32, ""); - } - break; - - case V_028714_SPI_SHADER_SINT16_ABGR: - args->enabled_channels = 0xf; - packi = ac_build_cvt_pk_i16; - if (is_16bit) { - for (unsigned chan = 0; chan < 4; chan++) - values[chan] = LLVMBuildSExt(ctx->ac.builder, ac_to_integer(&ctx->ac, values[chan]), - ctx->ac.i32, ""); - } - break; - - default: - case V_028714_SPI_SHADER_32_ABGR: - memcpy(&args->out[0], values, sizeof(values[0]) * 4); - break; - } - - /* Replace NaN by zero (for 32-bit float formats) to fix game bugs if requested. */ - if (enable_mrt_output_nan_fixup && !is_16bit) { - for (unsigned i = 0; i < 4; i++) { - LLVMValueRef class_args[2] = {values[i], - LLVMConstInt(ctx->ac.i32, S_NAN | Q_NAN, false)}; - LLVMValueRef isnan = ac_build_intrinsic(&ctx->ac, "llvm.amdgcn.class.f32", ctx->ac.i1, - class_args, 2, 0); - values[i] = LLVMBuildSelect(ctx->ac.builder, isnan, ctx->ac.f32_0, values[i], ""); - } - } - - /* Pack f16 or norm_i16/u16. */ - if (packf) { - for (unsigned chan = 0; chan < 2; chan++) { - LLVMValueRef pack_args[2] = {values[2 * chan], values[2 * chan + 1]}; - LLVMValueRef packed; - - packed = packf(&ctx->ac, pack_args); - args->out[chan] = ac_to_float(&ctx->ac, packed); - } - } - - /* Pack i16/u16. */ - if (packi) { - for (unsigned chan = 0; chan < 2; chan++) { - LLVMValueRef pack_args[2] = {ac_to_integer(&ctx->ac, values[2 * chan]), - ac_to_integer(&ctx->ac, values[2 * chan + 1])}; - LLVMValueRef packed; - - packed = packi(&ctx->ac, pack_args, is_int8 ? 8 : is_int10 ? 10 : 16, chan == 1); - args->out[chan] = ac_to_float(&ctx->ac, packed); - } - } - - if (packf || packi) { - if (ctx->options->gfx_level >= GFX11) { - args->enabled_channels = 0x3; - } else { - args->compr = 1; /* COMPR flag */ - } - } - - return; - } - - if (is_16bit) { - for (unsigned chan = 0; chan < 4; chan++) { - values[chan] = LLVMBuildBitCast(ctx->ac.builder, values[chan], ctx->ac.i16, ""); - args->out[chan] = LLVMBuildZExt(ctx->ac.builder, values[chan], ctx->ac.i32, ""); - } - } else - memcpy(&args->out[0], values, sizeof(values[0]) * 4); - - for (unsigned i = 0; i < 4; ++i) - args->out[i] = ac_to_float(&ctx->ac, args->out[i]); -} - static LLVMValueRef radv_load_output(struct radv_shader_context *ctx, unsigned index, unsigned chan) { @@ -482,96 +312,6 @@ radv_load_output(struct radv_shader_context *ctx, unsigned index, unsigned chan) return LLVMBuildLoad2(ctx->ac.builder, type, output, ""); } -static bool -si_export_mrt_color(struct radv_shader_context *ctx, LLVMValueRef *color, unsigned target, - unsigned index, struct ac_export_args *args) -{ - unsigned mrt_target = V_008DFC_SQ_EXP_MRT + target; - - if (ctx->options->gfx_level >= GFX11 && ctx->options->key.ps.epilog.mrt0_is_dual_src && - (target == 0 || target == 1)) { - mrt_target += 21; - } - - si_llvm_init_export_args(ctx, color, 0xf, mrt_target, index, args); - if (!args->enabled_channels) - return false; /* unnecessary NULL export */ - - return true; -} - -static void -radv_export_mrt_z(struct radv_shader_context *ctx, LLVMValueRef depth, LLVMValueRef stencil, - LLVMValueRef samplemask) -{ - struct ac_export_args args; - - ac_export_mrt_z(&ctx->ac, depth, stencil, samplemask, NULL, true, &args); - - ac_build_export(&ctx->ac, &args); -} - -static void -handle_fs_outputs_post(struct radv_shader_context *ctx) -{ - unsigned index = 0; - LLVMValueRef depth = NULL, stencil = NULL, samplemask = NULL; - struct ac_export_args color_args[8]; - - for (unsigned i = 0; i < AC_LLVM_MAX_OUTPUTS; ++i) { - LLVMValueRef values[4]; - - if (!(ctx->output_mask & (1ull << i))) - continue; - - if (i < FRAG_RESULT_DATA0) - continue; - - for (unsigned j = 0; j < 4; j++) - values[j] = ac_to_float(&ctx->ac, radv_load_output(ctx, i, j)); - - bool ret = si_export_mrt_color(ctx, values, index, i - FRAG_RESULT_DATA0, &color_args[index]); - if (ret) - index++; - } - - /* Process depth, stencil, samplemask. */ - if (ctx->shader_info->ps.writes_z) { - depth = ac_to_float(&ctx->ac, radv_load_output(ctx, FRAG_RESULT_DEPTH, 0)); - } - if (ctx->shader_info->ps.writes_stencil) { - stencil = ac_to_float(&ctx->ac, radv_load_output(ctx, FRAG_RESULT_STENCIL, 0)); - } - if (ctx->shader_info->ps.writes_sample_mask) { - samplemask = ac_to_float(&ctx->ac, radv_load_output(ctx, FRAG_RESULT_SAMPLE_MASK, 0)); - } - - /* Set the DONE bit on last non-null color export only if Z isn't - * exported. - */ - if (index > 0 && !ctx->shader_info->ps.writes_z && - !ctx->shader_info->ps.writes_stencil && - !ctx->shader_info->ps.writes_sample_mask) { - unsigned last = index - 1; - - color_args[last].valid_mask = 1; /* whether the EXEC mask is valid */ - color_args[last].done = 1; /* DONE bit */ - - if (ctx->options->gfx_level >= GFX11 && ctx->options->key.ps.epilog.mrt0_is_dual_src) { - ac_build_dual_src_blend_swizzle(&ctx->ac, &color_args[0], &color_args[1]); - } - } - - /* Export PS outputs. */ - for (unsigned i = 0; i < index; i++) - ac_build_export(&ctx->ac, &color_args[i]); - - if (depth || stencil || samplemask) - radv_export_mrt_z(ctx, depth, stencil, samplemask); - else if (!index) - ac_build_export_null(&ctx->ac, true); -} - static void emit_gs_epilogue(struct radv_shader_context *ctx) { @@ -590,10 +330,8 @@ handle_shader_outputs_post(struct ac_shader_abi *abi) case MESA_SHADER_VERTEX: case MESA_SHADER_TESS_CTRL: case MESA_SHADER_TESS_EVAL: - break; /* Lowered in NIR */ case MESA_SHADER_FRAGMENT: - handle_fs_outputs_post(ctx); - break; + break; /* Lowered in NIR */ case MESA_SHADER_GEOMETRY: if (ctx->shader_info->is_ngg) break; /* Lowered in NIR */ diff --git a/src/amd/vulkan/radv_pipeline.c b/src/amd/vulkan/radv_pipeline.c index 007e276..b5322b7 100644 --- a/src/amd/vulkan/radv_pipeline.c +++ b/src/amd/vulkan/radv_pipeline.c @@ -570,10 +570,9 @@ radv_postprocess_nir(struct radv_device *device, const struct radv_pipeline_layo /* Lower I/O intrinsics to memory instructions. */ bool io_to_mem = radv_nir_lower_io_to_mem(device, stage); bool lowered_ngg = stage->info.is_ngg && stage->stage == last_vgt_api_stage; - if (lowered_ngg) + if (lowered_ngg) { radv_lower_ngg(device, stage, pipeline_key); - - if (stage->stage == last_vgt_api_stage && !lowered_ngg) { + } else if (stage->stage == last_vgt_api_stage) { if (stage->stage != MESA_SHADER_GEOMETRY) { NIR_PASS_V(stage->nir, ac_nir_lower_legacy_vs, gfx_level, stage->info.outinfo.clip_dist_mask | stage->info.outinfo.cull_dist_mask, @@ -588,6 +587,32 @@ radv_postprocess_nir(struct radv_device *device, const struct radv_pipeline_layo }; NIR_PASS_V(stage->nir, ac_nir_lower_legacy_gs, false, false, &gs_out_info); } + } else if (stage->stage == MESA_SHADER_FRAGMENT) { + ac_nir_lower_ps_options options = { + .gfx_level = gfx_level, + .family = device->physical_device->rad_info.family, + .use_aco = !radv_use_llvm_for_stage(device, stage->stage), + .uses_discard = true, + /* Compared to radv_pipeline_key.ps.alpha_to_coverage_via_mrtz, + * radv_shader_info.ps.writes_mrt0_alpha need any depth/stencil/sample_mask exist. + * ac_nir_lower_ps() require this field to reflect whether alpha via mrtz is really + * present. + */ + .alpha_to_coverage_via_mrtz = stage->info.ps.writes_mrt0_alpha, + .dual_src_blend_swizzle = + pipeline_key->ps.epilog.mrt0_is_dual_src && gfx_level >= GFX11, + /* Need to filter out unwritten color slots. */ + .spi_shader_col_format = + pipeline_key->ps.epilog.spi_shader_col_format & stage->info.ps.colors_written, + .color_is_int8 = pipeline_key->ps.epilog.color_is_int8, + .color_is_int10 = pipeline_key->ps.epilog.color_is_int10, + .alpha_func = PIPE_FUNC_ALWAYS, + + .enable_mrt_output_nan_fixup = pipeline_key->ps.epilog.enable_mrt_output_nan_fixup, + .no_color_export = stage->info.ps.has_epilog, + }; + + NIR_PASS_V(stage->nir, ac_nir_lower_ps, &options); } NIR_PASS(_, stage->nir, nir_opt_idiv_const, 8); -- 2.7.4