From fd1480ec75c7b783264b0526e38fbbf2e62a3b6e Mon Sep 17 00:00:00 2001 From: Egor Bogatov Date: Tue, 10 Mar 2020 11:49:05 +0300 Subject: [PATCH] [Mono] Intrinsify Interlocked.And and Interlocked.Or using LLVM (#33253) * Intrinsify Interlocked.And and Interlocked.Or --- src/mono/mono/mini/intrinsics.c | 36 ++++++++++++++++++++++++++---------- src/mono/mono/mini/mini-llvm-cpp.cpp | 6 ++++++ src/mono/mono/mini/mini-llvm-cpp.h | 2 ++ src/mono/mono/mini/mini-llvm.c | 21 +++++++++++++++++---- src/mono/mono/mini/mini-ops.h | 4 ++++ 5 files changed, 55 insertions(+), 14 deletions(-) diff --git a/src/mono/mono/mini/intrinsics.c b/src/mono/mono/mini/intrinsics.c index d9c7a38..aa59329 100644 --- a/src/mono/mono/mini/intrinsics.c +++ b/src/mono/mono/mini/intrinsics.c @@ -1186,26 +1186,42 @@ mini_emit_inst_for_method (MonoCompile *cfg, MonoMethod *cmethod, MonoMethodSign ins->type = (opcode == OP_ATOMIC_ADD_I4) ? STACK_I4 : STACK_I8; MONO_ADD_INS (cfg->cbb, ins); } - } else if (strcmp (cmethod->name, "Add") == 0 && fsig->param_count == 2) { + } else if (fsig->param_count == 2 && + ((strcmp (cmethod->name, "Add") == 0) || + (strcmp (cmethod->name, "And") == 0) || + (strcmp (cmethod->name, "Or") == 0))) { guint32 opcode = 0; + guint32 opcode_i4 = 0; + guint32 opcode_i8 = 0; + + if (strcmp (cmethod->name, "Add") == 0) { + opcode_i4 = OP_ATOMIC_ADD_I4; + opcode_i8 = OP_ATOMIC_ADD_I8; + } else if (strcmp (cmethod->name, "And") == 0) { + opcode_i4 = OP_ATOMIC_AND_I4; + opcode_i8 = OP_ATOMIC_AND_I8; + } else if (strcmp (cmethod->name, "Or") == 0) { + opcode_i4 = OP_ATOMIC_OR_I4; + opcode_i8 = OP_ATOMIC_OR_I8; + } else { + g_assert_not_reached (); + } if (fsig->params [0]->type == MONO_TYPE_I4) { - opcode = OP_ATOMIC_ADD_I4; + opcode = opcode_i4; cfg->has_atomic_add_i4 = TRUE; + } else if (fsig->params [0]->type == MONO_TYPE_I8 && SIZEOF_REGISTER == 8) { + opcode = opcode_i8; } -#if SIZEOF_REGISTER == 8 - else if (fsig->params [0]->type == MONO_TYPE_I8) - opcode = OP_ATOMIC_ADD_I8; -#endif - if (opcode) { - if (!mono_arch_opcode_supported (opcode)) - return NULL; + + // For now, only Add is supported in non-LLVM back-ends + if (opcode && (COMPILE_LLVM (cfg) || mono_arch_opcode_supported (opcode))) { MONO_INST_NEW (cfg, ins, opcode); ins->dreg = mono_alloc_ireg (cfg); ins->inst_basereg = args [0]->dreg; ins->inst_offset = 0; ins->sreg2 = args [1]->dreg; - ins->type = (opcode == OP_ATOMIC_ADD_I4) ? STACK_I4 : STACK_I8; + ins->type = (opcode == opcode_i4) ? STACK_I4 : STACK_I8; MONO_ADD_INS (cfg->cbb, ins); } } diff --git a/src/mono/mono/mini/mini-llvm-cpp.cpp b/src/mono/mono/mini/mini-llvm-cpp.cpp index b4524b7..e3f8dc0 100644 --- a/src/mono/mono/mini/mini-llvm-cpp.cpp +++ b/src/mono/mono/mini/mini-llvm-cpp.cpp @@ -174,6 +174,12 @@ mono_llvm_build_atomic_rmw (LLVMBuilderRef builder, AtomicRMWOp op, LLVMValueRef case LLVM_ATOMICRMW_OP_ADD: aop = AtomicRMWInst::Add; break; + case LLVM_ATOMICRMW_OP_AND: + aop = AtomicRMWInst::And; + break; + case LLVM_ATOMICRMW_OP_OR: + aop = AtomicRMWInst::Or; + break; default: g_assert_not_reached (); break; diff --git a/src/mono/mono/mini/mini-llvm-cpp.h b/src/mono/mono/mini/mini-llvm-cpp.h index 500fbb2..d9b2e42 100644 --- a/src/mono/mono/mini/mini-llvm-cpp.h +++ b/src/mono/mono/mini/mini-llvm-cpp.h @@ -42,6 +42,8 @@ typedef enum { typedef enum { LLVM_ATOMICRMW_OP_XCHG = 0, LLVM_ATOMICRMW_OP_ADD = 1, + LLVM_ATOMICRMW_OP_AND = 2, + LLVM_ATOMICRMW_OP_OR = 3, } AtomicRMWOp; typedef enum { diff --git a/src/mono/mono/mini/mini-llvm.c b/src/mono/mono/mini/mini-llvm.c index 137cf15..b9ea744 100644 --- a/src/mono/mono/mini/mini-llvm.c +++ b/src/mono/mono/mini/mini-llvm.c @@ -6315,11 +6315,15 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) break; } case OP_ATOMIC_ADD_I4: - case OP_ATOMIC_ADD_I8: { + case OP_ATOMIC_ADD_I8: + case OP_ATOMIC_AND_I4: + case OP_ATOMIC_AND_I8: + case OP_ATOMIC_OR_I4: + case OP_ATOMIC_OR_I8: { LLVMValueRef args [2]; LLVMTypeRef t; - - if (ins->opcode == OP_ATOMIC_ADD_I4) + + if (ins->type == STACK_I4) t = LLVMInt32Type (); else t = LLVMInt64Type (); @@ -6329,7 +6333,16 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) args [0] = convert (ctx, lhs, LLVMPointerType (t, 0)); args [1] = convert (ctx, rhs, t); ARM64_ATOMIC_FENCE_FIX; - values [ins->dreg] = LLVMBuildAdd (builder, mono_llvm_build_atomic_rmw (builder, LLVM_ATOMICRMW_OP_ADD, args [0], args [1]), args [1], dname); + if (ins->opcode == OP_ATOMIC_ADD_I4 || ins->opcode == OP_ATOMIC_ADD_I8) + // Interlocked.Add returns new value (that's why we emit additional Add here) + // see https://github.com/dotnet/runtime/pull/33102 + values [ins->dreg] = LLVMBuildAdd (builder, mono_llvm_build_atomic_rmw (builder, LLVM_ATOMICRMW_OP_ADD, args [0], args [1]), args [1], dname); + else if (ins->opcode == OP_ATOMIC_AND_I4 || ins->opcode == OP_ATOMIC_AND_I8) + values [ins->dreg] = mono_llvm_build_atomic_rmw (builder, LLVM_ATOMICRMW_OP_AND, args [0], args [1]); + else if (ins->opcode == OP_ATOMIC_OR_I4 || ins->opcode == OP_ATOMIC_OR_I8) + values [ins->dreg] = mono_llvm_build_atomic_rmw (builder, LLVM_ATOMICRMW_OP_OR, args [0], args [1]); + else + g_assert_not_reached (); ARM64_ATOMIC_FENCE_FIX; break; } diff --git a/src/mono/mono/mini/mini-ops.h b/src/mono/mono/mini/mini-ops.h index e0cd478..d205ae5 100644 --- a/src/mono/mono/mini/mini-ops.h +++ b/src/mono/mono/mini/mini-ops.h @@ -1140,6 +1140,10 @@ MINI_OP(OP_ATOMIC_STORE_R8, "atomic_store_r8", IREG, FREG, NONE) MINI_OP(OP_ATOMIC_ADD_I4, "atomic_add_i4", IREG, IREG, IREG) MINI_OP(OP_ATOMIC_ADD_I8, "atomic_add_i8", IREG, IREG, IREG) +MINI_OP(OP_ATOMIC_AND_I4, "atomic_and_i4", IREG, IREG, IREG) +MINI_OP(OP_ATOMIC_AND_I8, "atomic_and_i8", IREG, IREG, IREG) +MINI_OP(OP_ATOMIC_OR_I4, "atomic_or_i4", IREG, IREG, IREG) +MINI_OP(OP_ATOMIC_OR_I8, "atomic_or_i8", IREG, IREG, IREG) MINI_OP(OP_ATOMIC_EXCHANGE_I4, "atomic_exchange_i4", IREG, IREG, IREG) MINI_OP(OP_ATOMIC_EXCHANGE_I8, "atomic_exchange_i8", IREG, IREG, IREG) -- 2.7.4