return Changed;
}
- bool changeToSPMDMode(Attributor &A, ChangeStatus &Changed) {
- if (!mayContainParallelRegion())
- return false;
-
+ void insertInstructionGuardsHelper(Attributor &A) {
auto &OMPInfoCache = static_cast<OMPInformationCache &>(A.getInfoCache());
- if (!SPMDCompatibilityTracker.isAssumed()) {
- for (Instruction *NonCompatibleI : SPMDCompatibilityTracker) {
- if (!NonCompatibleI)
- continue;
-
- // Skip diagnostics on calls to known OpenMP runtime functions for now.
- if (auto *CB = dyn_cast<CallBase>(NonCompatibleI))
- if (OMPInfoCache.RTLFunctions.contains(CB->getCalledFunction()))
- continue;
-
- auto Remark = [&](OptimizationRemarkAnalysis ORA) {
- ORA << "Value has potential side effects preventing SPMD-mode "
- "execution";
- if (isa<CallBase>(NonCompatibleI)) {
- ORA << ". Add `__attribute__((assume(\"ompx_spmd_amenable\")))` to "
- "the called function to override";
- }
- return ORA << ".";
- };
- A.emitRemark<OptimizationRemarkAnalysis>(NonCompatibleI, "OMP121",
- Remark);
-
- LLVM_DEBUG(dbgs() << TAG << "SPMD-incompatible side-effect: "
- << *NonCompatibleI << "\n");
- }
-
- return false;
- }
-
- // Get the actual kernel, could be the caller of the anchor scope if we have
- // a debug wrapper.
- Function *Kernel = getAnchorScope();
- if (Kernel->hasLocalLinkage()) {
- assert(Kernel->hasOneUse() && "Unexpected use of debug kernel wrapper.");
- auto *CB = cast<CallBase>(Kernel->user_back());
- Kernel = CB->getCaller();
- }
- assert(OMPInfoCache.Kernels.count(Kernel) && "Expected kernel function!");
-
- // Check if the kernel is already in SPMD mode, if so, return success.
- GlobalVariable *ExecMode = Kernel->getParent()->getGlobalVariable(
- (Kernel->getName() + "_exec_mode").str());
- assert(ExecMode && "Kernel without exec mode?");
- assert(ExecMode->getInitializer() && "ExecMode doesn't have initializer!");
-
- // Set the global exec mode flag to indicate SPMD-Generic mode.
- assert(isa<ConstantInt>(ExecMode->getInitializer()) &&
- "ExecMode is not an integer!");
- const int8_t ExecModeVal =
- cast<ConstantInt>(ExecMode->getInitializer())->getSExtValue();
- if (ExecModeVal != OMP_TGT_EXEC_MODE_GENERIC)
- return true;
-
- // We will now unconditionally modify the IR, indicate a change.
- Changed = ChangeStatus::CHANGED;
-
auto CreateGuardedRegion = [&](Instruction *RegionStartI,
Instruction *RegionEndI) {
LoopInfo *LI = nullptr;
for (auto &GR : GuardedRegions)
CreateGuardedRegion(GR.first, GR.second);
+ }
+
+ void forceSingleThreadPerWorkgroupHelper(Attributor &A) {
+ // Only allow 1 thread per workgroup to continue executing the user code.
+ //
+ // InitCB = __kmpc_target_init(...)
+ // ThreadIdInBlock = __kmpc_get_hardware_thread_id_in_block();
+ // if (ThreadIdInBlock != 0) return;
+ // UserCode:
+ // // user code
+ //
+ auto &Ctx = getAnchorValue().getContext();
+ Function *Kernel = getAssociatedFunction();
+ assert(Kernel && "Expected an associated function!");
+
+ // Create block for user code to branch to from initial block.
+ BasicBlock *InitBB = KernelInitCB->getParent();
+ BasicBlock *UserCodeBB = InitBB->splitBasicBlock(
+ KernelInitCB->getNextNode(), "main.thread.user_code");
+ BasicBlock *ReturnBB =
+ BasicBlock::Create(Ctx, "exit.threads", Kernel, UserCodeBB);
+
+ // Register blocks with attributor:
+ A.registerManifestAddedBasicBlock(*InitBB);
+ A.registerManifestAddedBasicBlock(*UserCodeBB);
+ A.registerManifestAddedBasicBlock(*ReturnBB);
+
+ // Debug location:
+ const DebugLoc &DLoc = KernelInitCB->getDebugLoc();
+ ReturnInst::Create(Ctx, ReturnBB)->setDebugLoc(DLoc);
+ InitBB->getTerminator()->eraseFromParent();
+
+ // Prepare call to OMPRTL___kmpc_get_hardware_thread_id_in_block.
+ Module &M = *Kernel->getParent();
+ auto &OMPInfoCache = static_cast<OMPInformationCache &>(A.getInfoCache());
+ FunctionCallee ThreadIdInBlockFn =
+ OMPInfoCache.OMPBuilder.getOrCreateRuntimeFunction(
+ M, OMPRTL___kmpc_get_hardware_thread_id_in_block);
+
+ // Get thread ID in block.
+ CallInst *ThreadIdInBlock =
+ CallInst::Create(ThreadIdInBlockFn, "thread_id.in.block", InitBB);
+ OMPInfoCache.setCallingConvention(ThreadIdInBlockFn, ThreadIdInBlock);
+ ThreadIdInBlock->setDebugLoc(DLoc);
+
+ // Eliminate all threads in the block with ID not equal to 0:
+ Instruction *IsMainThread =
+ ICmpInst::Create(ICmpInst::ICmp, CmpInst::ICMP_NE, ThreadIdInBlock,
+ ConstantInt::get(ThreadIdInBlock->getType(), 0),
+ "thread.is_main", InitBB);
+ IsMainThread->setDebugLoc(DLoc);
+ BranchInst::Create(ReturnBB, UserCodeBB, IsMainThread, InitBB);
+ }
+
+ bool changeToSPMDMode(Attributor &A, ChangeStatus &Changed) {
+ auto &OMPInfoCache = static_cast<OMPInformationCache &>(A.getInfoCache());
+
+ if (!SPMDCompatibilityTracker.isAssumed()) {
+ for (Instruction *NonCompatibleI : SPMDCompatibilityTracker) {
+ if (!NonCompatibleI)
+ continue;
+
+ // Skip diagnostics on calls to known OpenMP runtime functions for now.
+ if (auto *CB = dyn_cast<CallBase>(NonCompatibleI))
+ if (OMPInfoCache.RTLFunctions.contains(CB->getCalledFunction()))
+ continue;
+
+ auto Remark = [&](OptimizationRemarkAnalysis ORA) {
+ ORA << "Value has potential side effects preventing SPMD-mode "
+ "execution";
+ if (isa<CallBase>(NonCompatibleI)) {
+ ORA << ". Add `__attribute__((assume(\"ompx_spmd_amenable\")))` to "
+ "the called function to override";
+ }
+ return ORA << ".";
+ };
+ A.emitRemark<OptimizationRemarkAnalysis>(NonCompatibleI, "OMP121",
+ Remark);
+
+ LLVM_DEBUG(dbgs() << TAG << "SPMD-incompatible side-effect: "
+ << *NonCompatibleI << "\n");
+ }
+
+ return false;
+ }
+
+ // Get the actual kernel, could be the caller of the anchor scope if we have
+ // a debug wrapper.
+ Function *Kernel = getAnchorScope();
+ if (Kernel->hasLocalLinkage()) {
+ assert(Kernel->hasOneUse() && "Unexpected use of debug kernel wrapper.");
+ auto *CB = cast<CallBase>(Kernel->user_back());
+ Kernel = CB->getCaller();
+ }
+ assert(OMPInfoCache.Kernels.count(Kernel) && "Expected kernel function!");
+
+ // Check if the kernel is already in SPMD mode, if so, return success.
+ GlobalVariable *ExecMode = Kernel->getParent()->getGlobalVariable(
+ (Kernel->getName() + "_exec_mode").str());
+ assert(ExecMode && "Kernel without exec mode?");
+ assert(ExecMode->getInitializer() && "ExecMode doesn't have initializer!");
+
+ // Set the global exec mode flag to indicate SPMD-Generic mode.
+ assert(isa<ConstantInt>(ExecMode->getInitializer()) &&
+ "ExecMode is not an integer!");
+ const int8_t ExecModeVal =
+ cast<ConstantInt>(ExecMode->getInitializer())->getSExtValue();
+ if (ExecModeVal != OMP_TGT_EXEC_MODE_GENERIC)
+ return true;
+
+ // We will now unconditionally modify the IR, indicate a change.
+ Changed = ChangeStatus::CHANGED;
+
+ // Do not use instruction guards when no parallel is present inside
+ // the target region.
+ if (mayContainParallelRegion())
+ insertInstructionGuardsHelper(A);
+ else
+ forceSingleThreadPerWorkgroupHelper(A);
// Adjust the global exec mode flag that tells the runtime what mode this
// kernel is executed in.
ReachedUnknownParallelRegions.indicateOptimisticFixpoint();
}
- // If we are sure there are no parallel regions in the kernel we do not
- // want SPMD mode.
- if (IsKernelEntry && ReachedUnknownParallelRegions.isAtFixpoint() &&
- ReachedKnownParallelRegions.isAtFixpoint() &&
- ReachedUnknownParallelRegions.isValidState() &&
- ReachedKnownParallelRegions.isValidState() &&
- !mayContainParallelRegion())
- SPMDCompatibilityTracker.indicatePessimisticFixpoint();
-
// If we haven't used any assumed information for the SPMD state we can fix
// it.
if (!UsedAssumedInformationInCheckRWInst &&
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --check-globals
-; RUN: opt -S -passes=openmp-opt < %s | FileCheck %s
-; RUN: opt -S -passes=openmp-opt -openmp-opt-disable-spmdization < %s | FileCheck %s
+; RUN: opt -S -passes=openmp-opt < %s | FileCheck %s --check-prefixes=CHECK
+; RUN: opt -S -passes=openmp-opt -openmp-opt-disable-spmdization < %s | FileCheck %s --check-prefixes=CHECK-DISABLE-SPMDIZATION
;
; int G;
;
;.
; CHECK: @[[GLOB0:[0-9]+]] = private unnamed_addr constant [23 x i8] c"
; CHECK: @[[GLOB1:[0-9]+]] = private unnamed_addr constant [[STRUCT_IDENT_T:%.*]] { i32 0, i32 2, i32 0, i32 0, i8* getelementptr inbounds ([23 x i8], [23 x i8]* @[[GLOB0]], i32 0, i32 0) }, align 8
-; CHECK: @[[__OMP_OFFLOADING_2B_10393B5_SPMD_L12_EXEC_MODE:[a-zA-Z0-9_$"\\.-]+]] = weak constant i8 1
-; CHECK: @[[__OMP_OFFLOADING_2B_10393B5_GENERIC_L20_EXEC_MODE:[a-zA-Z0-9_$"\\.-]+]] = weak constant i8 1
+; CHECK: @[[__OMP_OFFLOADING_2B_10393B5_SPMD_L12_EXEC_MODE:[a-zA-Z0-9_$"\\.-]+]] = weak constant i8 3
+; CHECK: @[[__OMP_OFFLOADING_2B_10393B5_GENERIC_L20_EXEC_MODE:[a-zA-Z0-9_$"\\.-]+]] = weak constant i8 3
; CHECK: @[[GLOB2:[0-9]+]] = private unnamed_addr constant [[STRUCT_IDENT_T:%.*]] { i32 0, i32 2, i32 2, i32 0, i8* getelementptr inbounds ([23 x i8], [23 x i8]* @[[GLOB0]], i32 0, i32 0) }, align 8
; CHECK: @[[G:[a-zA-Z0-9_$"\\.-]+]] = external global i32, align 4
; CHECK: @[[LLVM_COMPILER_USED:[a-zA-Z0-9_$"\\.-]+]] = appending global [2 x i8*] [i8* @__omp_offloading_2b_10393b5_spmd_l12_exec_mode, i8* @__omp_offloading_2b_10393b5_generic_l20_exec_mode], section "llvm.metadata"
-; CHECK: @[[__OMP_OUTLINED___WRAPPER_ID:[a-zA-Z0-9_$"\\.-]+]] = private constant i8 undef
+; CHECK: @[[GLOB3:[0-9]+]] = private unnamed_addr constant [[STRUCT_IDENT_T:%.*]] { i32 0, i32 2, i32 0, i32 22, i8* getelementptr inbounds ([23 x i8], [23 x i8]* @[[GLOB0]], i32 0, i32 0) }, align 8
+;.
+; CHECK-DISABLE-SPMDIZATION: @[[GLOB0:[0-9]+]] = private unnamed_addr constant [23 x i8] c"
+; CHECK-DISABLE-SPMDIZATION: @[[GLOB1:[0-9]+]] = private unnamed_addr constant [[STRUCT_IDENT_T:%.*]] { i32 0, i32 2, i32 0, i32 0, i8* getelementptr inbounds ([23 x i8], [23 x i8]* @[[GLOB0]], i32 0, i32 0) }, align 8
+; CHECK-DISABLE-SPMDIZATION: @[[__OMP_OFFLOADING_2B_10393B5_SPMD_L12_EXEC_MODE:[a-zA-Z0-9_$"\\.-]+]] = weak constant i8 1
+; CHECK-DISABLE-SPMDIZATION: @[[__OMP_OFFLOADING_2B_10393B5_GENERIC_L20_EXEC_MODE:[a-zA-Z0-9_$"\\.-]+]] = weak constant i8 1
+; CHECK-DISABLE-SPMDIZATION: @[[GLOB2:[0-9]+]] = private unnamed_addr constant [[STRUCT_IDENT_T:%.*]] { i32 0, i32 2, i32 2, i32 0, i8* getelementptr inbounds ([23 x i8], [23 x i8]* @[[GLOB0]], i32 0, i32 0) }, align 8
+; CHECK-DISABLE-SPMDIZATION: @[[G:[a-zA-Z0-9_$"\\.-]+]] = external global i32, align 4
+; CHECK-DISABLE-SPMDIZATION: @[[LLVM_COMPILER_USED:[a-zA-Z0-9_$"\\.-]+]] = appending global [2 x i8*] [i8* @__omp_offloading_2b_10393b5_spmd_l12_exec_mode, i8* @__omp_offloading_2b_10393b5_generic_l20_exec_mode], section "llvm.metadata"
+; CHECK-DISABLE-SPMDIZATION: @[[__OMP_OUTLINED___WRAPPER_ID:[a-zA-Z0-9_$"\\.-]+]] = private constant i8 undef
;.
define weak void @__omp_offloading_2b_10393b5_spmd_l12() #0 {
; CHECK-LABEL: define {{[^@]+}}@__omp_offloading_2b_10393b5_spmd_l12
; CHECK-SAME: () #[[ATTR0:[0-9]+]] {
; CHECK-NEXT: entry:
-; CHECK-NEXT: [[WORKER_WORK_FN_ADDR:%.*]] = alloca i8*, align 8
-; CHECK-NEXT: [[TMP0:%.*]] = call i32 @__kmpc_target_init(%struct.ident_t* @[[GLOB1]], i8 1, i1 false, i1 true)
-; CHECK-NEXT: [[THREAD_IS_WORKER:%.*]] = icmp ne i32 [[TMP0]], -1
-; CHECK-NEXT: br i1 [[THREAD_IS_WORKER]], label [[IS_WORKER_CHECK:%.*]], label [[THREAD_USER_CODE_CHECK:%.*]]
-; CHECK: is_worker_check:
-; CHECK-NEXT: [[BLOCK_HW_SIZE:%.*]] = call i32 @__kmpc_get_hardware_num_threads_in_block()
-; CHECK-NEXT: [[WARP_SIZE:%.*]] = call i32 @__kmpc_get_warp_size()
-; CHECK-NEXT: [[BLOCK_SIZE:%.*]] = sub i32 [[BLOCK_HW_SIZE]], [[WARP_SIZE]]
-; CHECK-NEXT: [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[BLOCK_SIZE]]
-; CHECK-NEXT: br i1 [[THREAD_IS_MAIN_OR_WORKER]], label [[WORKER_STATE_MACHINE_BEGIN:%.*]], label [[WORKER_STATE_MACHINE_FINISHED:%.*]]
-; CHECK: worker_state_machine.begin:
-; CHECK-NEXT: call void @__kmpc_barrier_simple_generic(%struct.ident_t* @[[GLOB1]], i32 [[TMP0]])
-; CHECK-NEXT: [[WORKER_IS_ACTIVE:%.*]] = call i1 @__kmpc_kernel_parallel(i8** [[WORKER_WORK_FN_ADDR]])
-; CHECK-NEXT: [[WORKER_WORK_FN:%.*]] = load i8*, i8** [[WORKER_WORK_FN_ADDR]], align 8
-; CHECK-NEXT: [[WORKER_WORK_FN_ADDR_CAST:%.*]] = bitcast i8* [[WORKER_WORK_FN]] to void (i16, i32)*
-; CHECK-NEXT: [[WORKER_IS_DONE:%.*]] = icmp eq i8* [[WORKER_WORK_FN]], null
-; CHECK-NEXT: br i1 [[WORKER_IS_DONE]], label [[WORKER_STATE_MACHINE_FINISHED]], label [[WORKER_STATE_MACHINE_IS_ACTIVE_CHECK:%.*]]
-; CHECK: worker_state_machine.finished:
-; CHECK-NEXT: ret void
-; CHECK: worker_state_machine.is_active.check:
-; CHECK-NEXT: br i1 [[WORKER_IS_ACTIVE]], label [[WORKER_STATE_MACHINE_PARALLEL_REGION_CHECK:%.*]], label [[WORKER_STATE_MACHINE_DONE_BARRIER:%.*]]
-; CHECK: worker_state_machine.parallel_region.check:
-; CHECK-NEXT: br i1 true, label [[WORKER_STATE_MACHINE_PARALLEL_REGION_EXECUTE:%.*]], label [[WORKER_STATE_MACHINE_PARALLEL_REGION_CHECK1:%.*]]
-; CHECK: worker_state_machine.parallel_region.execute:
-; CHECK-NEXT: call void @__omp_outlined___wrapper(i16 0, i32 [[TMP0]])
-; CHECK-NEXT: br label [[WORKER_STATE_MACHINE_PARALLEL_REGION_END:%.*]]
-; CHECK: worker_state_machine.parallel_region.check1:
-; CHECK-NEXT: br label [[WORKER_STATE_MACHINE_PARALLEL_REGION_END]]
-; CHECK: worker_state_machine.parallel_region.end:
-; CHECK-NEXT: call void @__kmpc_kernel_end_parallel()
-; CHECK-NEXT: br label [[WORKER_STATE_MACHINE_DONE_BARRIER]]
-; CHECK: worker_state_machine.done.barrier:
-; CHECK-NEXT: call void @__kmpc_barrier_simple_generic(%struct.ident_t* @[[GLOB1]], i32 [[TMP0]])
-; CHECK-NEXT: br label [[WORKER_STATE_MACHINE_BEGIN]]
-; CHECK: thread.user_code.check:
+; CHECK-NEXT: [[TMP0:%.*]] = call i32 @__kmpc_target_init(%struct.ident_t* @[[GLOB1]], i8 2, i1 false, i1 false)
; CHECK-NEXT: [[EXEC_USER_CODE:%.*]] = icmp eq i32 [[TMP0]], -1
; CHECK-NEXT: br i1 [[EXEC_USER_CODE]], label [[USER_CODE_ENTRY:%.*]], label [[WORKER_EXIT:%.*]]
; CHECK: user_code.entry:
; CHECK-NEXT: call void @spmd_helper() #[[ATTR5:[0-9]+]]
-; CHECK-NEXT: call void @__kmpc_target_deinit(%struct.ident_t* @[[GLOB1]], i8 1, i1 true)
+; CHECK-NEXT: call void @__kmpc_target_deinit(%struct.ident_t* @[[GLOB1]], i8 2, i1 false)
; CHECK-NEXT: ret void
; CHECK: worker.exit:
; CHECK-NEXT: ret void
;
+; CHECK-DISABLE-SPMDIZATION-LABEL: define {{[^@]+}}@__omp_offloading_2b_10393b5_spmd_l12
+; CHECK-DISABLE-SPMDIZATION-SAME: () #[[ATTR0:[0-9]+]] {
+; CHECK-DISABLE-SPMDIZATION-NEXT: entry:
+; CHECK-DISABLE-SPMDIZATION-NEXT: [[WORKER_WORK_FN_ADDR:%.*]] = alloca i8*, align 8
+; CHECK-DISABLE-SPMDIZATION-NEXT: [[TMP0:%.*]] = call i32 @__kmpc_target_init(%struct.ident_t* @[[GLOB1]], i8 1, i1 false, i1 true)
+; CHECK-DISABLE-SPMDIZATION-NEXT: [[THREAD_IS_WORKER:%.*]] = icmp ne i32 [[TMP0]], -1
+; CHECK-DISABLE-SPMDIZATION-NEXT: br i1 [[THREAD_IS_WORKER]], label [[IS_WORKER_CHECK:%.*]], label [[THREAD_USER_CODE_CHECK:%.*]]
+; CHECK-DISABLE-SPMDIZATION: is_worker_check:
+; CHECK-DISABLE-SPMDIZATION-NEXT: [[BLOCK_HW_SIZE:%.*]] = call i32 @__kmpc_get_hardware_num_threads_in_block()
+; CHECK-DISABLE-SPMDIZATION-NEXT: [[WARP_SIZE:%.*]] = call i32 @__kmpc_get_warp_size()
+; CHECK-DISABLE-SPMDIZATION-NEXT: [[BLOCK_SIZE:%.*]] = sub i32 [[BLOCK_HW_SIZE]], [[WARP_SIZE]]
+; CHECK-DISABLE-SPMDIZATION-NEXT: [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[BLOCK_SIZE]]
+; CHECK-DISABLE-SPMDIZATION-NEXT: br i1 [[THREAD_IS_MAIN_OR_WORKER]], label [[WORKER_STATE_MACHINE_BEGIN:%.*]], label [[WORKER_STATE_MACHINE_FINISHED:%.*]]
+; CHECK-DISABLE-SPMDIZATION: worker_state_machine.begin:
+; CHECK-DISABLE-SPMDIZATION-NEXT: call void @__kmpc_barrier_simple_generic(%struct.ident_t* @[[GLOB1]], i32 [[TMP0]])
+; CHECK-DISABLE-SPMDIZATION-NEXT: [[WORKER_IS_ACTIVE:%.*]] = call i1 @__kmpc_kernel_parallel(i8** [[WORKER_WORK_FN_ADDR]])
+; CHECK-DISABLE-SPMDIZATION-NEXT: [[WORKER_WORK_FN:%.*]] = load i8*, i8** [[WORKER_WORK_FN_ADDR]], align 8
+; CHECK-DISABLE-SPMDIZATION-NEXT: [[WORKER_WORK_FN_ADDR_CAST:%.*]] = bitcast i8* [[WORKER_WORK_FN]] to void (i16, i32)*
+; CHECK-DISABLE-SPMDIZATION-NEXT: [[WORKER_IS_DONE:%.*]] = icmp eq i8* [[WORKER_WORK_FN]], null
+; CHECK-DISABLE-SPMDIZATION-NEXT: br i1 [[WORKER_IS_DONE]], label [[WORKER_STATE_MACHINE_FINISHED]], label [[WORKER_STATE_MACHINE_IS_ACTIVE_CHECK:%.*]]
+; CHECK-DISABLE-SPMDIZATION: worker_state_machine.finished:
+; CHECK-DISABLE-SPMDIZATION-NEXT: ret void
+; CHECK-DISABLE-SPMDIZATION: worker_state_machine.is_active.check:
+; CHECK-DISABLE-SPMDIZATION-NEXT: br i1 [[WORKER_IS_ACTIVE]], label [[WORKER_STATE_MACHINE_PARALLEL_REGION_CHECK:%.*]], label [[WORKER_STATE_MACHINE_DONE_BARRIER:%.*]]
+; CHECK-DISABLE-SPMDIZATION: worker_state_machine.parallel_region.check:
+; CHECK-DISABLE-SPMDIZATION-NEXT: br i1 true, label [[WORKER_STATE_MACHINE_PARALLEL_REGION_EXECUTE:%.*]], label [[WORKER_STATE_MACHINE_PARALLEL_REGION_CHECK1:%.*]]
+; CHECK-DISABLE-SPMDIZATION: worker_state_machine.parallel_region.execute:
+; CHECK-DISABLE-SPMDIZATION-NEXT: call void @__omp_outlined___wrapper(i16 0, i32 [[TMP0]])
+; CHECK-DISABLE-SPMDIZATION-NEXT: br label [[WORKER_STATE_MACHINE_PARALLEL_REGION_END:%.*]]
+; CHECK-DISABLE-SPMDIZATION: worker_state_machine.parallel_region.check1:
+; CHECK-DISABLE-SPMDIZATION-NEXT: br label [[WORKER_STATE_MACHINE_PARALLEL_REGION_END]]
+; CHECK-DISABLE-SPMDIZATION: worker_state_machine.parallel_region.end:
+; CHECK-DISABLE-SPMDIZATION-NEXT: call void @__kmpc_kernel_end_parallel()
+; CHECK-DISABLE-SPMDIZATION-NEXT: br label [[WORKER_STATE_MACHINE_DONE_BARRIER]]
+; CHECK-DISABLE-SPMDIZATION: worker_state_machine.done.barrier:
+; CHECK-DISABLE-SPMDIZATION-NEXT: call void @__kmpc_barrier_simple_generic(%struct.ident_t* @[[GLOB1]], i32 [[TMP0]])
+; CHECK-DISABLE-SPMDIZATION-NEXT: br label [[WORKER_STATE_MACHINE_BEGIN]]
+; CHECK-DISABLE-SPMDIZATION: thread.user_code.check:
+; CHECK-DISABLE-SPMDIZATION-NEXT: [[EXEC_USER_CODE:%.*]] = icmp eq i32 [[TMP0]], -1
+; CHECK-DISABLE-SPMDIZATION-NEXT: br i1 [[EXEC_USER_CODE]], label [[USER_CODE_ENTRY:%.*]], label [[WORKER_EXIT:%.*]]
+; CHECK-DISABLE-SPMDIZATION: user_code.entry:
+; CHECK-DISABLE-SPMDIZATION-NEXT: call void @spmd_helper() #[[ATTR5:[0-9]+]]
+; CHECK-DISABLE-SPMDIZATION-NEXT: call void @__kmpc_target_deinit(%struct.ident_t* @[[GLOB1]], i8 1, i1 true)
+; CHECK-DISABLE-SPMDIZATION-NEXT: ret void
+; CHECK-DISABLE-SPMDIZATION: worker.exit:
+; CHECK-DISABLE-SPMDIZATION-NEXT: ret void
+;
entry:
%0 = call i32 @__kmpc_target_init(%struct.ident_t* @1, i8 1, i1 true, i1 true)
%exec_user_code = icmp eq i32 %0, -1
; CHECK-LABEL: define {{[^@]+}}@__omp_offloading_2b_10393b5_generic_l20
; CHECK-SAME: () #[[ATTR0]] {
; CHECK-NEXT: entry:
-; CHECK-NEXT: [[TMP0:%.*]] = call i32 @__kmpc_target_init(%struct.ident_t* @[[GLOB1]], i8 1, i1 false, i1 true)
+; CHECK-NEXT: [[TMP0:%.*]] = call i32 @__kmpc_target_init(%struct.ident_t* @[[GLOB1]], i8 2, i1 false, i1 false)
+; CHECK-NEXT: [[THREAD_ID_IN_BLOCK:%.*]] = call i32 @__kmpc_get_hardware_thread_id_in_block()
+; CHECK-NEXT: [[THREAD_IS_MAIN:%.*]] = icmp ne i32 [[THREAD_ID_IN_BLOCK]], 0
+; CHECK-NEXT: br i1 [[THREAD_IS_MAIN]], label [[EXIT_THREADS:%.*]], label [[MAIN_THREAD_USER_CODE:%.*]]
+; CHECK: exit.threads:
+; CHECK-NEXT: ret void
+; CHECK: main.thread.user_code:
; CHECK-NEXT: [[EXEC_USER_CODE:%.*]] = icmp eq i32 [[TMP0]], -1
; CHECK-NEXT: br i1 [[EXEC_USER_CODE]], label [[USER_CODE_ENTRY:%.*]], label [[WORKER_EXIT:%.*]]
; CHECK: user_code.entry:
; CHECK-NEXT: call void @generic_helper() #[[ATTR6:[0-9]+]]
-; CHECK-NEXT: call void @__kmpc_target_deinit(%struct.ident_t* @[[GLOB1]], i8 1, i1 true)
+; CHECK-NEXT: call void @__kmpc_target_deinit(%struct.ident_t* @[[GLOB1]], i8 2, i1 false)
; CHECK-NEXT: ret void
; CHECK: worker.exit:
; CHECK-NEXT: ret void
;
+; CHECK-DISABLE-SPMDIZATION-LABEL: define {{[^@]+}}@__omp_offloading_2b_10393b5_generic_l20
+; CHECK-DISABLE-SPMDIZATION-SAME: () #[[ATTR0]] {
+; CHECK-DISABLE-SPMDIZATION-NEXT: entry:
+; CHECK-DISABLE-SPMDIZATION-NEXT: [[TMP0:%.*]] = call i32 @__kmpc_target_init(%struct.ident_t* @[[GLOB1]], i8 1, i1 false, i1 true)
+; CHECK-DISABLE-SPMDIZATION-NEXT: [[EXEC_USER_CODE:%.*]] = icmp eq i32 [[TMP0]], -1
+; CHECK-DISABLE-SPMDIZATION-NEXT: br i1 [[EXEC_USER_CODE]], label [[USER_CODE_ENTRY:%.*]], label [[WORKER_EXIT:%.*]]
+; CHECK-DISABLE-SPMDIZATION: user_code.entry:
+; CHECK-DISABLE-SPMDIZATION-NEXT: call void @generic_helper() #[[ATTR6:[0-9]+]]
+; CHECK-DISABLE-SPMDIZATION-NEXT: call void @__kmpc_target_deinit(%struct.ident_t* @[[GLOB1]], i8 1, i1 true)
+; CHECK-DISABLE-SPMDIZATION-NEXT: ret void
+; CHECK-DISABLE-SPMDIZATION: worker.exit:
+; CHECK-DISABLE-SPMDIZATION-NEXT: ret void
+;
entry:
%0 = call i32 @__kmpc_target_init(%struct.ident_t* @1, i8 1, i1 true, i1 true)
%exec_user_code = icmp eq i32 %0, -1
; CHECK-NEXT: call void @leaf() #[[ATTR6]]
; CHECK-NEXT: [[TMP0:%.*]] = call i32 @__kmpc_global_thread_num(%struct.ident_t* @[[GLOB2]]) #[[ATTR2:[0-9]+]]
; CHECK-NEXT: [[TMP1:%.*]] = bitcast [0 x i8*]* [[CAPTURED_VARS_ADDRS]] to i8**
-; CHECK-NEXT: call void @__kmpc_parallel_51(%struct.ident_t* @[[GLOB2]], i32 [[TMP0]], i32 1, i32 -1, i32 -1, i8* bitcast (void (i32*, i32*)* @__omp_outlined__ to i8*), i8* @__omp_outlined___wrapper.ID, i8** [[TMP1]], i64 0)
+; CHECK-NEXT: call void @__kmpc_parallel_51(%struct.ident_t* @[[GLOB2]], i32 [[TMP0]], i32 1, i32 -1, i32 -1, i8* bitcast (void (i32*, i32*)* @__omp_outlined__ to i8*), i8* bitcast (void (i16, i32)* @__omp_outlined___wrapper to i8*), i8** [[TMP1]], i64 0)
; CHECK-NEXT: ret void
;
+; CHECK-DISABLE-SPMDIZATION-LABEL: define {{[^@]+}}@spmd_helper
+; CHECK-DISABLE-SPMDIZATION-SAME: () #[[ATTR1:[0-9]+]] {
+; CHECK-DISABLE-SPMDIZATION-NEXT: entry:
+; CHECK-DISABLE-SPMDIZATION-NEXT: [[CAPTURED_VARS_ADDRS:%.*]] = alloca [0 x i8*], align 8
+; CHECK-DISABLE-SPMDIZATION-NEXT: call void @leaf() #[[ATTR6]]
+; CHECK-DISABLE-SPMDIZATION-NEXT: [[TMP0:%.*]] = call i32 @__kmpc_global_thread_num(%struct.ident_t* @[[GLOB2]]) #[[ATTR2:[0-9]+]]
+; CHECK-DISABLE-SPMDIZATION-NEXT: [[TMP1:%.*]] = bitcast [0 x i8*]* [[CAPTURED_VARS_ADDRS]] to i8**
+; CHECK-DISABLE-SPMDIZATION-NEXT: call void @__kmpc_parallel_51(%struct.ident_t* @[[GLOB2]], i32 [[TMP0]], i32 1, i32 -1, i32 -1, i8* bitcast (void (i32*, i32*)* @__omp_outlined__ to i8*), i8* @__omp_outlined___wrapper.ID, i8** [[TMP1]], i64 0)
+; CHECK-DISABLE-SPMDIZATION-NEXT: ret void
+;
entry:
%captured_vars_addrs = alloca [0 x i8*], align 8
call void @leaf() #5
; CHECK-NEXT: call void @unknown() #[[ATTR7:[0-9]+]]
; CHECK-NEXT: ret void
;
+; CHECK-DISABLE-SPMDIZATION-LABEL: define {{[^@]+}}@__omp_outlined__
+; CHECK-DISABLE-SPMDIZATION-SAME: (i32* noalias [[DOTGLOBAL_TID_:%.*]], i32* noalias [[DOTBOUND_TID_:%.*]]) #[[ATTR0]] {
+; CHECK-DISABLE-SPMDIZATION-NEXT: entry:
+; CHECK-DISABLE-SPMDIZATION-NEXT: [[DOTGLOBAL_TID__ADDR:%.*]] = alloca i32*, align 8
+; CHECK-DISABLE-SPMDIZATION-NEXT: [[DOTBOUND_TID__ADDR:%.*]] = alloca i32*, align 8
+; CHECK-DISABLE-SPMDIZATION-NEXT: call void @unknown() #[[ATTR7:[0-9]+]]
+; CHECK-DISABLE-SPMDIZATION-NEXT: ret void
+;
entry:
%.global_tid..addr = alloca i32*, align 8
%.bound_tid..addr = alloca i32*, align 8
; CHECK-NEXT: call void @__omp_outlined__(i32* [[DOTADDR1]], i32* [[DOTZERO_ADDR]]) #[[ATTR2]]
; CHECK-NEXT: ret void
;
+; CHECK-DISABLE-SPMDIZATION-LABEL: define {{[^@]+}}@__omp_outlined___wrapper
+; CHECK-DISABLE-SPMDIZATION-SAME: (i16 zeroext [[TMP0:%.*]], i32 [[TMP1:%.*]]) #[[ATTR0]] {
+; CHECK-DISABLE-SPMDIZATION-NEXT: entry:
+; CHECK-DISABLE-SPMDIZATION-NEXT: [[DOTADDR:%.*]] = alloca i16, align 2
+; CHECK-DISABLE-SPMDIZATION-NEXT: [[DOTADDR1:%.*]] = alloca i32, align 4
+; CHECK-DISABLE-SPMDIZATION-NEXT: [[DOTZERO_ADDR:%.*]] = alloca i32, align 4
+; CHECK-DISABLE-SPMDIZATION-NEXT: [[GLOBAL_ARGS:%.*]] = alloca i8**, align 8
+; CHECK-DISABLE-SPMDIZATION-NEXT: call void @__kmpc_get_shared_variables(i8*** [[GLOBAL_ARGS]])
+; CHECK-DISABLE-SPMDIZATION-NEXT: call void @__omp_outlined__(i32* [[DOTADDR1]], i32* [[DOTZERO_ADDR]]) #[[ATTR2]]
+; CHECK-DISABLE-SPMDIZATION-NEXT: ret void
+;
entry:
%.addr = alloca i16, align 2
%.addr1 = alloca i32, align 4
; CHECK-LABEL: define {{[^@]+}}@leaf
; CHECK-SAME: () #[[ATTR4:[0-9]+]] {
; CHECK-NEXT: entry:
+; CHECK-NEXT: br label [[REGION_CHECK_TID:%.*]]
+; CHECK: region.check.tid:
+; CHECK-NEXT: [[TMP0:%.*]] = call i32 @__kmpc_get_hardware_thread_id_in_block()
+; CHECK-NEXT: [[TMP1:%.*]] = icmp eq i32 [[TMP0]], 0
+; CHECK-NEXT: br i1 [[TMP1]], label [[REGION_GUARDED:%.*]], label [[REGION_BARRIER:%.*]]
+; CHECK: region.guarded:
; CHECK-NEXT: store i32 42, i32* @G, align 4
+; CHECK-NEXT: br label [[REGION_GUARDED_END:%.*]]
+; CHECK: region.guarded.end:
+; CHECK-NEXT: br label [[REGION_BARRIER]]
+; CHECK: region.barrier:
+; CHECK-NEXT: call void @__kmpc_barrier_simple_spmd(%struct.ident_t* @[[GLOB3]], i32 [[TMP0]])
+; CHECK-NEXT: br label [[REGION_EXIT:%.*]]
+; CHECK: region.exit:
; CHECK-NEXT: ret void
;
+; CHECK-DISABLE-SPMDIZATION-LABEL: define {{[^@]+}}@leaf
+; CHECK-DISABLE-SPMDIZATION-SAME: () #[[ATTR4:[0-9]+]] {
+; CHECK-DISABLE-SPMDIZATION-NEXT: entry:
+; CHECK-DISABLE-SPMDIZATION-NEXT: store i32 42, i32* @G, align 4
+; CHECK-DISABLE-SPMDIZATION-NEXT: ret void
+;
entry:
store i32 42, i32* @G, align 4
ret void
; CHECK-NEXT: call void @leaf() #[[ATTR6]]
; CHECK-NEXT: ret void
;
+; CHECK-DISABLE-SPMDIZATION-LABEL: define {{[^@]+}}@generic_helper
+; CHECK-DISABLE-SPMDIZATION-SAME: () #[[ATTR4]] {
+; CHECK-DISABLE-SPMDIZATION-NEXT: entry:
+; CHECK-DISABLE-SPMDIZATION-NEXT: call void @leaf() #[[ATTR6]]
+; CHECK-DISABLE-SPMDIZATION-NEXT: ret void
+;
entry:
call void @leaf() #5
ret void
; CHECK: attributes #[[ATTR6]] = { convergent nounwind writeonly }
; CHECK: attributes #[[ATTR7]] = { convergent }
;.
+; CHECK-DISABLE-SPMDIZATION: attributes #[[ATTR0]] = { convergent noinline norecurse nounwind "frame-pointer"="all" "min-legal-vector-width"="0" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="sm_53" "target-features"="+ptx32,+sm_53" }
+; CHECK-DISABLE-SPMDIZATION: attributes #[[ATTR1]] = { convergent noinline nounwind "frame-pointer"="all" "min-legal-vector-width"="0" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="sm_53" "target-features"="+ptx32,+sm_53" }
+; CHECK-DISABLE-SPMDIZATION: attributes #[[ATTR2]] = { nounwind }
+; CHECK-DISABLE-SPMDIZATION: attributes #[[ATTR3:[0-9]+]] = { alwaysinline }
+; CHECK-DISABLE-SPMDIZATION: attributes #[[ATTR4]] = { convergent noinline nounwind writeonly "frame-pointer"="all" "min-legal-vector-width"="0" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="sm_53" "target-features"="+ptx32,+sm_53" }
+; CHECK-DISABLE-SPMDIZATION: attributes #[[ATTR5]] = { convergent nounwind }
+; CHECK-DISABLE-SPMDIZATION: attributes #[[ATTR6]] = { convergent nounwind writeonly }
+; CHECK-DISABLE-SPMDIZATION: attributes #[[ATTR7]] = { convergent }
+;.
; CHECK: [[META0:![0-9]+]] = !{i32 0, i32 43, i32 17011637, !"spmd", i32 12, i32 0}
; CHECK: [[META1:![0-9]+]] = !{i32 0, i32 43, i32 17011637, !"generic", i32 20, i32 1}
; CHECK: [[META2:![0-9]+]] = !{void ()* @__omp_offloading_2b_10393b5_spmd_l12, !"kernel", i32 1}
; CHECK: [[META8:![0-9]+]] = !{i32 7, !"frame-pointer", i32 2}
; CHECK: [[META9:![0-9]+]] = !{!"clang version 14.0.0"}
;.
+; CHECK-DISABLE-SPMDIZATION: [[META0:![0-9]+]] = !{i32 0, i32 43, i32 17011637, !"spmd", i32 12, i32 0}
+; CHECK-DISABLE-SPMDIZATION: [[META1:![0-9]+]] = !{i32 0, i32 43, i32 17011637, !"generic", i32 20, i32 1}
+; CHECK-DISABLE-SPMDIZATION: [[META2:![0-9]+]] = !{void ()* @__omp_offloading_2b_10393b5_spmd_l12, !"kernel", i32 1}
+; CHECK-DISABLE-SPMDIZATION: [[META3:![0-9]+]] = !{void ()* @__omp_offloading_2b_10393b5_generic_l20, !"kernel", i32 1}
+; CHECK-DISABLE-SPMDIZATION: [[META4:![0-9]+]] = !{i32 1, !"wchar_size", i32 4}
+; CHECK-DISABLE-SPMDIZATION: [[META5:![0-9]+]] = !{i32 7, !"openmp", i32 50}
+; CHECK-DISABLE-SPMDIZATION: [[META6:![0-9]+]] = !{i32 7, !"openmp-device", i32 50}
+; CHECK-DISABLE-SPMDIZATION: [[META7:![0-9]+]] = !{i32 8, !"PIC Level", i32 2}
+; CHECK-DISABLE-SPMDIZATION: [[META8:![0-9]+]] = !{i32 7, !"frame-pointer", i32 2}
+; CHECK-DISABLE-SPMDIZATION: [[META9:![0-9]+]] = !{!"clang version 14.0.0"}
+;.