AMDGPU: Add pass to optimize reqd_work_group_size

Eliminate loads from the dispatch packet when they will have
a known value.

Also pattern match the code used by the library to handle partial
workgroup dispatches, which isn't necessary if reqd_work_group_size
is used.

llvm-svn: 332771

diff --git a/llvm/lib/Target/AMDGPU/AMDGPU.h b/llvm/lib/Target/AMDGPU/AMDGPU.h
index 510e935..9cc582d 100644 (file)
--- a/llvm/lib/Target/AMDGPU/AMDGPU.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPU.h
@@ -73,6 +73,10 @@ ModulePass *createAMDGPULowerIntrinsicsPass();
 void initializeAMDGPULowerIntrinsicsPass(PassRegistry &);
 extern char &AMDGPULowerIntrinsicsID;
 
+ModulePass *createAMDGPULowerKernelAttributesPass();
+void initializeAMDGPULowerKernelAttributesPass(PassRegistry &);
+extern char &AMDGPULowerKernelAttributesID;
+
 void initializeAMDGPURewriteOutArgumentsPass(PassRegistry &);
 extern char &AMDGPURewriteOutArgumentsID;
 

diff --git a/llvm/lib/Target/AMDGPU/AMDGPULowerKernelAttributes.cpp b/llvm/lib/Target/AMDGPU/AMDGPULowerKernelAttributes.cpp
new file mode 100644 (file)
index 0000000..a43dcef
--- /dev/null
+++ b/llvm/lib/Target/AMDGPU/AMDGPULowerKernelAttributes.cpp
@@ -0,0 +1,270 @@
+//===-- AMDGPULowerKernelAttributes.cpp ------------------------------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+/// \file This pass does attempts to make use of reqd_work_group_size metadata
+/// to eliminate loads from the dispatch packet and to constant fold OpenCL
+/// get_local_size-like functions.
+//
+//===----------------------------------------------------------------------===//
+
+#include "AMDGPU.h"
+#include "AMDGPUTargetMachine.h"
+#include "llvm/Analysis/ValueTracking.h"
+#include "llvm/CodeGen/Passes.h"
+#include "llvm/CodeGen/TargetPassConfig.h"
+#include "llvm/IR/Constants.h"
+#include "llvm/IR/Function.h"
+#include "llvm/IR/Instructions.h"
+#include "llvm/IR/PatternMatch.h"
+#include "llvm/Pass.h"
+
+#define DEBUG_TYPE "amdgpu-lower-kernel-attributes"
+
+using namespace llvm;
+
+namespace {
+
+// Field offsets in hsa_kernel_dispatch_packet_t.
+enum DispatchPackedOffsets {
+  WORKGROUP_SIZE_X = 4,
+  WORKGROUP_SIZE_Y = 6,
+  WORKGROUP_SIZE_Z = 8,
+
+  GRID_SIZE_X = 12,
+  GRID_SIZE_Y = 16,
+  GRID_SIZE_Z = 20
+};
+
+class AMDGPULowerKernelAttributes : public ModulePass {
+  Module *Mod = nullptr;
+
+public:
+  static char ID;
+
+  AMDGPULowerKernelAttributes() : ModulePass(ID) {}
+
+  bool processUse(CallInst *CI);
+
+  bool doInitialization(Module &M) override;
+  bool runOnModule(Module &M) override;
+
+  StringRef getPassName() const override {
+    return "AMDGPU Kernel Attributes";
+  }
+
+  void getAnalysisUsage(AnalysisUsage &AU) const override {
+    AU.setPreservesAll();
+ }
+};
+
+} // end anonymous namespace
+
+bool AMDGPULowerKernelAttributes::doInitialization(Module &M) {
+  Mod = &M;
+  return false;
+}
+
+bool AMDGPULowerKernelAttributes::processUse(CallInst *CI) {
+  Function *F = CI->getParent()->getParent();
+
+  auto MD = F->getMetadata("reqd_work_group_size");
+  const bool HasReqdWorkGroupSize = MD && MD->getNumOperands() == 3;
+
+  const bool HasUniformWorkGroupSize =
+    F->getFnAttribute("uniform-work-group-size").getValueAsString() == "true";
+
+  if (!HasReqdWorkGroupSize && !HasUniformWorkGroupSize)
+    return false;
+
+  Value *WorkGroupSizeX = nullptr;
+  Value *WorkGroupSizeY = nullptr;
+  Value *WorkGroupSizeZ = nullptr;
+
+  Value *GridSizeX = nullptr;
+  Value *GridSizeY = nullptr;
+  Value *GridSizeZ = nullptr;
+
+  const DataLayout &DL = Mod->getDataLayout();
+
+  // We expect to see several GEP users, casted to the appropriate type and
+  // loaded.
+  for (User *U : CI->users()) {
+    if (!U->hasOneUse())
+      continue;
+
+    int64_t Offset = 0;
+    if (GetPointerBaseWithConstantOffset(U, Offset, DL) != CI)
+      continue;
+
+    auto *BCI = dyn_cast<BitCastInst>(*U->user_begin());
+    if (!BCI || !BCI->hasOneUse())
+      continue;
+
+    auto *Load = dyn_cast<LoadInst>(*BCI->user_begin());
+    if (!Load || !Load->isSimple())
+      continue;
+
+    unsigned LoadSize = DL.getTypeStoreSize(Load->getType());
+
+    // TODO: Handle merged loads.
+    switch (Offset) {
+    case WORKGROUP_SIZE_X:
+      if (LoadSize == 2)
+        WorkGroupSizeX = Load;
+      break;
+    case WORKGROUP_SIZE_Y:
+      if (LoadSize == 2)
+        WorkGroupSizeY = Load;
+      break;
+    case WORKGROUP_SIZE_Z:
+      if (LoadSize == 2)
+        WorkGroupSizeZ = Load;
+      break;
+    case GRID_SIZE_X:
+      if (LoadSize == 4)
+        GridSizeX = Load;
+      break;
+    case GRID_SIZE_Y:
+      if (LoadSize == 4)
+        GridSizeY = Load;
+      break;
+    case GRID_SIZE_Z:
+      if (LoadSize == 4)
+        GridSizeZ = Load;
+      break;
+    default:
+      break;
+    }
+  }
+
+  // Pattern match the code used to handle partial workgroup dispatches in the
+  // library implementation of get_local_size, so the entire function can be
+  // constant folded with a known group size.
+  //
+  // uint r = grid_size - group_id * group_size;
+  // get_local_size = (r < group_size) ? r : group_size;
+  //
+  // If we have uniform-work-group-size (which is the default in OpenCL 1.2),
+  // the grid_size is required to be a multiple of group_size). In this case:
+  //
+  // grid_size - (group_id * group_size) < group_size
+  // ->
+  // grid_size < group_size + (group_id * group_size)
+  //
+  // (grid_size / group_size) < 1 + group_id
+  //
+  // grid_size / group_size is at least 1, so we can conclude the select
+  // condition is false (except for group_id == 0, where the select result is
+  // the same).
+
+  bool MadeChange = false;
+  Value *WorkGroupSizes[3] = { WorkGroupSizeX, WorkGroupSizeY, WorkGroupSizeZ };
+  Value *GridSizes[3] = { GridSizeX, GridSizeY, GridSizeZ };
+
+  for (int I = 0; HasUniformWorkGroupSize && I < 3; ++I) {
+    Value *GroupSize = WorkGroupSizes[I];
+    Value *GridSize = GridSizes[I];
+    if (!GroupSize || !GridSize)
+      continue;
+
+    for (User *U : GroupSize->users()) {
+      auto *ZextGroupSize = dyn_cast<ZExtInst>(U);
+      if (!ZextGroupSize)
+        continue;
+
+      for (User *ZextUser : ZextGroupSize->users()) {
+        auto *SI = dyn_cast<SelectInst>(ZextUser);
+        if (!SI)
+          continue;
+
+        using namespace llvm::PatternMatch;
+        auto GroupIDIntrin = I == 0 ?
+          m_Intrinsic<Intrinsic::amdgcn_workgroup_id_x>() :
+            (I == 1 ? m_Intrinsic<Intrinsic::amdgcn_workgroup_id_y>() :
+                      m_Intrinsic<Intrinsic::amdgcn_workgroup_id_z>());
+
+        auto SubExpr = m_Sub(m_Specific(GridSize),
+                             m_Mul(GroupIDIntrin, m_Specific(ZextGroupSize)));
+
+        ICmpInst::Predicate Pred;
+        if (match(SI,
+                  m_Select(m_ICmp(Pred, SubExpr, m_Specific(ZextGroupSize)),
+                           SubExpr,
+                           m_Specific(ZextGroupSize))) &&
+            Pred == ICmpInst::ICMP_ULT) {
+          if (HasReqdWorkGroupSize) {
+            ConstantInt *KnownSize
+              = mdconst::extract<ConstantInt>(MD->getOperand(I));
+            SI->replaceAllUsesWith(ConstantExpr::getIntegerCast(KnownSize,
+                                                                SI->getType(),
+                                                                false));
+          } else {
+            SI->replaceAllUsesWith(ZextGroupSize);
+          }
+
+          MadeChange = true;
+        }
+      }
+    }
+  }
+
+  if (!HasReqdWorkGroupSize)
+    return MadeChange;
+
+  // Eliminate any other loads we can from the dispatch packet.
+  for (int I = 0; I < 3; ++I) {
+    Value *GroupSize = WorkGroupSizes[I];
+    if (!GroupSize)
+      continue;
+
+    ConstantInt *KnownSize = mdconst::extract<ConstantInt>(MD->getOperand(I));
+    GroupSize->replaceAllUsesWith(
+      ConstantExpr::getIntegerCast(KnownSize,
+                                   GroupSize->getType(),
+                                   false));
+    MadeChange = true;
+  }
+
+  return MadeChange;
+}
+
+// TODO: Move makeLIDRangeMetadata usage into here. Seem to not get
+// TargetPassConfig for subtarget.
+bool AMDGPULowerKernelAttributes::runOnModule(Module &M) {
+  StringRef DispatchPtrName
+    = Intrinsic::getName(Intrinsic::amdgcn_dispatch_ptr);
+
+  Function *DispatchPtr = Mod->getFunction(DispatchPtrName);
+  if (!DispatchPtr) // Dispatch ptr not used.
+    return false;
+
+  bool MadeChange = false;
+
+  SmallPtrSet<Instruction *, 4> HandledUses;
+  for (auto *U : DispatchPtr->users()) {
+    CallInst *CI = cast<CallInst>(U);
+    if (HandledUses.insert(CI).second) {
+      if (processUse(CI))
+        MadeChange = true;
+    }
+  }
+
+  return MadeChange;
+}
+
+INITIALIZE_PASS_BEGIN(AMDGPULowerKernelAttributes, DEBUG_TYPE,
+                      "AMDGPU IR optimizations", false, false)
+INITIALIZE_PASS_END(AMDGPULowerKernelAttributes, DEBUG_TYPE, "AMDGPU IR optimizations",
+                    false, false)
+
+char AMDGPULowerKernelAttributes::ID = 0;
+
+ModulePass *llvm::createAMDGPULowerKernelAttributesPass() {
+  return new AMDGPULowerKernelAttributes();
+}

diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
index eee7578..76edea9 100644 (file)
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
@@ -161,6 +161,7 @@ extern "C" void LLVMInitializeAMDGPUTarget() {
   initializeAMDGPUAnnotateKernelFeaturesPass(*PR);
   initializeAMDGPUAnnotateUniformValuesPass(*PR);
   initializeAMDGPUArgumentUsageInfoPass(*PR);
+  initializeAMDGPULowerKernelAttributesPass(*PR);
   initializeAMDGPULowerIntrinsicsPass(*PR);
   initializeAMDGPUOpenCLEnqueuedBlockLoweringPass(*PR);
   initializeAMDGPUPromoteAllocaPass(*PR);
@@ -403,6 +404,10 @@ void AMDGPUTargetMachine::adjustPassManager(PassManagerBuilder &Builder) {
       // Add infer address spaces pass to the opt pipeline after inlining
       // but before SROA to increase SROA opportunities.
       PM.add(createInferAddressSpacesPass());
+
+      // This should run after inlining to have any chance of doing anything,
+      // and before other cleanup optimizations.
+      PM.add(createAMDGPULowerKernelAttributesPass());
   });
 }
 

diff --git a/llvm/lib/Target/AMDGPU/CMakeLists.txt b/llvm/lib/Target/AMDGPU/CMakeLists.txt
index dbcd280..26d3269 100644 (file)
--- a/llvm/lib/Target/AMDGPU/CMakeLists.txt
+++ b/llvm/lib/Target/AMDGPU/CMakeLists.txt
@@ -39,6 +39,7 @@ add_llvm_target(AMDGPUCodeGen
   AMDGPULibCalls.cpp
   AMDGPULibFunc.cpp
   AMDGPULowerIntrinsics.cpp
+  AMDGPULowerKernelAttributes.cpp
   AMDGPUMachineCFGStructurizer.cpp
   AMDGPUMachineFunction.cpp
   AMDGPUMachineModuleInfo.cpp

diff --git a/llvm/test/CodeGen/AMDGPU/reqd-work-group-size.ll b/llvm/test/CodeGen/AMDGPU/reqd-work-group-size.ll
new file mode 100644 (file)
index 0000000..51a0cf6
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/reqd-work-group-size.ll
@@ -0,0 +1,501 @@
+; RUN: opt -mtriple=amdgcn-amd-amdhsa -S -amdgpu-lower-kernel-attributes -instcombine %s | FileCheck -enable-var-scope %s
+
+; CHECK-LABEL: @invalid_reqd_work_group_size(
+; CHECK: load i16,
+define amdgpu_kernel void @invalid_reqd_work_group_size(i16 addrspace(1)* %out) #0 !reqd_work_group_size !1 {
+  %dispatch.ptr = tail call i8 addrspace(4)* @llvm.amdgcn.dispatch.ptr()
+  %gep.group.size.x = getelementptr inbounds i8, i8 addrspace(4)* %dispatch.ptr, i64 4
+  %gep.group.size.x.bc = bitcast i8 addrspace(4)* %gep.group.size.x to i16 addrspace(4)*
+  %group.size.x = load i16, i16 addrspace(4)* %gep.group.size.x.bc, align 4
+  store i16 %group.size.x, i16 addrspace(1)* %out
+  ret void
+}
+
+; CHECK-LABEL: @volatile_load_group_size_x(
+; CHECK: load volatile i16,
+define amdgpu_kernel void @volatile_load_group_size_x(i16 addrspace(1)* %out) #0 !reqd_work_group_size !0 {
+  %dispatch.ptr = tail call i8 addrspace(4)* @llvm.amdgcn.dispatch.ptr()
+  %gep.group.size.x = getelementptr inbounds i8, i8 addrspace(4)* %dispatch.ptr, i64 4
+  %gep.group.size.x.bc = bitcast i8 addrspace(4)* %gep.group.size.x to i16 addrspace(4)*
+  %group.size.x = load volatile i16, i16 addrspace(4)* %gep.group.size.x.bc, align 4
+  store i16 %group.size.x, i16 addrspace(1)* %out
+  ret void
+}
+
+; CHECK-LABEL: @load_group_size_x(
+; CHECK-NEXT: store i16 8,
+define amdgpu_kernel void @load_group_size_x(i16 addrspace(1)* %out) #0 !reqd_work_group_size !0 {
+  %dispatch.ptr = tail call i8 addrspace(4)* @llvm.amdgcn.dispatch.ptr()
+  %gep.group.size.x = getelementptr inbounds i8, i8 addrspace(4)* %dispatch.ptr, i64 4
+  %gep.group.size.x.bc = bitcast i8 addrspace(4)* %gep.group.size.x to i16 addrspace(4)*
+  %group.size.x = load i16, i16 addrspace(4)* %gep.group.size.x.bc, align 4
+  store i16 %group.size.x, i16 addrspace(1)* %out
+  ret void
+}
+
+; CHECK-LABEL: @load_group_size_y(
+; CHECK-NEXT: store i16 16,
+define amdgpu_kernel void @load_group_size_y(i16 addrspace(1)* %out) #0 !reqd_work_group_size !0 {
+  %dispatch.ptr = tail call i8 addrspace(4)* @llvm.amdgcn.dispatch.ptr()
+  %gep.group.size.y = getelementptr inbounds i8, i8 addrspace(4)* %dispatch.ptr, i64 6
+  %gep.group.size.y.bc = bitcast i8 addrspace(4)* %gep.group.size.y to i16 addrspace(4)*
+  %group.size.y = load i16, i16 addrspace(4)* %gep.group.size.y.bc, align 4
+  store i16 %group.size.y, i16 addrspace(1)* %out
+  ret void
+}
+
+; CHECK-LABEL: @load_group_size_z(
+; CHECK-NEXT: store i16 2,
+define amdgpu_kernel void @load_group_size_z(i16 addrspace(1)* %out) #0 !reqd_work_group_size !0 {
+  %dispatch.ptr = tail call i8 addrspace(4)* @llvm.amdgcn.dispatch.ptr()
+  %gep.group.size.z = getelementptr inbounds i8, i8 addrspace(4)* %dispatch.ptr, i64 8
+  %gep.group.size.z.bc = bitcast i8 addrspace(4)* %gep.group.size.z to i16 addrspace(4)*
+  %group.size.z = load i16, i16 addrspace(4)* %gep.group.size.z.bc, align 4
+  store i16 %group.size.z, i16 addrspace(1)* %out
+  ret void
+}
+
+; Metadata uses i64 instead of i32
+; CHECK-LABEL: @load_group_size_x_reqd_work_group_size_i64(
+; CHECK-NEXT: store i16 8,
+define amdgpu_kernel void @load_group_size_x_reqd_work_group_size_i64(i16 addrspace(1)* %out) #0 !reqd_work_group_size !2 {
+  %dispatch.ptr = tail call i8 addrspace(4)* @llvm.amdgcn.dispatch.ptr()
+  %gep.group.size.x = getelementptr inbounds i8, i8 addrspace(4)* %dispatch.ptr, i64 4
+  %gep.group.size.x.bc = bitcast i8 addrspace(4)* %gep.group.size.x to i16 addrspace(4)*
+  %group.size.x = load i16, i16 addrspace(4)* %gep.group.size.x.bc, align 4
+  store i16 %group.size.x, i16 addrspace(1)* %out
+  ret void
+}
+
+; Metadata uses i16 instead of i32
+; CHECK-LABEL: @load_group_size_x_reqd_work_group_size_i16(
+; CHECK-NEXT: store i16 8,
+define amdgpu_kernel void @load_group_size_x_reqd_work_group_size_i16(i16 addrspace(1)* %out) #0 !reqd_work_group_size !3 {
+  %dispatch.ptr = tail call i8 addrspace(4)* @llvm.amdgcn.dispatch.ptr()
+  %gep.group.size.x = getelementptr inbounds i8, i8 addrspace(4)* %dispatch.ptr, i64 4
+  %gep.group.size.x.bc = bitcast i8 addrspace(4)* %gep.group.size.x to i16 addrspace(4)*
+  %group.size.x = load i16, i16 addrspace(4)* %gep.group.size.x.bc, align 4
+  store i16 %group.size.x, i16 addrspace(1)* %out
+  ret void
+}
+
+; CHECK-LABEL: @use_local_size_x_8_16_2(
+; CHECK-NEXT: store i64 8,
+define amdgpu_kernel void @use_local_size_x_8_16_2(i64 addrspace(1)* %out) #0 !reqd_work_group_size !0 {
+  %dispatch.ptr = tail call i8 addrspace(4)* @llvm.amdgcn.dispatch.ptr()
+  %gep.group.size.x = getelementptr inbounds i8, i8 addrspace(4)* %dispatch.ptr, i64 4
+  %gep.group.size.x.bc = bitcast i8 addrspace(4)* %gep.group.size.x to i16 addrspace(4)*
+  %group.size.x = load i16, i16 addrspace(4)* %gep.group.size.x.bc, align 4
+  %gep.grid.size.x = getelementptr inbounds i8, i8 addrspace(4)* %dispatch.ptr, i64 12
+  %gep.grid.size.x.bc = bitcast i8 addrspace(4)* %gep.grid.size.x to i32 addrspace(4)*
+  %grid.size.x = load i32, i32 addrspace(4)* %gep.grid.size.x.bc, align 4
+  %group.id = tail call i32 @llvm.amdgcn.workgroup.id.x()
+  %group.size.x.zext = zext i16 %group.size.x to i32
+  %group.id_x_group.size.x = mul i32 %group.id, %group.size.x.zext
+  %sub = sub i32 %grid.size.x, %group.id_x_group.size.x
+  %cmp = icmp ult i32 %sub, %group.size.x.zext
+  %select = select i1 %cmp, i32 %sub, i32 %group.size.x.zext
+  %zext = zext i32 %select to i64
+  store i64 %zext, i64 addrspace(1)* %out
+  ret void
+}
+
+; CHECK-LABEL: @use_local_size_y_8_16_2(
+; CHECK-NEXT: store i64 16,
+define amdgpu_kernel void @use_local_size_y_8_16_2(i64 addrspace(1)* %out) #0 !reqd_work_group_size !0 {
+  %dispatch.ptr = tail call i8 addrspace(4)* @llvm.amdgcn.dispatch.ptr()
+  %gep.group.size.y = getelementptr inbounds i8, i8 addrspace(4)* %dispatch.ptr, i64 6
+  %gep.group.size.y.bc = bitcast i8 addrspace(4)* %gep.group.size.y to i16 addrspace(4)*
+  %group.size.y = load i16, i16 addrspace(4)* %gep.group.size.y.bc, align 4
+  %gep.grid.size.y = getelementptr inbounds i8, i8 addrspace(4)* %dispatch.ptr, i64 16
+  %gep.grid.size.y.bc = bitcast i8 addrspace(4)* %gep.grid.size.y to i32 addrspace(4)*
+  %grid.size.y = load i32, i32 addrspace(4)* %gep.grid.size.y.bc, align 4
+  %group.id = tail call i32 @llvm.amdgcn.workgroup.id.y()
+  %group.size.y.zext = zext i16 %group.size.y to i32
+  %group.id_x_group.size.y = mul i32 %group.id, %group.size.y.zext
+  %sub = sub i32 %grid.size.y, %group.id_x_group.size.y
+  %cmp = icmp ult i32 %sub, %group.size.y.zext
+  %select = select i1 %cmp, i32 %sub, i32 %group.size.y.zext
+  %zext = zext i32 %select to i64
+  store i64 %zext, i64 addrspace(1)* %out
+  ret void
+}
+
+; CHECK-LABEL: @use_local_size_z_8_16_2(
+; CHECK-NEXT: store i64 2,
+define amdgpu_kernel void @use_local_size_z_8_16_2(i64 addrspace(1)* %out) #0 !reqd_work_group_size !0 {
+  %dispatch.ptr = tail call i8 addrspace(4)* @llvm.amdgcn.dispatch.ptr()
+  %gep.group.size.z = getelementptr inbounds i8, i8 addrspace(4)* %dispatch.ptr, i64 8
+  %gep.group.size.z.bc = bitcast i8 addrspace(4)* %gep.group.size.z to i16 addrspace(4)*
+  %group.size.z = load i16, i16 addrspace(4)* %gep.group.size.z.bc, align 4
+  %gep.grid.size.z = getelementptr inbounds i8, i8 addrspace(4)* %dispatch.ptr, i64 20
+  %gep.grid.size.z.bc = bitcast i8 addrspace(4)* %gep.grid.size.z to i32 addrspace(4)*
+  %grid.size.z = load i32, i32 addrspace(4)* %gep.grid.size.z.bc, align 4
+  %group.id = tail call i32 @llvm.amdgcn.workgroup.id.z()
+  %group.size.z.zext = zext i16 %group.size.z to i32
+  %group.id_x_group.size.z = mul i32 %group.id, %group.size.z.zext
+  %sub = sub i32 %grid.size.z, %group.id_x_group.size.z
+  %cmp = icmp ult i32 %sub, %group.size.z.zext
+  %select = select i1 %cmp, i32 %sub, i32 %group.size.z.zext
+  %zext = zext i32 %select to i64
+  store i64 %zext, i64 addrspace(1)* %out
+  ret void
+}
+
+; Simplification on select is invalid, but we can still eliminate the
+; load of the group size.
+
+; CHECK-LABEL: @local_size_x_8_16_2_wrong_group_id(
+; CHECK: %group.id = tail call i32 @llvm.amdgcn.workgroup.id.y()
+; CHECK: %group.id_x_group.size.x = shl i32 %group.id, 3
+define amdgpu_kernel void @local_size_x_8_16_2_wrong_group_id(i64 addrspace(1)* %out) #0 !reqd_work_group_size !0 {
+  %dispatch.ptr = tail call i8 addrspace(4)* @llvm.amdgcn.dispatch.ptr()
+  %gep.group.size.x = getelementptr inbounds i8, i8 addrspace(4)* %dispatch.ptr, i64 4
+  %gep.group.size.x.bc = bitcast i8 addrspace(4)* %gep.group.size.x to i16 addrspace(4)*
+  %group.size.x = load i16, i16 addrspace(4)* %gep.group.size.x.bc, align 4
+  %gep.grid.size.x = getelementptr inbounds i8, i8 addrspace(4)* %dispatch.ptr, i64 12
+  %gep.grid.size.x.bc = bitcast i8 addrspace(4)* %gep.grid.size.x to i32 addrspace(4)*
+  %grid.size.x = load i32, i32 addrspace(4)* %gep.grid.size.x.bc, align 4
+  %group.id = tail call i32 @llvm.amdgcn.workgroup.id.y()
+  %group.size.x.zext = zext i16 %group.size.x to i32
+  %group.id_x_group.size.x = mul i32 %group.id, %group.size.x.zext
+  %sub = sub i32 %grid.size.x, %group.id_x_group.size.x
+  %cmp = icmp ult i32 %sub, %group.size.x.zext
+  %select = select i1 %cmp, i32 %sub, i32 %group.size.x.zext
+  %zext = zext i32 %select to i64
+  store i64 %zext, i64 addrspace(1)* %out
+  ret void
+}
+
+; CHECK-LABEL: @local_size_x_8_16_2_wrong_grid_size(
+; CHECK: %grid.size.x = load i32, i32 addrspace(4)* %gep.grid.size.x.bc, align 4
+; CHECK: %group.id = tail call i32 @llvm.amdgcn.workgroup.id.x()
+; CHECK: %group.id_x_group.size.x = shl i32 %group.id, 3
+; CHECK: %sub = sub i32 %grid.size.x, %group.id_x_group.size.x
+  define amdgpu_kernel void @local_size_x_8_16_2_wrong_grid_size(i64 addrspace(1)* %out) #0 !reqd_work_group_size !0 {
+  %dispatch.ptr = tail call i8 addrspace(4)* @llvm.amdgcn.dispatch.ptr()
+  %gep.group.size.x = getelementptr inbounds i8, i8 addrspace(4)* %dispatch.ptr, i64 4
+  %gep.group.size.x.bc = bitcast i8 addrspace(4)* %gep.group.size.x to i16 addrspace(4)*
+  %group.size.x = load i16, i16 addrspace(4)* %gep.group.size.x.bc, align 4
+  %gep.grid.size.x = getelementptr inbounds i8, i8 addrspace(4)* %dispatch.ptr, i64 16
+  %gep.grid.size.x.bc = bitcast i8 addrspace(4)* %gep.grid.size.x to i32 addrspace(4)*
+  %grid.size.x = load i32, i32 addrspace(4)* %gep.grid.size.x.bc, align 4
+  %group.id = tail call i32 @llvm.amdgcn.workgroup.id.x()
+  %group.size.x.zext = zext i16 %group.size.x to i32
+  %group.id_x_group.size.x = mul i32 %group.id, %group.size.x.zext
+  %sub = sub i32 %grid.size.x, %group.id_x_group.size.x
+  %cmp = icmp ult i32 %sub, %group.size.x.zext
+  %select = select i1 %cmp, i32 %sub, i32 %group.size.x.zext
+  %zext = zext i32 %select to i64
+  store i64 %zext, i64 addrspace(1)* %out
+  ret void
+}
+
+; CHECK-LABEL: @local_size_x_8_16_2_wrong_cmp_type(
+; CHECK: %grid.size.x = load i32, i32 addrspace(4)* %gep.grid.size.x.bc, align 4
+; CHECK: %group.id = tail call i32 @llvm.amdgcn.workgroup.id.x()
+; CHECK: %group.id_x_group.size.x = shl i32 %group.id, 3
+; CHECK: %sub = sub i32 %grid.size.x, %group.id_x_group.size.x
+; CHECK: %cmp = icmp slt i32 %sub, 8
+; CHECK: %select = select i1 %cmp, i32 %sub, i32 8
+define amdgpu_kernel void @local_size_x_8_16_2_wrong_cmp_type(i64 addrspace(1)* %out) #0 !reqd_work_group_size !0 {
+  %dispatch.ptr = tail call i8 addrspace(4)* @llvm.amdgcn.dispatch.ptr()
+  %gep.group.size.x = getelementptr inbounds i8, i8 addrspace(4)* %dispatch.ptr, i64 4
+  %gep.group.size.x.bc = bitcast i8 addrspace(4)* %gep.group.size.x to i16 addrspace(4)*
+  %group.size.x = load i16, i16 addrspace(4)* %gep.group.size.x.bc, align 4
+  %gep.grid.size.x = getelementptr inbounds i8, i8 addrspace(4)* %dispatch.ptr, i64 12
+  %gep.grid.size.x.bc = bitcast i8 addrspace(4)* %gep.grid.size.x to i32 addrspace(4)*
+  %grid.size.x = load i32, i32 addrspace(4)* %gep.grid.size.x.bc, align 4
+  %group.id = tail call i32 @llvm.amdgcn.workgroup.id.x()
+  %group.size.x.zext = zext i16 %group.size.x to i32
+  %group.id_x_group.size.x = mul i32 %group.id, %group.size.x.zext
+  %sub = sub i32 %grid.size.x, %group.id_x_group.size.x
+  %cmp = icmp slt i32 %sub, %group.size.x.zext
+  %select = select i1 %cmp, i32 %sub, i32 %group.size.x.zext
+  %zext = zext i32 %select to i64
+  store i64 %zext, i64 addrspace(1)* %out
+  ret void
+}
+
+; CHECK-LABEL: @local_size_x_8_16_2_wrong_select(
+; CHECK: %group.id_x_group.size.x = shl i32 %group.id, 3
+; CHECK: %sub = sub i32 %grid.size.x, %group.id_x_group.size.x
+; CHECK: %1 = icmp ugt i32 %sub, 8
+; CHECK: %select = select i1 %1, i32 %sub, i32 8
+; CHECK: %zext = zext i32 %select to i64
+define amdgpu_kernel void @local_size_x_8_16_2_wrong_select(i64 addrspace(1)* %out) #0 !reqd_work_group_size !0 {
+  %dispatch.ptr = tail call i8 addrspace(4)* @llvm.amdgcn.dispatch.ptr()
+  %gep.group.size.x = getelementptr inbounds i8, i8 addrspace(4)* %dispatch.ptr, i64 4
+  %gep.group.size.x.bc = bitcast i8 addrspace(4)* %gep.group.size.x to i16 addrspace(4)*
+  %group.size.x = load i16, i16 addrspace(4)* %gep.group.size.x.bc, align 4
+  %gep.grid.size.x = getelementptr inbounds i8, i8 addrspace(4)* %dispatch.ptr, i64 12
+  %gep.grid.size.x.bc = bitcast i8 addrspace(4)* %gep.grid.size.x to i32 addrspace(4)*
+  %grid.size.x = load i32, i32 addrspace(4)* %gep.grid.size.x.bc, align 4
+  %group.id = tail call i32 @llvm.amdgcn.workgroup.id.x()
+  %group.size.x.zext = zext i16 %group.size.x to i32
+  %group.id_x_group.size.x = mul i32 %group.id, %group.size.x.zext
+  %sub = sub i32 %grid.size.x, %group.id_x_group.size.x
+  %cmp = icmp ult i32 %sub, %group.size.x.zext
+  %select = select i1 %cmp, i32 %group.size.x.zext, i32 %sub
+  %zext = zext i32 %select to i64
+  store i64 %zext, i64 addrspace(1)* %out
+  ret void
+}
+
+; CHECK-LABEL: @use_local_size_x_8_16_2_wrong_grid_load_size(
+; CHECK: %grid.size.x = load i16, i16 addrspace(4)* %gep.grid.size.x.bc, align 4
+; CHECK: %grid.size.x.zext = zext i16 %grid.size.x to i32
+; CHECK: %group.id = tail call i32 @llvm.amdgcn.workgroup.id.x()
+; CHECK: %group.id_x_group.size.x = shl i32 %group.id, 3
+; CHECK: %sub = sub i32 %grid.size.x.zext, %group.id_x_group.size.x
+define amdgpu_kernel void @use_local_size_x_8_16_2_wrong_grid_load_size(i64 addrspace(1)* %out) #0 !reqd_work_group_size !0 {
+  %dispatch.ptr = tail call i8 addrspace(4)* @llvm.amdgcn.dispatch.ptr()
+  %gep.group.size.x = getelementptr inbounds i8, i8 addrspace(4)* %dispatch.ptr, i64 4
+  %gep.group.size.x.bc = bitcast i8 addrspace(4)* %gep.group.size.x to i16 addrspace(4)*
+  %group.size.x = load i16, i16 addrspace(4)* %gep.group.size.x.bc, align 4
+  %gep.grid.size.x = getelementptr inbounds i8, i8 addrspace(4)* %dispatch.ptr, i64 12
+  %gep.grid.size.x.bc = bitcast i8 addrspace(4)* %gep.grid.size.x to i16 addrspace(4)*
+  %grid.size.x = load i16, i16 addrspace(4)* %gep.grid.size.x.bc, align 4
+  %grid.size.x.zext = zext i16 %grid.size.x to i32
+  %group.id = tail call i32 @llvm.amdgcn.workgroup.id.x()
+  %group.size.x.zext = zext i16 %group.size.x to i32
+  %group.id_x_group.size.x = mul i32 %group.id, %group.size.x.zext
+  %sub = sub i32 %grid.size.x.zext, %group.id_x_group.size.x
+  %cmp = icmp ult i32 %sub, %group.size.x.zext
+  %select = select i1 %cmp, i32 %sub, i32 %group.size.x.zext
+  %zext = zext i32 %select to i64
+  store i64 %zext, i64 addrspace(1)* %out
+  ret void
+}
+
+; CHECK-LABEL: @func_group_size_x(
+; CHECK-NEXT: ret i32 8
+define i32 @func_group_size_x(i16 addrspace(1)* %out) #0 !reqd_work_group_size !0 {
+  %dispatch.ptr = tail call i8 addrspace(4)* @llvm.amdgcn.dispatch.ptr()
+  %gep.group.size.x = getelementptr inbounds i8, i8 addrspace(4)* %dispatch.ptr, i64 4
+  %gep.group.size.x.bc = bitcast i8 addrspace(4)* %gep.group.size.x to i16 addrspace(4)*
+  %group.size.x = load i16, i16 addrspace(4)* %gep.group.size.x.bc, align 4
+  %zext = zext i16 %group.size.x to i32
+  ret i32 %zext
+}
+
+; CHECK-LABEL: @__ockl_get_local_size_reqd_size(
+; CHECK: %group.size = phi i32 [ 2, %bb17 ], [ 16, %bb9 ], [ 8, %bb1 ], [ 1, %bb ]
+define i64 @__ockl_get_local_size_reqd_size(i32 %arg) #1 !reqd_work_group_size !0 {
+bb:
+  %tmp = tail call i8 addrspace(4)* @llvm.amdgcn.dispatch.ptr() #2
+  switch i32 %arg, label %bb25 [
+    i32 0, label %bb1
+    i32 1, label %bb9
+    i32 2, label %bb17
+  ]
+
+bb1:                                              ; preds = %bb
+  %tmp2 = tail call i32 @llvm.amdgcn.workgroup.id.x()
+  %tmp3 = getelementptr inbounds i8, i8 addrspace(4)* %tmp, i64 12
+  %tmp4 = bitcast i8 addrspace(4)* %tmp3 to i32 addrspace(4)*
+  %tmp5 = load i32, i32 addrspace(4)* %tmp4, align 4
+  %tmp6 = getelementptr inbounds i8, i8 addrspace(4)* %tmp, i64 4
+  %tmp7 = bitcast i8 addrspace(4)* %tmp6 to i16 addrspace(4)*
+  %tmp8 = load i16, i16 addrspace(4)* %tmp7, align 4
+  br label %bb25
+
+bb9:                                              ; preds = %bb
+  %tmp10 = tail call i32 @llvm.amdgcn.workgroup.id.y()
+  %tmp11 = getelementptr inbounds i8, i8 addrspace(4)* %tmp, i64 16
+  %tmp12 = bitcast i8 addrspace(4)* %tmp11 to i32 addrspace(4)*
+  %tmp13 = load i32, i32 addrspace(4)* %tmp12, align 8
+  %tmp14 = getelementptr inbounds i8, i8 addrspace(4)* %tmp, i64 6
+  %tmp15 = bitcast i8 addrspace(4)* %tmp14 to i16 addrspace(4)*
+  %tmp16 = load i16, i16 addrspace(4)* %tmp15, align 2
+  br label %bb25
+
+bb17:                                             ; preds = %bb
+  %tmp18 = tail call i32 @llvm.amdgcn.workgroup.id.z()
+  %tmp19 = getelementptr inbounds i8, i8 addrspace(4)* %tmp, i64 20
+  %tmp20 = bitcast i8 addrspace(4)* %tmp19 to i32 addrspace(4)*
+  %tmp21 = load i32, i32 addrspace(4)* %tmp20, align 4
+  %tmp22 = getelementptr inbounds i8, i8 addrspace(4)* %tmp, i64 8
+  %tmp23 = bitcast i8 addrspace(4)* %tmp22 to i16 addrspace(4)*
+  %tmp24 = load i16, i16 addrspace(4)* %tmp23, align 8
+  br label %bb25
+
+bb25:                                             ; preds = %bb17, %bb9, %bb1, %bb
+  %tmp26 = phi i32 [ %tmp21, %bb17 ], [ %tmp13, %bb9 ], [ %tmp5, %bb1 ], [ 0, %bb ]
+  %group.size = phi i16 [ %tmp24, %bb17 ], [ %tmp16, %bb9 ], [ %tmp8, %bb1 ], [ 1, %bb ]
+  %tmp28 = phi i32 [ %tmp18, %bb17 ], [ %tmp10, %bb9 ], [ %tmp2, %bb1 ], [ 0, %bb ]
+  %tmp29 = zext i16 %group.size to i32
+  %tmp30 = mul i32 %tmp28, %tmp29
+  %tmp31 = sub i32 %tmp26, %tmp30
+  %tmp32 = icmp ult i32 %tmp31, %tmp29
+  %tmp33 = select i1 %tmp32, i32 %tmp31, i32 %tmp29
+  %tmp34 = zext i32 %tmp33 to i64
+  ret i64 %tmp34
+}
+
+; CHECK-LABEL: @all_local_size(
+; CHECK-NEXT: store volatile i64 8, i64 addrspace(1)* %out, align 4
+; CHECK-NEXT: store volatile i64 16, i64 addrspace(1)* %out, align 4
+; CHECK-NEXT: store volatile i64 2, i64 addrspace(1)* %out, align 4
+define amdgpu_kernel void @all_local_size(i64 addrspace(1)* nocapture readnone %out) #0 !reqd_work_group_size !0 {
+  %tmp.i = tail call i8 addrspace(4)* @llvm.amdgcn.dispatch.ptr() #0
+  %tmp2.i = tail call i32 @llvm.amdgcn.workgroup.id.x() #0
+  %tmp3.i = getelementptr inbounds i8, i8 addrspace(4)* %tmp.i, i64 12
+  %tmp4.i = bitcast i8 addrspace(4)* %tmp3.i to i32 addrspace(4)*
+  %tmp5.i = load i32, i32 addrspace(4)* %tmp4.i, align 4
+  %tmp6.i = getelementptr inbounds i8, i8 addrspace(4)* %tmp.i, i64 4
+  %tmp7.i = bitcast i8 addrspace(4)* %tmp6.i to i16 addrspace(4)*
+  %tmp8.i = load i16, i16 addrspace(4)* %tmp7.i, align 4
+  %tmp29.i = zext i16 %tmp8.i to i32
+  %tmp30.i = mul i32 %tmp2.i, %tmp29.i
+  %tmp31.i = sub i32 %tmp5.i, %tmp30.i
+  %tmp32.i = icmp ult i32 %tmp31.i, %tmp29.i
+  %tmp33.i = select i1 %tmp32.i, i32 %tmp31.i, i32 %tmp29.i
+  %tmp34.i = zext i32 %tmp33.i to i64
+  %tmp10.i = tail call i32 @llvm.amdgcn.workgroup.id.y() #0
+  %tmp11.i = getelementptr inbounds i8, i8 addrspace(4)* %tmp.i, i64 16
+  %tmp12.i = bitcast i8 addrspace(4)* %tmp11.i to i32 addrspace(4)*
+  %tmp13.i = load i32, i32 addrspace(4)* %tmp12.i, align 8
+  %tmp14.i = getelementptr inbounds i8, i8 addrspace(4)* %tmp.i, i64 6
+  %tmp15.i = bitcast i8 addrspace(4)* %tmp14.i to i16 addrspace(4)*
+  %tmp16.i = load i16, i16 addrspace(4)* %tmp15.i, align 2
+  %tmp29.i9 = zext i16 %tmp16.i to i32
+  %tmp30.i10 = mul i32 %tmp10.i, %tmp29.i9
+  %tmp31.i11 = sub i32 %tmp13.i, %tmp30.i10
+  %tmp32.i12 = icmp ult i32 %tmp31.i11, %tmp29.i9
+  %tmp33.i13 = select i1 %tmp32.i12, i32 %tmp31.i11, i32 %tmp29.i9
+  %tmp34.i14 = zext i32 %tmp33.i13 to i64
+  %tmp18.i = tail call i32 @llvm.amdgcn.workgroup.id.z() #0
+  %tmp19.i = getelementptr inbounds i8, i8 addrspace(4)* %tmp.i, i64 20
+  %tmp20.i = bitcast i8 addrspace(4)* %tmp19.i to i32 addrspace(4)*
+  %tmp21.i = load i32, i32 addrspace(4)* %tmp20.i, align 4
+  %tmp22.i = getelementptr inbounds i8, i8 addrspace(4)* %tmp.i, i64 8
+  %tmp23.i = bitcast i8 addrspace(4)* %tmp22.i to i16 addrspace(4)*
+  %tmp24.i = load i16, i16 addrspace(4)* %tmp23.i, align 8
+  %tmp29.i2 = zext i16 %tmp24.i to i32
+  %tmp30.i3 = mul i32 %tmp18.i, %tmp29.i2
+  %tmp31.i4 = sub i32 %tmp21.i, %tmp30.i3
+  %tmp32.i5 = icmp ult i32 %tmp31.i4, %tmp29.i2
+  %tmp33.i6 = select i1 %tmp32.i5, i32 %tmp31.i4, i32 %tmp29.i2
+  %tmp34.i7 = zext i32 %tmp33.i6 to i64
+  store volatile i64 %tmp34.i, i64 addrspace(1)* %out, align 4
+  store volatile i64 %tmp34.i14, i64 addrspace(1)* %out, align 4
+  store volatile i64 %tmp34.i7, i64 addrspace(1)* %out, align 4
+  ret void
+}
+
+; TODO: Should be able to handle this, but not much reason to.
+; CHECK-LABEL: @partial_load_group_size_x(
+; CHECK-NEXT: %dispatch.ptr = tail call i8 addrspace(4)* @llvm.amdgcn.dispatch.ptr()
+; CHECK-NEXT: %gep.group.size.x = getelementptr inbounds i8, i8 addrspace(4)* %dispatch.ptr, i64 4
+; CHECK-NEXT: %group.size.x.lo = load i8, i8 addrspace(4)* %gep.group.size.x, align 1
+; CHECK-NEXT: store i8 %group.size.x.lo, i8 addrspace(1)* %out, align 1
+define amdgpu_kernel void @partial_load_group_size_x(i8 addrspace(1)* %out) #0 !reqd_work_group_size !0 {
+  %dispatch.ptr = tail call i8 addrspace(4)* @llvm.amdgcn.dispatch.ptr()
+  %gep.group.size.x = getelementptr inbounds i8, i8 addrspace(4)* %dispatch.ptr, i64 4
+  %group.size.x.lo = load i8, i8 addrspace(4)* %gep.group.size.x, align 1
+  store i8 %group.size.x.lo, i8 addrspace(1)* %out
+  ret void
+}
+
+; TODO: Should be able to handle this
+; CHECK-LABEL: @load_group_size_xy_i32(
+; CHECK: %group.size.xy = load i32,
+; CHECK: store i32 %group.size.xy
+define amdgpu_kernel void @load_group_size_xy_i32(i32 addrspace(1)* %out) #0 !reqd_work_group_size !0 {
+  %dispatch.ptr = tail call i8 addrspace(4)* @llvm.amdgcn.dispatch.ptr()
+  %gep.group.size.x = getelementptr inbounds i8, i8 addrspace(4)* %dispatch.ptr, i64 4
+  %gep.group.size.x.bc = bitcast i8 addrspace(4)* %gep.group.size.x to i32 addrspace(4)*
+  %group.size.xy = load i32, i32 addrspace(4)* %gep.group.size.x.bc, align 4
+  store i32 %group.size.xy, i32 addrspace(1)* %out
+  ret void
+}
+
+; CHECK-LABEL: @load_group_size_x_y_multiple_dispatch_ptr(
+; CHECK-NEXT: store volatile i16 8, i16 addrspace(1)* %out, align 2
+; CHECK-NEXT: store volatile i16 16, i16 addrspace(1)* %out, align 2
+define amdgpu_kernel void @load_group_size_x_y_multiple_dispatch_ptr(i16 addrspace(1)* %out) #0 !reqd_work_group_size !0 {
+  %dispatch.ptr0 = tail call i8 addrspace(4)* @llvm.amdgcn.dispatch.ptr()
+  %gep.group.size.x = getelementptr inbounds i8, i8 addrspace(4)* %dispatch.ptr0, i64 4
+  %gep.group.size.x.bc = bitcast i8 addrspace(4)* %gep.group.size.x to i16 addrspace(4)*
+  %group.size.x = load i16, i16 addrspace(4)* %gep.group.size.x.bc, align 4
+  store volatile i16 %group.size.x, i16 addrspace(1)* %out
+
+  %dispatch.ptr1 = tail call i8 addrspace(4)* @llvm.amdgcn.dispatch.ptr()
+  %gep.group.size.y = getelementptr inbounds i8, i8 addrspace(4)* %dispatch.ptr1, i64 6
+  %gep.group.size.y.bc = bitcast i8 addrspace(4)* %gep.group.size.y to i16 addrspace(4)*
+  %group.size.y = load i16, i16 addrspace(4)* %gep.group.size.y.bc, align 4
+  store volatile i16 %group.size.y, i16 addrspace(1)* %out
+
+  ret void
+}
+
+; CHECK-LABEL: @use_local_size_x_uniform_work_group_size(
+; CHECK-NEXT: %dispatch.ptr = tail call i8 addrspace(4)* @llvm.amdgcn.dispatch.ptr()
+; CHECK-NEXT: %gep.group.size.x = getelementptr inbounds i8, i8 addrspace(4)* %dispatch.ptr, i64 4
+; CHECK-NEXT: %gep.group.size.x.bc = bitcast i8 addrspace(4)* %gep.group.size.x to i16 addrspace(4)*
+; CHECK-NEXT: %group.size.x = load i16, i16 addrspace(4)* %gep.group.size.x.bc, align 4
+; CHECK-NEXT: %zext = zext i16 %group.size.x to i64
+; CHECK-NEXT: store i64 %zext, i64 addrspace(1)* %out, align 4
+define amdgpu_kernel void @use_local_size_x_uniform_work_group_size(i64 addrspace(1)* %out) #2 {
+  %dispatch.ptr = tail call i8 addrspace(4)* @llvm.amdgcn.dispatch.ptr()
+  %gep.group.size.x = getelementptr inbounds i8, i8 addrspace(4)* %dispatch.ptr, i64 4
+  %gep.group.size.x.bc = bitcast i8 addrspace(4)* %gep.group.size.x to i16 addrspace(4)*
+  %group.size.x = load i16, i16 addrspace(4)* %gep.group.size.x.bc, align 4
+  %gep.grid.size.x = getelementptr inbounds i8, i8 addrspace(4)* %dispatch.ptr, i64 12
+  %gep.grid.size.x.bc = bitcast i8 addrspace(4)* %gep.grid.size.x to i32 addrspace(4)*
+  %grid.size.x = load i32, i32 addrspace(4)* %gep.grid.size.x.bc, align 4
+  %group.id = tail call i32 @llvm.amdgcn.workgroup.id.x()
+  %group.size.x.zext = zext i16 %group.size.x to i32
+  %group.id_x_group.size.x = mul i32 %group.id, %group.size.x.zext
+  %sub = sub i32 %grid.size.x, %group.id_x_group.size.x
+  %cmp = icmp ult i32 %sub, %group.size.x.zext
+  %select = select i1 %cmp, i32 %sub, i32 %group.size.x.zext
+  %zext = zext i32 %select to i64
+  store i64 %zext, i64 addrspace(1)* %out
+  ret void
+}
+
+; CHECK-LABEL: @use_local_size_x_uniform_work_group_size_false(
+; CHECK: icmp ult
+; CHECK: select
+define amdgpu_kernel void @use_local_size_x_uniform_work_group_size_false(i64 addrspace(1)* %out) #3 {
+  %dispatch.ptr = tail call i8 addrspace(4)* @llvm.amdgcn.dispatch.ptr()
+  %gep.group.size.x = getelementptr inbounds i8, i8 addrspace(4)* %dispatch.ptr, i64 4
+  %gep.group.size.x.bc = bitcast i8 addrspace(4)* %gep.group.size.x to i16 addrspace(4)*
+  %group.size.x = load i16, i16 addrspace(4)* %gep.group.size.x.bc, align 4
+  %gep.grid.size.x = getelementptr inbounds i8, i8 addrspace(4)* %dispatch.ptr, i64 12
+  %gep.grid.size.x.bc = bitcast i8 addrspace(4)* %gep.grid.size.x to i32 addrspace(4)*
+  %grid.size.x = load i32, i32 addrspace(4)* %gep.grid.size.x.bc, align 4
+  %group.id = tail call i32 @llvm.amdgcn.workgroup.id.x()
+  %group.size.x.zext = zext i16 %group.size.x to i32
+  %group.id_x_group.size.x = mul i32 %group.id, %group.size.x.zext
+  %sub = sub i32 %grid.size.x, %group.id_x_group.size.x
+  %cmp = icmp ult i32 %sub, %group.size.x.zext
+  %select = select i1 %cmp, i32 %sub, i32 %group.size.x.zext
+  %zext = zext i32 %select to i64
+  store i64 %zext, i64 addrspace(1)* %out
+  ret void
+}
+
+; CHECK-LABEL: @no_use_dispatch_ptr(
+; CHECK-NEXT: ret void
+define amdgpu_kernel void @no_use_dispatch_ptr() {
+  %dispatch.ptr = tail call i8 addrspace(4)* @llvm.amdgcn.dispatch.ptr()
+  ret void
+}
+
+declare i8 addrspace(4)* @llvm.amdgcn.dispatch.ptr() #1
+declare i32 @llvm.amdgcn.workgroup.id.x() #1
+declare i32 @llvm.amdgcn.workgroup.id.y() #1
+declare i32 @llvm.amdgcn.workgroup.id.z() #1
+
+attributes #0 = { nounwind "uniform-work-group-size"="true" }
+attributes #1 = { nounwind readnone speculatable }
+attributes #2 = { nounwind "uniform-work-group-size"="true" }
+attributes #3 = { nounwind "uniform-work-group-size"="false" }
+
+!0 = !{i32 8, i32 16, i32 2}
+!1 = !{i32 8, i32 16}
+!2 = !{i64 8, i64 16, i64 2}
+!3 = !{i16 8, i16 16, i16 2}