dali-physics/third-party/bullet3/src/Bullet3OpenCL/BroadphaseCollision/kernels/parallelLinearBvhKernels.h

   1 //this file is autogenerated using stringify.bat (premake --stringify) in the build folder of this project
   2 static const char* parallelLinearBvhCL =
   3         "/*\n"
   4         "This software is provided 'as-is', without any express or implied warranty.\n"
   5         "In no event will the authors be held liable for any damages arising from the use of this software.\n"
   6         "Permission is granted to anyone to use this software for any purpose,\n"
   7         "including commercial applications, and to alter it and redistribute it freely,\n"
   8         "subject to the following restrictions:\n"
   9         "1. The origin of this software must not be misrepresented; you must not claim that you wrote the original software. If you use this software in a product, an acknowledgment in the product documentation would be appreciated but is not required.\n"
  10         "2. Altered source versions must be plainly marked as such, and must not be misrepresented as being the original software.\n"
  11         "3. This notice may not be removed or altered from any source distribution.\n"
  12         "*/\n"
  13         "//Initial Author Jackson Lee, 2014\n"
  14         "typedef float b3Scalar;\n"
  15         "typedef float4 b3Vector3;\n"
  16         "#define b3Max max\n"
  17         "#define b3Min min\n"
  18         "#define b3Sqrt sqrt\n"
  19         "typedef struct\n"
  20         "{\n"
  21         "       unsigned int m_key;\n"
  22         "       unsigned int m_value;\n"
  23         "} SortDataCL;\n"
  24         "typedef struct \n"
  25         "{\n"
  26         "       union\n"
  27         "       {\n"
  28         "               float4  m_min;\n"
  29         "               float   m_minElems[4];\n"
  30         "               int                     m_minIndices[4];\n"
  31         "       };\n"
  32         "       union\n"
  33         "       {\n"
  34         "               float4  m_max;\n"
  35         "               float   m_maxElems[4];\n"
  36         "               int                     m_maxIndices[4];\n"
  37         "       };\n"
  38         "} b3AabbCL;\n"
  39         "unsigned int interleaveBits(unsigned int x)\n"
  40         "{\n"
  41         "       //........ ........ ......12 3456789A   //x\n"
  42         "       //....1..2 ..3..4.. 5..6..7. .8..9..A   //x after interleaving bits\n"
  43         "       \n"
  44         "       //......12 3456789A ......12 3456789A   //x ^ (x << 16)\n"
  45         "       //11111111 ........ ........ 11111111   //0x FF 00 00 FF\n"
  46         "       //......12 ........ ........ 3456789A   //x = (x ^ (x << 16)) & 0xFF0000FF;\n"
  47         "       \n"
  48         "       //......12 ........ 3456789A 3456789A   //x ^ (x <<  8)\n"
  49         "       //......11 ........ 1111.... ....1111   //0x 03 00 F0 0F\n"
  50         "       //......12 ........ 3456.... ....789A   //x = (x ^ (x <<  8)) & 0x0300F00F;\n"
  51         "       \n"
  52         "       //..12..12 ....3456 3456.... 789A789A   //x ^ (x <<  4)\n"
  53         "       //......11 ....11.. ..11.... 11....11   //0x 03 0C 30 C3\n"
  54         "       //......12 ....34.. ..56.... 78....9A   //x = (x ^ (x <<  4)) & 0x030C30C3;\n"
  55         "       \n"
  56         "       //....1212 ..3434.. 5656..78 78..9A9A   //x ^ (x <<  2)\n"
  57         "       //....1..1 ..1..1.. 1..1..1. .1..1..1   //0x 09 24 92 49\n"
  58         "       //....1..2 ..3..4.. 5..6..7. .8..9..A   //x = (x ^ (x <<  2)) & 0x09249249;\n"
  59         "       \n"
  60         "       //........ ........ ......11 11111111   //0x000003FF\n"
  61         "       x &= 0x000003FF;                //Clear all bits above bit 10\n"
  62         "       \n"
  63         "       x = (x ^ (x << 16)) & 0xFF0000FF;\n"
  64         "       x = (x ^ (x <<  8)) & 0x0300F00F;\n"
  65         "       x = (x ^ (x <<  4)) & 0x030C30C3;\n"
  66         "       x = (x ^ (x <<  2)) & 0x09249249;\n"
  67         "       \n"
  68         "       return x;\n"
  69         "}\n"
  70         "unsigned int getMortonCode(unsigned int x, unsigned int y, unsigned int z)\n"
  71         "{\n"
  72         "       return interleaveBits(x) << 0 | interleaveBits(y) << 1 | interleaveBits(z) << 2;\n"
  73         "}\n"
  74         "__kernel void separateAabbs(__global b3AabbCL* unseparatedAabbs, __global int* aabbIndices, __global b3AabbCL* out_aabbs, int numAabbsToSeparate)\n"
  75         "{\n"
  76         "       int separatedAabbIndex = get_global_id(0);\n"
  77         "       if(separatedAabbIndex >= numAabbsToSeparate) return;\n"
  78         "       int unseparatedAabbIndex = aabbIndices[separatedAabbIndex];\n"
  79         "       out_aabbs[separatedAabbIndex] = unseparatedAabbs[unseparatedAabbIndex];\n"
  80         "}\n"
  81         "//Should replace with an optimized parallel reduction\n"
  82         "__kernel void findAllNodesMergedAabb(__global b3AabbCL* out_mergedAabb, int numAabbsNeedingMerge)\n"
  83         "{\n"
  84         "       //Each time this kernel is added to the command queue, \n"
  85         "       //the number of AABBs needing to be merged is halved\n"
  86         "       //\n"
  87         "       //Example with 159 AABBs:\n"
  88         "       //      numRemainingAabbs == 159 / 2 + 159 % 2 == 80\n"
  89         "       //      numMergedAabbs == 159 - 80 == 79\n"
  90         "       //So, indices [0, 78] are merged with [0 + 80, 78 + 80]\n"
  91         "       \n"
  92         "       int numRemainingAabbs = numAabbsNeedingMerge / 2 + numAabbsNeedingMerge % 2;\n"
  93         "       int numMergedAabbs = numAabbsNeedingMerge - numRemainingAabbs;\n"
  94         "       \n"
  95         "       int aabbIndex = get_global_id(0);\n"
  96         "       if(aabbIndex >= numMergedAabbs) return;\n"
  97         "       \n"
  98         "       int otherAabbIndex = aabbIndex + numRemainingAabbs;\n"
  99         "       \n"
 100         "       b3AabbCL aabb = out_mergedAabb[aabbIndex];\n"
 101         "       b3AabbCL otherAabb = out_mergedAabb[otherAabbIndex];\n"
 102         "               \n"
 103         "       b3AabbCL mergedAabb;\n"
 104         "       mergedAabb.m_min = b3Min(aabb.m_min, otherAabb.m_min);\n"
 105         "       mergedAabb.m_max = b3Max(aabb.m_max, otherAabb.m_max);\n"
 106         "       out_mergedAabb[aabbIndex] = mergedAabb;\n"
 107         "}\n"
 108         "__kernel void assignMortonCodesAndAabbIndicies(__global b3AabbCL* worldSpaceAabbs, __global b3AabbCL* mergedAabbOfAllNodes, \n"
 109         "                                                                                               __global SortDataCL* out_mortonCodesAndAabbIndices, int numAabbs)\n"
 110         "{\n"
 111         "       int leafNodeIndex = get_global_id(0);   //Leaf node index == AABB index\n"
 112         "       if(leafNodeIndex >= numAabbs) return;\n"
 113         "       \n"
 114         "       b3AabbCL mergedAabb = mergedAabbOfAllNodes[0];\n"
 115         "       b3Vector3 gridCenter = (mergedAabb.m_min + mergedAabb.m_max) * 0.5f;\n"
 116         "       b3Vector3 gridCellSize = (mergedAabb.m_max - mergedAabb.m_min) / (float)1024;\n"
 117         "       \n"
 118         "       b3AabbCL aabb = worldSpaceAabbs[leafNodeIndex];\n"
 119         "       b3Vector3 aabbCenter = (aabb.m_min + aabb.m_max) * 0.5f;\n"
 120         "       b3Vector3 aabbCenterRelativeToGrid = aabbCenter - gridCenter;\n"
 121         "       \n"
 122         "       //Quantize into integer coordinates\n"
 123         "       //floor() is needed to prevent the center cell, at (0,0,0) from being twice the size\n"
 124         "       b3Vector3 gridPosition = aabbCenterRelativeToGrid / gridCellSize;\n"
 125         "       \n"
 126         "       int4 discretePosition;\n"
 127         "       discretePosition.x = (int)( (gridPosition.x >= 0.0f) ? gridPosition.x : floor(gridPosition.x) );\n"
 128         "       discretePosition.y = (int)( (gridPosition.y >= 0.0f) ? gridPosition.y : floor(gridPosition.y) );\n"
 129         "       discretePosition.z = (int)( (gridPosition.z >= 0.0f) ? gridPosition.z : floor(gridPosition.z) );\n"
 130         "       \n"
 131         "       //Clamp coordinates into [-512, 511], then convert range from [-512, 511] to [0, 1023]\n"
 132         "       discretePosition = b3Max( -512, b3Min(discretePosition, 511) );\n"
 133         "       discretePosition += 512;\n"
 134         "       \n"
 135         "       //Interleave bits(assign a morton code, also known as a z-curve)\n"
 136         "       unsigned int mortonCode = getMortonCode(discretePosition.x, discretePosition.y, discretePosition.z);\n"
 137         "       \n"
 138         "       //\n"
 139         "       SortDataCL mortonCodeIndexPair;\n"
 140         "       mortonCodeIndexPair.m_key = mortonCode;\n"
 141         "       mortonCodeIndexPair.m_value = leafNodeIndex;\n"
 142         "       \n"
 143         "       out_mortonCodesAndAabbIndices[leafNodeIndex] = mortonCodeIndexPair;\n"
 144         "}\n"
 145         "#define B3_PLVBH_TRAVERSE_MAX_STACK_SIZE 128\n"
 146         "//The most significant bit(0x80000000) of a int32 is used to distinguish between leaf and internal nodes.\n"
 147         "//If it is set, then the index is for an internal node; otherwise, it is a leaf node. \n"
 148         "//In both cases, the bit should be cleared to access the actual node index.\n"
 149         "int isLeafNode(int index) { return (index >> 31 == 0); }\n"
 150         "int getIndexWithInternalNodeMarkerRemoved(int index) { return index & (~0x80000000); }\n"
 151         "int getIndexWithInternalNodeMarkerSet(int isLeaf, int index) { return (isLeaf) ? index : (index | 0x80000000); }\n"
 152         "//From sap.cl\n"
 153         "#define NEW_PAIR_MARKER -1\n"
 154         "bool TestAabbAgainstAabb2(const b3AabbCL* aabb1, const b3AabbCL* aabb2)\n"
 155         "{\n"
 156         "       bool overlap = true;\n"
 157         "       overlap = (aabb1->m_min.x > aabb2->m_max.x || aabb1->m_max.x < aabb2->m_min.x) ? false : overlap;\n"
 158         "       overlap = (aabb1->m_min.z > aabb2->m_max.z || aabb1->m_max.z < aabb2->m_min.z) ? false : overlap;\n"
 159         "       overlap = (aabb1->m_min.y > aabb2->m_max.y || aabb1->m_max.y < aabb2->m_min.y) ? false : overlap;\n"
 160         "       return overlap;\n"
 161         "}\n"
 162         "//From sap.cl\n"
 163         "__kernel void plbvhCalculateOverlappingPairs(__global b3AabbCL* rigidAabbs, \n"
 164         "                                                                                       __global int* rootNodeIndex, \n"
 165         "                                                                                       __global int2* internalNodeChildIndices, \n"
 166         "                                                                                       __global b3AabbCL* internalNodeAabbs,\n"
 167         "                                                                                       __global int2* internalNodeLeafIndexRanges,\n"
 168         "                                                                                       \n"
 169         "                                                                                       __global SortDataCL* mortonCodesAndAabbIndices,\n"
 170         "                                                                                       __global int* out_numPairs, __global int4* out_overlappingPairs, \n"
 171         "                                                                                       int maxPairs, int numQueryAabbs)\n"
 172         "{\n"
 173         "       //Using get_group_id()/get_local_id() is Faster than get_global_id(0) since\n"
 174         "       //mortonCodesAndAabbIndices[] contains rigid body indices sorted along the z-curve (more spatially coherent)\n"
 175         "       int queryBvhNodeIndex = get_group_id(0) * get_local_size(0) + get_local_id(0);\n"
 176         "       if(queryBvhNodeIndex >= numQueryAabbs) return;\n"
 177         "       \n"
 178         "       int queryRigidIndex = mortonCodesAndAabbIndices[queryBvhNodeIndex].m_value;\n"
 179         "       b3AabbCL queryAabb = rigidAabbs[queryRigidIndex];\n"
 180         "       \n"
 181         "       int stack[B3_PLVBH_TRAVERSE_MAX_STACK_SIZE];\n"
 182         "       \n"
 183         "       int stackSize = 1;\n"
 184         "       stack[0] = *rootNodeIndex;\n"
 185         "       \n"
 186         "       while(stackSize)\n"
 187         "       {\n"
 188         "               int internalOrLeafNodeIndex = stack[ stackSize - 1 ];\n"
 189         "               --stackSize;\n"
 190         "               \n"
 191         "               int isLeaf = isLeafNode(internalOrLeafNodeIndex);       //Internal node if false\n"
 192         "               int bvhNodeIndex = getIndexWithInternalNodeMarkerRemoved(internalOrLeafNodeIndex);\n"
 193         "               \n"
 194         "               //Optimization - if the BVH is structured as a binary radix tree, then\n"
 195         "               //each internal node corresponds to a contiguous range of leaf nodes(internalNodeLeafIndexRanges[]).\n"
 196         "               //This can be used to avoid testing each AABB-AABB pair twice, including preventing each node from colliding with itself.\n"
 197         "               {\n"
 198         "                       int highestLeafIndex = (isLeaf) ? bvhNodeIndex : internalNodeLeafIndexRanges[bvhNodeIndex].y;\n"
 199         "                       if(highestLeafIndex <= queryBvhNodeIndex) continue;\n"
 200         "               }\n"
 201         "               \n"
 202         "               //bvhRigidIndex is not used if internal node\n"
 203         "               int bvhRigidIndex = (isLeaf) ? mortonCodesAndAabbIndices[bvhNodeIndex].m_value : -1;\n"
 204         "       \n"
 205         "               b3AabbCL bvhNodeAabb = (isLeaf) ? rigidAabbs[bvhRigidIndex] : internalNodeAabbs[bvhNodeIndex];\n"
 206         "               if( TestAabbAgainstAabb2(&queryAabb, &bvhNodeAabb) )\n"
 207         "               {\n"
 208         "                       if(isLeaf)\n"
 209         "                       {\n"
 210         "                               int4 pair;\n"
 211         "                               pair.x = rigidAabbs[queryRigidIndex].m_minIndices[3];\n"
 212         "                               pair.y = rigidAabbs[bvhRigidIndex].m_minIndices[3];\n"
 213         "                               pair.z = NEW_PAIR_MARKER;\n"
 214         "                               pair.w = NEW_PAIR_MARKER;\n"
 215         "                               \n"
 216         "                               int pairIndex = atomic_inc(out_numPairs);\n"
 217         "                               if(pairIndex < maxPairs) out_overlappingPairs[pairIndex] = pair;\n"
 218         "                       }\n"
 219         "                       \n"
 220         "                       if(!isLeaf)     //Internal node\n"
 221         "                       {\n"
 222         "                               if(stackSize + 2 > B3_PLVBH_TRAVERSE_MAX_STACK_SIZE)\n"
 223         "                               {\n"
 224         "                                       //Error\n"
 225         "                               }\n"
 226         "                               else\n"
 227         "                               {\n"
 228         "                                       stack[ stackSize++ ] = internalNodeChildIndices[bvhNodeIndex].x;\n"
 229         "                                       stack[ stackSize++ ] = internalNodeChildIndices[bvhNodeIndex].y;\n"
 230         "                               }\n"
 231         "                       }\n"
 232         "               }\n"
 233         "               \n"
 234         "       }\n"
 235         "}\n"
 236         "//From rayCastKernels.cl\n"
 237         "typedef struct\n"
 238         "{\n"
 239         "       float4 m_from;\n"
 240         "       float4 m_to;\n"
 241         "} b3RayInfo;\n"
 242         "//From rayCastKernels.cl\n"
 243         "b3Vector3 b3Vector3_normalize(b3Vector3 v)\n"
 244         "{\n"
 245         "       b3Vector3 normal = (b3Vector3){v.x, v.y, v.z, 0.f};\n"
 246         "       return normalize(normal);       //OpenCL normalize == vector4 normalize\n"
 247         "}\n"
 248         "b3Scalar b3Vector3_length2(b3Vector3 v) { return v.x*v.x + v.y*v.y + v.z*v.z; }\n"
 249         "b3Scalar b3Vector3_dot(b3Vector3 a, b3Vector3 b) { return a.x*b.x + a.y*b.y + a.z*b.z; }\n"
 250         "int rayIntersectsAabb(b3Vector3 rayOrigin, b3Scalar rayLength, b3Vector3 rayNormalizedDirection, b3AabbCL aabb)\n"
 251         "{\n"
 252         "       //AABB is considered as 3 pairs of 2 planes( {x_min, x_max}, {y_min, y_max}, {z_min, z_max} ).\n"
 253         "       //t_min is the point of intersection with the closer plane, t_max is the point of intersection with the farther plane.\n"
 254         "       //\n"
 255         "       //if (rayNormalizedDirection.x < 0.0f), then max.x will be the near plane \n"
 256         "       //and min.x will be the far plane; otherwise, it is reversed.\n"
 257         "       //\n"
 258         "       //In order for there to be a collision, the t_min and t_max of each pair must overlap.\n"
 259         "       //This can be tested for by selecting the highest t_min and lowest t_max and comparing them.\n"
 260         "       \n"
 261         "       int4 isNegative = isless( rayNormalizedDirection, ((b3Vector3){0.0f, 0.0f, 0.0f, 0.0f}) );      //isless(x,y) returns (x < y)\n"
 262         "       \n"
 263         "       //When using vector types, the select() function checks the most signficant bit, \n"
 264         "       //but isless() sets the least significant bit.\n"
 265         "       isNegative <<= 31;\n"
 266         "       //select(b, a, condition) == condition ? a : b\n"
 267         "       //When using select() with vector types, (condition[i]) is true if its most significant bit is 1\n"
 268         "       b3Vector3 t_min = ( select(aabb.m_min, aabb.m_max, isNegative) - rayOrigin ) / rayNormalizedDirection;\n"
 269         "       b3Vector3 t_max = ( select(aabb.m_max, aabb.m_min, isNegative) - rayOrigin ) / rayNormalizedDirection;\n"
 270         "       \n"
 271         "       b3Scalar t_min_final = 0.0f;\n"
 272         "       b3Scalar t_max_final = rayLength;\n"
 273         "       \n"
 274         "       //Must use fmin()/fmax(); if one of the parameters is NaN, then the parameter that is not NaN is returned. \n"
 275         "       //Behavior of min()/max() with NaNs is undefined. (See OpenCL Specification 1.2 [6.12.2] and [6.12.4])\n"
 276         "       //Since the innermost fmin()/fmax() is always not NaN, this should never return NaN.\n"
 277         "       t_min_final = fmax( t_min.z, fmax(t_min.y, fmax(t_min.x, t_min_final)) );\n"
 278         "       t_max_final = fmin( t_max.z, fmin(t_max.y, fmin(t_max.x, t_max_final)) );\n"
 279         "       \n"
 280         "       return (t_min_final <= t_max_final);\n"
 281         "}\n"
 282         "__kernel void plbvhRayTraverse(__global b3AabbCL* rigidAabbs,\n"
 283         "                                                               __global int* rootNodeIndex, \n"
 284         "                                                               __global int2* internalNodeChildIndices, \n"
 285         "                                                               __global b3AabbCL* internalNodeAabbs,\n"
 286         "                                                               __global int2* internalNodeLeafIndexRanges,\n"
 287         "                                                               __global SortDataCL* mortonCodesAndAabbIndices,\n"
 288         "                                                               \n"
 289         "                                                               __global b3RayInfo* rays,\n"
 290         "                                                               \n"
 291         "                                                               __global int* out_numRayRigidPairs, \n"
 292         "                                                               __global int2* out_rayRigidPairs,\n"
 293         "                                                               int maxRayRigidPairs, int numRays)\n"
 294         "{\n"
 295         "       int rayIndex = get_global_id(0);\n"
 296         "       if(rayIndex >= numRays) return;\n"
 297         "       \n"
 298         "       //\n"
 299         "       b3Vector3 rayFrom = rays[rayIndex].m_from;\n"
 300         "       b3Vector3 rayTo = rays[rayIndex].m_to;\n"
 301         "       b3Vector3 rayNormalizedDirection = b3Vector3_normalize(rayTo - rayFrom);\n"
 302         "       b3Scalar rayLength = b3Sqrt( b3Vector3_length2(rayTo - rayFrom) );\n"
 303         "       \n"
 304         "       //\n"
 305         "       int stack[B3_PLVBH_TRAVERSE_MAX_STACK_SIZE];\n"
 306         "       \n"
 307         "       int stackSize = 1;\n"
 308         "       stack[0] = *rootNodeIndex;\n"
 309         "       \n"
 310         "       while(stackSize)\n"
 311         "       {\n"
 312         "               int internalOrLeafNodeIndex = stack[ stackSize - 1 ];\n"
 313         "               --stackSize;\n"
 314         "               \n"
 315         "               int isLeaf = isLeafNode(internalOrLeafNodeIndex);       //Internal node if false\n"
 316         "               int bvhNodeIndex = getIndexWithInternalNodeMarkerRemoved(internalOrLeafNodeIndex);\n"
 317         "               \n"
 318         "               //bvhRigidIndex is not used if internal node\n"
 319         "               int bvhRigidIndex = (isLeaf) ? mortonCodesAndAabbIndices[bvhNodeIndex].m_value : -1;\n"
 320         "       \n"
 321         "               b3AabbCL bvhNodeAabb = (isLeaf) ? rigidAabbs[bvhRigidIndex] : internalNodeAabbs[bvhNodeIndex];\n"
 322         "               if( rayIntersectsAabb(rayFrom, rayLength, rayNormalizedDirection, bvhNodeAabb)  )\n"
 323         "               {\n"
 324         "                       if(isLeaf)\n"
 325         "                       {\n"
 326         "                               int2 rayRigidPair;\n"
 327         "                               rayRigidPair.x = rayIndex;\n"
 328         "                               rayRigidPair.y = rigidAabbs[bvhRigidIndex].m_minIndices[3];\n"
 329         "                               \n"
 330         "                               int pairIndex = atomic_inc(out_numRayRigidPairs);\n"
 331         "                               if(pairIndex < maxRayRigidPairs) out_rayRigidPairs[pairIndex] = rayRigidPair;\n"
 332         "                       }\n"
 333         "                       \n"
 334         "                       if(!isLeaf)     //Internal node\n"
 335         "                       {\n"
 336         "                               if(stackSize + 2 > B3_PLVBH_TRAVERSE_MAX_STACK_SIZE)\n"
 337         "                               {\n"
 338         "                                       //Error\n"
 339         "                               }\n"
 340         "                               else\n"
 341         "                               {\n"
 342         "                                       stack[ stackSize++ ] = internalNodeChildIndices[bvhNodeIndex].x;\n"
 343         "                                       stack[ stackSize++ ] = internalNodeChildIndices[bvhNodeIndex].y;\n"
 344         "                               }\n"
 345         "                       }\n"
 346         "               }\n"
 347         "       }\n"
 348         "}\n"
 349         "__kernel void plbvhLargeAabbAabbTest(__global b3AabbCL* smallAabbs, __global b3AabbCL* largeAabbs, \n"
 350         "                                                                       __global int* out_numPairs, __global int4* out_overlappingPairs, \n"
 351         "                                                                       int maxPairs, int numLargeAabbRigids, int numSmallAabbRigids)\n"
 352         "{\n"
 353         "       int smallAabbIndex = get_global_id(0);\n"
 354         "       if(smallAabbIndex >= numSmallAabbRigids) return;\n"
 355         "       \n"
 356         "       b3AabbCL smallAabb = smallAabbs[smallAabbIndex];\n"
 357         "       for(int i = 0; i < numLargeAabbRigids; ++i)\n"
 358         "       {\n"
 359         "               b3AabbCL largeAabb = largeAabbs[i];\n"
 360         "               if( TestAabbAgainstAabb2(&smallAabb, &largeAabb) )\n"
 361         "               {\n"
 362         "                       int4 pair;\n"
 363         "                       pair.x = largeAabb.m_minIndices[3];\n"
 364         "                       pair.y = smallAabb.m_minIndices[3];\n"
 365         "                       pair.z = NEW_PAIR_MARKER;\n"
 366         "                       pair.w = NEW_PAIR_MARKER;\n"
 367         "                       \n"
 368         "                       int pairIndex = atomic_inc(out_numPairs);\n"
 369         "                       if(pairIndex < maxPairs) out_overlappingPairs[pairIndex] = pair;\n"
 370         "               }\n"
 371         "       }\n"
 372         "}\n"
 373         "__kernel void plbvhLargeAabbRayTest(__global b3AabbCL* largeRigidAabbs, __global b3RayInfo* rays,\n"
 374         "                                                                       __global int* out_numRayRigidPairs,  __global int2* out_rayRigidPairs,\n"
 375         "                                                                       int numLargeAabbRigids, int maxRayRigidPairs, int numRays)\n"
 376         "{\n"
 377         "       int rayIndex = get_global_id(0);\n"
 378         "       if(rayIndex >= numRays) return;\n"
 379         "       \n"
 380         "       b3Vector3 rayFrom = rays[rayIndex].m_from;\n"
 381         "       b3Vector3 rayTo = rays[rayIndex].m_to;\n"
 382         "       b3Vector3 rayNormalizedDirection = b3Vector3_normalize(rayTo - rayFrom);\n"
 383         "       b3Scalar rayLength = b3Sqrt( b3Vector3_length2(rayTo - rayFrom) );\n"
 384         "       \n"
 385         "       for(int i = 0; i < numLargeAabbRigids; ++i)\n"
 386         "       {\n"
 387         "               b3AabbCL rigidAabb = largeRigidAabbs[i];\n"
 388         "               if( rayIntersectsAabb(rayFrom, rayLength, rayNormalizedDirection, rigidAabb) )\n"
 389         "               {\n"
 390         "                       int2 rayRigidPair;\n"
 391         "                       rayRigidPair.x = rayIndex;\n"
 392         "                       rayRigidPair.y = rigidAabb.m_minIndices[3];\n"
 393         "                       \n"
 394         "                       int pairIndex = atomic_inc(out_numRayRigidPairs);\n"
 395         "                       if(pairIndex < maxRayRigidPairs) out_rayRigidPairs[pairIndex] = rayRigidPair;\n"
 396         "               }\n"
 397         "       }\n"
 398         "}\n"
 399         "//Set so that it is always greater than the actual common prefixes, and never selected as a parent node.\n"
 400         "//If there are no duplicates, then the highest common prefix is 32 or 64, depending on the number of bits used for the z-curve.\n"
 401         "//Duplicate common prefixes increase the highest common prefix at most by the number of bits used to index the leaf node.\n"
 402         "//Since 32 bit ints are used to index leaf nodes, the max prefix is 64(32 + 32 bit z-curve) or 96(32 + 64 bit z-curve).\n"
 403         "#define B3_PLBVH_INVALID_COMMON_PREFIX 128\n"
 404         "#define B3_PLBVH_ROOT_NODE_MARKER -1\n"
 405         "#define b3Int64 long\n"
 406         "int computeCommonPrefixLength(b3Int64 i, b3Int64 j) { return (int)clz(i ^ j); }\n"
 407         "b3Int64 computeCommonPrefix(b3Int64 i, b3Int64 j) \n"
 408         "{\n"
 409         "       //This function only needs to return (i & j) in order for the algorithm to work,\n"
 410         "       //but it may help with debugging to mask out the lower bits.\n"
 411         "       b3Int64 commonPrefixLength = (b3Int64)computeCommonPrefixLength(i, j);\n"
 412         "       b3Int64 sharedBits = i & j;\n"
 413         "       b3Int64 bitmask = ((b3Int64)(~0)) << (64 - commonPrefixLength); //Set all bits after the common prefix to 0\n"
 414         "       \n"
 415         "       return sharedBits & bitmask;\n"
 416         "}\n"
 417         "//Same as computeCommonPrefixLength(), but allows for prefixes with different lengths\n"
 418         "int getSharedPrefixLength(b3Int64 prefixA, int prefixLengthA, b3Int64 prefixB, int prefixLengthB)\n"
 419         "{\n"
 420         "       return b3Min( computeCommonPrefixLength(prefixA, prefixB), b3Min(prefixLengthA, prefixLengthB) );\n"
 421         "}\n"
 422         "__kernel void computeAdjacentPairCommonPrefix(__global SortDataCL* mortonCodesAndAabbIndices,\n"
 423         "                                                                                       __global b3Int64* out_commonPrefixes,\n"
 424         "                                                                                       __global int* out_commonPrefixLengths,\n"
 425         "                                                                                       int numInternalNodes)\n"
 426         "{\n"
 427         "       int internalNodeIndex = get_global_id(0);\n"
 428         "       if (internalNodeIndex >= numInternalNodes) return;\n"
 429         "       \n"
 430         "       //Here, (internalNodeIndex + 1) is never out of bounds since it is a leaf node index,\n"
 431         "       //and the number of internal nodes is always numLeafNodes - 1\n"
 432         "       int leftLeafIndex = internalNodeIndex;\n"
 433         "       int rightLeafIndex = internalNodeIndex + 1;\n"
 434         "       \n"
 435         "       int leftLeafMortonCode = mortonCodesAndAabbIndices[leftLeafIndex].m_key;\n"
 436         "       int rightLeafMortonCode = mortonCodesAndAabbIndices[rightLeafIndex].m_key;\n"
 437         "       \n"
 438         "       //Binary radix tree construction algorithm does not work if there are duplicate morton codes.\n"
 439         "       //Append the index of each leaf node to each morton code so that there are no duplicates.\n"
 440         "       //The algorithm also requires that the morton codes are sorted in ascending order; this requirement\n"
 441         "       //is also satisfied with this method, as (leftLeafIndex < rightLeafIndex) is always true.\n"
 442         "       //\n"
 443         "       //upsample(a, b) == ( ((b3Int64)a) << 32) | b\n"
 444         "       b3Int64 nonduplicateLeftMortonCode = upsample(leftLeafMortonCode, leftLeafIndex);\n"
 445         "       b3Int64 nonduplicateRightMortonCode = upsample(rightLeafMortonCode, rightLeafIndex);\n"
 446         "       \n"
 447         "       out_commonPrefixes[internalNodeIndex] = computeCommonPrefix(nonduplicateLeftMortonCode, nonduplicateRightMortonCode);\n"
 448         "       out_commonPrefixLengths[internalNodeIndex] = computeCommonPrefixLength(nonduplicateLeftMortonCode, nonduplicateRightMortonCode);\n"
 449         "}\n"
 450         "__kernel void buildBinaryRadixTreeLeafNodes(__global int* commonPrefixLengths, __global int* out_leafNodeParentNodes,\n"
 451         "                                                                                       __global int2* out_childNodes, int numLeafNodes)\n"
 452         "{\n"
 453         "       int leafNodeIndex = get_global_id(0);\n"
 454         "       if (leafNodeIndex >= numLeafNodes) return;\n"
 455         "       \n"
 456         "       int numInternalNodes = numLeafNodes - 1;\n"
 457         "       \n"
 458         "       int leftSplitIndex = leafNodeIndex - 1;\n"
 459         "       int rightSplitIndex = leafNodeIndex;\n"
 460         "       \n"
 461         "       int leftCommonPrefix = (leftSplitIndex >= 0) ? commonPrefixLengths[leftSplitIndex] : B3_PLBVH_INVALID_COMMON_PREFIX;\n"
 462         "       int rightCommonPrefix = (rightSplitIndex < numInternalNodes) ? commonPrefixLengths[rightSplitIndex] : B3_PLBVH_INVALID_COMMON_PREFIX;\n"
 463         "       \n"
 464         "       //Parent node is the highest adjacent common prefix that is lower than the node's common prefix\n"
 465         "       //Leaf nodes are considered as having the highest common prefix\n"
 466         "       int isLeftHigherCommonPrefix = (leftCommonPrefix > rightCommonPrefix);\n"
 467         "       \n"
 468         "       //Handle cases for the edge nodes; the first and last node\n"
 469         "       //For leaf nodes, leftCommonPrefix and rightCommonPrefix should never both be B3_PLBVH_INVALID_COMMON_PREFIX\n"
 470         "       if(leftCommonPrefix == B3_PLBVH_INVALID_COMMON_PREFIX) isLeftHigherCommonPrefix = false;\n"
 471         "       if(rightCommonPrefix == B3_PLBVH_INVALID_COMMON_PREFIX) isLeftHigherCommonPrefix = true;\n"
 472         "       \n"
 473         "       int parentNodeIndex = (isLeftHigherCommonPrefix) ? leftSplitIndex : rightSplitIndex;\n"
 474         "       out_leafNodeParentNodes[leafNodeIndex] = parentNodeIndex;\n"
 475         "       \n"
 476         "       int isRightChild = (isLeftHigherCommonPrefix);  //If the left node is the parent, then this node is its right child and vice versa\n"
 477         "       \n"
 478         "       //out_childNodesAsInt[0] == int2.x == left child\n"
 479         "       //out_childNodesAsInt[1] == int2.y == right child\n"
 480         "       int isLeaf = 1;\n"
 481         "       __global int* out_childNodesAsInt = (__global int*)(&out_childNodes[parentNodeIndex]);\n"
 482         "       out_childNodesAsInt[isRightChild] = getIndexWithInternalNodeMarkerSet(isLeaf, leafNodeIndex);\n"
 483         "}\n"
 484         "__kernel void buildBinaryRadixTreeInternalNodes(__global b3Int64* commonPrefixes, __global int* commonPrefixLengths,\n"
 485         "                                                                                               __global int2* out_childNodes,\n"
 486         "                                                                                               __global int* out_internalNodeParentNodes, __global int* out_rootNodeIndex,\n"
 487         "                                                                                               int numInternalNodes)\n"
 488         "{\n"
 489         "       int internalNodeIndex = get_group_id(0) * get_local_size(0) + get_local_id(0);\n"
 490         "       if(internalNodeIndex >= numInternalNodes) return;\n"
 491         "       \n"
 492         "       b3Int64 nodePrefix = commonPrefixes[internalNodeIndex];\n"
 493         "       int nodePrefixLength = commonPrefixLengths[internalNodeIndex];\n"
 494         "       \n"
 495         "//#define USE_LINEAR_SEARCH\n"
 496         "#ifdef USE_LINEAR_SEARCH\n"
 497         "       int leftIndex = -1;\n"
 498         "       int rightIndex = -1;\n"
 499         "       \n"
 500         "       //Find nearest element to left with a lower common prefix\n"
 501         "       for(int i = internalNodeIndex - 1; i >= 0; --i)\n"
 502         "       {\n"
 503         "               int nodeLeftSharedPrefixLength = getSharedPrefixLength(nodePrefix, nodePrefixLength, commonPrefixes[i], commonPrefixLengths[i]);\n"
 504         "               if(nodeLeftSharedPrefixLength < nodePrefixLength)\n"
 505         "               {\n"
 506         "                       leftIndex = i;\n"
 507         "                       break;\n"
 508         "               }\n"
 509         "       }\n"
 510         "       \n"
 511         "       //Find nearest element to right with a lower common prefix\n"
 512         "       for(int i = internalNodeIndex + 1; i < numInternalNodes; ++i)\n"
 513         "       {\n"
 514         "               int nodeRightSharedPrefixLength = getSharedPrefixLength(nodePrefix, nodePrefixLength, commonPrefixes[i], commonPrefixLengths[i]);\n"
 515         "               if(nodeRightSharedPrefixLength < nodePrefixLength)\n"
 516         "               {\n"
 517         "                       rightIndex = i;\n"
 518         "                       break;\n"
 519         "               }\n"
 520         "       }\n"
 521         "       \n"
 522         "#else //Use binary search\n"
 523         "       //Find nearest element to left with a lower common prefix\n"
 524         "       int leftIndex = -1;\n"
 525         "       {\n"
 526         "               int lower = 0;\n"
 527         "               int upper = internalNodeIndex - 1;\n"
 528         "               \n"
 529         "               while(lower <= upper)\n"
 530         "               {\n"
 531         "                       int mid = (lower + upper) / 2;\n"
 532         "                       b3Int64 midPrefix = commonPrefixes[mid];\n"
 533         "                       int midPrefixLength = commonPrefixLengths[mid];\n"
 534         "                       \n"
 535         "                       int nodeMidSharedPrefixLength = getSharedPrefixLength(nodePrefix, nodePrefixLength, midPrefix, midPrefixLength);\n"
 536         "                       if(nodeMidSharedPrefixLength < nodePrefixLength) \n"
 537         "                       {\n"
 538         "                               int right = mid + 1;\n"
 539         "                               if(right < internalNodeIndex)\n"
 540         "                               {\n"
 541         "                                       b3Int64 rightPrefix = commonPrefixes[right];\n"
 542         "                                       int rightPrefixLength = commonPrefixLengths[right];\n"
 543         "                                       \n"
 544         "                                       int nodeRightSharedPrefixLength = getSharedPrefixLength(nodePrefix, nodePrefixLength, rightPrefix, rightPrefixLength);\n"
 545         "                                       if(nodeRightSharedPrefixLength < nodePrefixLength) \n"
 546         "                                       {\n"
 547         "                                               lower = right;\n"
 548         "                                               leftIndex = right;\n"
 549         "                                       }\n"
 550         "                                       else \n"
 551         "                                       {\n"
 552         "                                               leftIndex = mid;\n"
 553         "                                               break;\n"
 554         "                                       }\n"
 555         "                               }\n"
 556         "                               else \n"
 557         "                               {\n"
 558         "                                       leftIndex = mid;\n"
 559         "                                       break;\n"
 560         "                               }\n"
 561         "                       }\n"
 562         "                       else upper = mid - 1;\n"
 563         "               }\n"
 564         "       }\n"
 565         "       \n"
 566         "       //Find nearest element to right with a lower common prefix\n"
 567         "       int rightIndex = -1;\n"
 568         "       {\n"
 569         "               int lower = internalNodeIndex + 1;\n"
 570         "               int upper = numInternalNodes - 1;\n"
 571         "               \n"
 572         "               while(lower <= upper)\n"
 573         "               {\n"
 574         "                       int mid = (lower + upper) / 2;\n"
 575         "                       b3Int64 midPrefix = commonPrefixes[mid];\n"
 576         "                       int midPrefixLength = commonPrefixLengths[mid];\n"
 577         "                       \n"
 578         "                       int nodeMidSharedPrefixLength = getSharedPrefixLength(nodePrefix, nodePrefixLength, midPrefix, midPrefixLength);\n"
 579         "                       if(nodeMidSharedPrefixLength < nodePrefixLength) \n"
 580         "                       {\n"
 581         "                               int left = mid - 1;\n"
 582         "                               if(left > internalNodeIndex)\n"
 583         "                               {\n"
 584         "                                       b3Int64 leftPrefix = commonPrefixes[left];\n"
 585         "                                       int leftPrefixLength = commonPrefixLengths[left];\n"
 586         "                               \n"
 587         "                                       int nodeLeftSharedPrefixLength = getSharedPrefixLength(nodePrefix, nodePrefixLength, leftPrefix, leftPrefixLength);\n"
 588         "                                       if(nodeLeftSharedPrefixLength < nodePrefixLength) \n"
 589         "                                       {\n"
 590         "                                               upper = left;\n"
 591         "                                               rightIndex = left;\n"
 592         "                                       }\n"
 593         "                                       else \n"
 594         "                                       {\n"
 595         "                                               rightIndex = mid;\n"
 596         "                                               break;\n"
 597         "                                       }\n"
 598         "                               }\n"
 599         "                               else \n"
 600         "                               {\n"
 601         "                                       rightIndex = mid;\n"
 602         "                                       break;\n"
 603         "                               }\n"
 604         "                       }\n"
 605         "                       else lower = mid + 1;\n"
 606         "               }\n"
 607         "       }\n"
 608         "#endif\n"
 609         "       \n"
 610         "       //Select parent\n"
 611         "       {\n"
 612         "               int leftPrefixLength = (leftIndex != -1) ? commonPrefixLengths[leftIndex] : B3_PLBVH_INVALID_COMMON_PREFIX;\n"
 613         "               int rightPrefixLength =  (rightIndex != -1) ? commonPrefixLengths[rightIndex] : B3_PLBVH_INVALID_COMMON_PREFIX;\n"
 614         "               \n"
 615         "               int isLeftHigherPrefixLength = (leftPrefixLength > rightPrefixLength);\n"
 616         "               \n"
 617         "               if(leftPrefixLength == B3_PLBVH_INVALID_COMMON_PREFIX) isLeftHigherPrefixLength = false;\n"
 618         "               else if(rightPrefixLength == B3_PLBVH_INVALID_COMMON_PREFIX) isLeftHigherPrefixLength = true;\n"
 619         "               \n"
 620         "               int parentNodeIndex = (isLeftHigherPrefixLength) ? leftIndex : rightIndex;\n"
 621         "               \n"
 622         "               int isRootNode = (leftIndex == -1 && rightIndex == -1);\n"
 623         "               out_internalNodeParentNodes[internalNodeIndex] = (!isRootNode) ? parentNodeIndex : B3_PLBVH_ROOT_NODE_MARKER;\n"
 624         "               \n"
 625         "               int isLeaf = 0;\n"
 626         "               if(!isRootNode)\n"
 627         "               {\n"
 628         "                       int isRightChild = (isLeftHigherPrefixLength);  //If the left node is the parent, then this node is its right child and vice versa\n"
 629         "                       \n"
 630         "                       //out_childNodesAsInt[0] == int2.x == left child\n"
 631         "                       //out_childNodesAsInt[1] == int2.y == right child\n"
 632         "                       __global int* out_childNodesAsInt = (__global int*)(&out_childNodes[parentNodeIndex]);\n"
 633         "                       out_childNodesAsInt[isRightChild] = getIndexWithInternalNodeMarkerSet(isLeaf, internalNodeIndex);\n"
 634         "               }\n"
 635         "               else *out_rootNodeIndex = getIndexWithInternalNodeMarkerSet(isLeaf, internalNodeIndex);\n"
 636         "       }\n"
 637         "}\n"
 638         "__kernel void findDistanceFromRoot(__global int* rootNodeIndex, __global int* internalNodeParentNodes,\n"
 639         "                                                                       __global int* out_maxDistanceFromRoot, __global int* out_distanceFromRoot, int numInternalNodes)\n"
 640         "{\n"
 641         "       if( get_global_id(0) == 0 ) atomic_xchg(out_maxDistanceFromRoot, 0);\n"
 642         "       int internalNodeIndex = get_global_id(0);\n"
 643         "       if(internalNodeIndex >= numInternalNodes) return;\n"
 644         "       \n"
 645         "       //\n"
 646         "       int distanceFromRoot = 0;\n"
 647         "       {\n"
 648         "               int parentIndex = internalNodeParentNodes[internalNodeIndex];\n"
 649         "               while(parentIndex != B3_PLBVH_ROOT_NODE_MARKER)\n"
 650         "               {\n"
 651         "                       parentIndex = internalNodeParentNodes[parentIndex];\n"
 652         "                       ++distanceFromRoot;\n"
 653         "               }\n"
 654         "       }\n"
 655         "       out_distanceFromRoot[internalNodeIndex] = distanceFromRoot;\n"
 656         "       \n"
 657         "       //\n"
 658         "       __local int localMaxDistanceFromRoot;\n"
 659         "       if( get_local_id(0) == 0 ) localMaxDistanceFromRoot = 0;\n"
 660         "       barrier(CLK_LOCAL_MEM_FENCE);\n"
 661         "       \n"
 662         "       atomic_max(&localMaxDistanceFromRoot, distanceFromRoot);\n"
 663         "       barrier(CLK_LOCAL_MEM_FENCE);\n"
 664         "       \n"
 665         "       if( get_local_id(0) == 0 ) atomic_max(out_maxDistanceFromRoot, localMaxDistanceFromRoot);\n"
 666         "}\n"
 667         "__kernel void buildBinaryRadixTreeAabbsRecursive(__global int* distanceFromRoot, __global SortDataCL* mortonCodesAndAabbIndices,\n"
 668         "                                                                                               __global int2* childNodes,\n"
 669         "                                                                                               __global b3AabbCL* leafNodeAabbs, __global b3AabbCL* internalNodeAabbs,\n"
 670         "                                                                                               int maxDistanceFromRoot, int processedDistance, int numInternalNodes)\n"
 671         "{\n"
 672         "       int internalNodeIndex = get_global_id(0);\n"
 673         "       if(internalNodeIndex >= numInternalNodes) return;\n"
 674         "       \n"
 675         "       int distance = distanceFromRoot[internalNodeIndex];\n"
 676         "       \n"
 677         "       if(distance == processedDistance)\n"
 678         "       {\n"
 679         "               int leftChildIndex = childNodes[internalNodeIndex].x;\n"
 680         "               int rightChildIndex = childNodes[internalNodeIndex].y;\n"
 681         "               \n"
 682         "               int isLeftChildLeaf = isLeafNode(leftChildIndex);\n"
 683         "               int isRightChildLeaf = isLeafNode(rightChildIndex);\n"
 684         "               \n"
 685         "               leftChildIndex = getIndexWithInternalNodeMarkerRemoved(leftChildIndex);\n"
 686         "               rightChildIndex = getIndexWithInternalNodeMarkerRemoved(rightChildIndex);\n"
 687         "               \n"
 688         "               //leftRigidIndex/rightRigidIndex is not used if internal node\n"
 689         "               int leftRigidIndex = (isLeftChildLeaf) ? mortonCodesAndAabbIndices[leftChildIndex].m_value : -1;\n"
 690         "               int rightRigidIndex = (isRightChildLeaf) ? mortonCodesAndAabbIndices[rightChildIndex].m_value : -1;\n"
 691         "               \n"
 692         "               b3AabbCL leftChildAabb = (isLeftChildLeaf) ? leafNodeAabbs[leftRigidIndex] : internalNodeAabbs[leftChildIndex];\n"
 693         "               b3AabbCL rightChildAabb = (isRightChildLeaf) ? leafNodeAabbs[rightRigidIndex] : internalNodeAabbs[rightChildIndex];\n"
 694         "               \n"
 695         "               b3AabbCL mergedAabb;\n"
 696         "               mergedAabb.m_min = b3Min(leftChildAabb.m_min, rightChildAabb.m_min);\n"
 697         "               mergedAabb.m_max = b3Max(leftChildAabb.m_max, rightChildAabb.m_max);\n"
 698         "               internalNodeAabbs[internalNodeIndex] = mergedAabb;\n"
 699         "       }\n"
 700         "}\n"
 701         "__kernel void findLeafIndexRanges(__global int2* internalNodeChildNodes, __global int2* out_leafIndexRanges, int numInternalNodes)\n"
 702         "{\n"
 703         "       int internalNodeIndex = get_global_id(0);\n"
 704         "       if(internalNodeIndex >= numInternalNodes) return;\n"
 705         "       \n"
 706         "       int numLeafNodes = numInternalNodes + 1;\n"
 707         "       \n"
 708         "       int2 childNodes = internalNodeChildNodes[internalNodeIndex];\n"
 709         "       \n"
 710         "       int2 leafIndexRange;    //x == min leaf index, y == max leaf index\n"
 711         "       \n"
 712         "       //Find lowest leaf index covered by this internal node\n"
 713         "       {\n"
 714         "               int lowestIndex = childNodes.x;         //childNodes.x == Left child\n"
 715         "               while( !isLeafNode(lowestIndex) ) lowestIndex = internalNodeChildNodes[ getIndexWithInternalNodeMarkerRemoved(lowestIndex) ].x;\n"
 716         "               leafIndexRange.x = lowestIndex;\n"
 717         "       }\n"
 718         "       \n"
 719         "       //Find highest leaf index covered by this internal node\n"
 720         "       {\n"
 721         "               int highestIndex = childNodes.y;        //childNodes.y == Right child\n"
 722         "               while( !isLeafNode(highestIndex) ) highestIndex = internalNodeChildNodes[ getIndexWithInternalNodeMarkerRemoved(highestIndex) ].y;\n"
 723         "               leafIndexRange.y = highestIndex;\n"
 724         "       }\n"
 725         "       \n"
 726         "       //\n"
 727         "       out_leafIndexRanges[internalNodeIndex] = leafIndexRange;\n"
 728         "}\n";