src/gallium/auxiliary/draw/draw_llvm.c

   1 /**************************************************************************
   2  *
   3  * Copyright 2010 VMware, Inc.
   4  * All Rights Reserved.
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a
   7  * copy of this software and associated documentation files (the
   8  * "Software"), to deal in the Software without restriction, including
   9  * without limitation the rights to use, copy, modify, merge, publish,
  10  * distribute, sub license, and/or sell copies of the Software, and to
  11  * permit persons to whom the Software is furnished to do so, subject to
  12  * the following conditions:
  13  *
  14  * The above copyright notice and this permission notice (including the
  15  * next paragraph) shall be included in all copies or substantial portions
  16  * of the Software.
  17  *
  18  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
  19  * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  20  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
  21  * IN NO EVENT SHALL VMWARE AND/OR ITS SUPPLIERS BE LIABLE FOR
  22  * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
  23  * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
  24  * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  25  *
  26  **************************************************************************/
  27
  28 #include "draw_llvm.h"
  29
  30 #include "draw_context.h"
  31 #include "draw_vs.h"
  32
  33 #include "gallivm/lp_bld_arit.h"
  34 #include "gallivm/lp_bld_logic.h"
  35 #include "gallivm/lp_bld_const.h"
  36 #include "gallivm/lp_bld_swizzle.h"
  37 #include "gallivm/lp_bld_struct.h"
  38 #include "gallivm/lp_bld_type.h"
  39 #include "gallivm/lp_bld_flow.h"
  40 #include "gallivm/lp_bld_debug.h"
  41 #include "gallivm/lp_bld_tgsi.h"
  42 #include "gallivm/lp_bld_printf.h"
  43 #include "gallivm/lp_bld_intr.h"
  44 #include "gallivm/lp_bld_init.h"
  45 #include "gallivm/lp_bld_type.h"
  46
  47 #include "tgsi/tgsi_exec.h"
  48 #include "tgsi/tgsi_dump.h"
  49
  50 #include "util/u_math.h"
  51 #include "util/u_pointer.h"
  52 #include "util/u_string.h"
  53 #include "util/u_simple_list.h"
  54
  55
  56 #define DEBUG_STORE 0
  57
  58
  59 /**
  60  * This function is called by the gallivm "garbage collector" when
  61  * the LLVM global data structures are freed.  We must free all LLVM-related
  62  * data.  Specifically, all JIT'd shader variants.
  63  */
  64 static void
  65 draw_llvm_garbage_collect_callback(void *cb_data)
  66 {
  67    struct draw_llvm *llvm = (struct draw_llvm *) cb_data;
  68    struct draw_llvm_variant_list_item *li;
  69
  70    /* free all shader variants */
  71    li = first_elem(&llvm->vs_variants_list);
  72    while (!at_end(&llvm->vs_variants_list, li)) {
  73       struct draw_llvm_variant_list_item *next = next_elem(li);
  74       draw_llvm_destroy_variant(li->base);
  75       li = next;
  76    }
  77
  78    /* Null-out these pointers so they get remade next time they're needed.
  79     * See the accessor functions below.
  80     */
  81    llvm->context_ptr_type = NULL;
  82    llvm->buffer_ptr_type = NULL;
  83    llvm->vb_ptr_type = NULL;
  84    llvm->vertex_header_ptr_type = NULL;
  85 }
  86
  87
  88 static void
  89 draw_llvm_generate(struct draw_llvm *llvm, struct draw_llvm_variant *var);
  90
  91 static void
  92 draw_llvm_generate_elts(struct draw_llvm *llvm, struct draw_llvm_variant *var);
  93
  94
  95 /**
  96  * Create LLVM type for struct draw_jit_texture
  97  */
  98 static LLVMTypeRef
  99 create_jit_texture_type(struct gallivm_state *gallivm)
 100 {
 101    LLVMTargetDataRef target = gallivm->target;
 102    LLVMTypeRef texture_type;
 103    LLVMTypeRef elem_types[DRAW_JIT_TEXTURE_NUM_FIELDS];
 104    LLVMTypeRef int32_type = LLVMInt32TypeInContext(gallivm->context);
 105
 106    elem_types[DRAW_JIT_TEXTURE_WIDTH]  =
 107    elem_types[DRAW_JIT_TEXTURE_HEIGHT] =
 108    elem_types[DRAW_JIT_TEXTURE_DEPTH] =
 109    elem_types[DRAW_JIT_TEXTURE_FIRST_LEVEL] =
 110    elem_types[DRAW_JIT_TEXTURE_LAST_LEVEL] = int32_type;
 111    elem_types[DRAW_JIT_TEXTURE_ROW_STRIDE] =
 112    elem_types[DRAW_JIT_TEXTURE_IMG_STRIDE] =
 113       LLVMArrayType(int32_type, PIPE_MAX_TEXTURE_LEVELS);
 114    elem_types[DRAW_JIT_TEXTURE_DATA] =
 115       LLVMArrayType(LLVMPointerType(LLVMInt8TypeInContext(gallivm->context), 0),
 116                     PIPE_MAX_TEXTURE_LEVELS);
 117    elem_types[DRAW_JIT_TEXTURE_MIN_LOD] =
 118    elem_types[DRAW_JIT_TEXTURE_MAX_LOD] =
 119    elem_types[DRAW_JIT_TEXTURE_LOD_BIAS] = LLVMFloatTypeInContext(gallivm->context);
 120    elem_types[DRAW_JIT_TEXTURE_BORDER_COLOR] =
 121       LLVMArrayType(LLVMFloatTypeInContext(gallivm->context), 4);
 122
 123    texture_type = LLVMStructTypeInContext(gallivm->context, elem_types,
 124                                           Elements(elem_types), 0);
 125
 126    /* Make sure the target's struct layout cache doesn't return
 127     * stale/invalid data.
 128     */
 129    LLVMInvalidateStructLayout(gallivm->target, texture_type);
 130
 131    LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, width,
 132                           target, texture_type,
 133                           DRAW_JIT_TEXTURE_WIDTH);
 134    LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, height,
 135                           target, texture_type,
 136                           DRAW_JIT_TEXTURE_HEIGHT);
 137    LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, depth,
 138                           target, texture_type,
 139                           DRAW_JIT_TEXTURE_DEPTH);
 140    LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, first_level,
 141                           target, texture_type,
 142                           DRAW_JIT_TEXTURE_FIRST_LEVEL);
 143    LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, last_level,
 144                           target, texture_type,
 145                           DRAW_JIT_TEXTURE_LAST_LEVEL);
 146    LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, row_stride,
 147                           target, texture_type,
 148                           DRAW_JIT_TEXTURE_ROW_STRIDE);
 149    LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, img_stride,
 150                           target, texture_type,
 151                           DRAW_JIT_TEXTURE_IMG_STRIDE);
 152    LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, data,
 153                           target, texture_type,
 154                           DRAW_JIT_TEXTURE_DATA);
 155    LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, min_lod,
 156                           target, texture_type,
 157                           DRAW_JIT_TEXTURE_MIN_LOD);
 158    LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, max_lod,
 159                           target, texture_type,
 160                           DRAW_JIT_TEXTURE_MAX_LOD);
 161    LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, lod_bias,
 162                           target, texture_type,
 163                           DRAW_JIT_TEXTURE_LOD_BIAS);
 164    LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, border_color,
 165                           target, texture_type,
 166                           DRAW_JIT_TEXTURE_BORDER_COLOR);
 167
 168    LP_CHECK_STRUCT_SIZE(struct draw_jit_texture, target, texture_type);
 169
 170    return texture_type;
 171 }
 172
 173
 174 /**
 175  * Create LLVM type for struct draw_jit_texture
 176  */
 177 static LLVMTypeRef
 178 create_jit_context_type(struct gallivm_state *gallivm,
 179                         LLVMTypeRef texture_type)
 180 {
 181    LLVMTargetDataRef target = gallivm->target;
 182    LLVMTypeRef float_type = LLVMFloatTypeInContext(gallivm->context);
 183    LLVMTypeRef elem_types[5];
 184    LLVMTypeRef context_type;
 185
 186    elem_types[0] = LLVMPointerType(float_type, 0); /* vs_constants */
 187    elem_types[1] = LLVMPointerType(float_type, 0); /* gs_constants */
 188    elem_types[2] = LLVMPointerType(LLVMArrayType(LLVMArrayType(float_type, 4), 12), 0); /* planes */
 189    elem_types[3] = LLVMPointerType(float_type, 0); /* viewport */
 190    elem_types[4] = LLVMArrayType(texture_type,
 191                                  PIPE_MAX_VERTEX_SAMPLERS); /* textures */
 192
 193    context_type = LLVMStructTypeInContext(gallivm->context, elem_types,
 194                                           Elements(elem_types), 0);
 195
 196    LLVMInvalidateStructLayout(gallivm->target, context_type);
 197
 198    LP_CHECK_MEMBER_OFFSET(struct draw_jit_context, vs_constants,
 199                           target, context_type, 0);
 200    LP_CHECK_MEMBER_OFFSET(struct draw_jit_context, gs_constants,
 201                           target, context_type, 1);
 202    LP_CHECK_MEMBER_OFFSET(struct draw_jit_context, planes,
 203                           target, context_type, 2);
 204    LP_CHECK_MEMBER_OFFSET(struct draw_jit_context, textures,
 205                           target, context_type,
 206                           DRAW_JIT_CTX_TEXTURES);
 207    LP_CHECK_STRUCT_SIZE(struct draw_jit_context,
 208                         target, context_type);
 209
 210    return context_type;
 211 }
 212
 213
 214 /**
 215  * Create LLVM type for struct pipe_vertex_buffer
 216  */
 217 static LLVMTypeRef
 218 create_jit_vertex_buffer_type(struct gallivm_state *gallivm)
 219 {
 220    LLVMTargetDataRef target = gallivm->target;
 221    LLVMTypeRef elem_types[3];
 222    LLVMTypeRef vb_type;
 223
 224    elem_types[0] =
 225    elem_types[1] = LLVMInt32TypeInContext(gallivm->context);
 226    elem_types[2] = LLVMPointerType(LLVMInt8TypeInContext(gallivm->context), 0); /* vs_constants */
 227
 228    vb_type = LLVMStructTypeInContext(gallivm->context, elem_types,
 229                                      Elements(elem_types), 0);
 230
 231    LLVMInvalidateStructLayout(gallivm->target, vb_type);
 232
 233    LP_CHECK_MEMBER_OFFSET(struct pipe_vertex_buffer, stride,
 234                           target, vb_type, 0);
 235    LP_CHECK_MEMBER_OFFSET(struct pipe_vertex_buffer, buffer_offset,
 236                           target, vb_type, 1);
 237
 238    LP_CHECK_STRUCT_SIZE(struct pipe_vertex_buffer, target, vb_type);
 239
 240    return vb_type;
 241 }
 242
 243
 244 /**
 245  * Create LLVM type for struct vertex_header;
 246  */
 247 static LLVMTypeRef
 248 create_jit_vertex_header(struct gallivm_state *gallivm, int data_elems)
 249 {
 250    LLVMTargetDataRef target = gallivm->target;
 251    LLVMTypeRef elem_types[3];
 252    LLVMTypeRef vertex_header;
 253    char struct_name[24];
 254
 255    util_snprintf(struct_name, 23, "vertex_header%d", data_elems);
 256
 257    elem_types[0]  = LLVMIntTypeInContext(gallivm->context, 32);
 258    elem_types[1]  = LLVMArrayType(LLVMFloatTypeInContext(gallivm->context), 4);
 259    elem_types[2]  = LLVMArrayType(elem_types[1], data_elems);
 260
 261    vertex_header = LLVMStructTypeInContext(gallivm->context, elem_types,
 262                                            Elements(elem_types), 0);
 263
 264    LLVMInvalidateStructLayout(gallivm->target, vertex_header);
 265
 266    /* these are bit-fields and we can't take address of them
 267       LP_CHECK_MEMBER_OFFSET(struct vertex_header, clipmask,
 268       target, vertex_header,
 269       DRAW_JIT_VERTEX_CLIPMASK);
 270       LP_CHECK_MEMBER_OFFSET(struct vertex_header, edgeflag,
 271       target, vertex_header,
 272       DRAW_JIT_VERTEX_EDGEFLAG);
 273       LP_CHECK_MEMBER_OFFSET(struct vertex_header, pad,
 274       target, vertex_header,
 275       DRAW_JIT_VERTEX_PAD);
 276       LP_CHECK_MEMBER_OFFSET(struct vertex_header, vertex_id,
 277       target, vertex_header,
 278       DRAW_JIT_VERTEX_VERTEX_ID);
 279    */
 280    LP_CHECK_MEMBER_OFFSET(struct vertex_header, clip,
 281                           target, vertex_header,
 282                           DRAW_JIT_VERTEX_CLIP);
 283    LP_CHECK_MEMBER_OFFSET(struct vertex_header, data,
 284                           target, vertex_header,
 285                           DRAW_JIT_VERTEX_DATA);
 286
 287    LLVMAddTypeName(gallivm->module, struct_name, vertex_header);
 288
 289    return vertex_header;
 290 }
 291
 292
 293 /**
 294  * Create LLVM types for various structures.
 295  */
 296 static void
 297 create_jit_types(struct draw_llvm *llvm)
 298 {
 299    struct gallivm_state *gallivm = llvm->gallivm;
 300    LLVMTypeRef texture_type, context_type, buffer_type, vb_type;
 301
 302    texture_type = create_jit_texture_type(gallivm);
 303    LLVMAddTypeName(gallivm->module, "texture", texture_type);
 304
 305    context_type = create_jit_context_type(gallivm, texture_type);
 306    LLVMAddTypeName(gallivm->module, "draw_jit_context", context_type);
 307    llvm->context_ptr_type = LLVMPointerType(context_type, 0);
 308
 309    buffer_type = LLVMPointerType(LLVMIntTypeInContext(gallivm->context, 8), 0);
 310    LLVMAddTypeName(gallivm->module, "buffer", buffer_type);
 311    llvm->buffer_ptr_type = LLVMPointerType(buffer_type, 0);
 312
 313    vb_type = create_jit_vertex_buffer_type(gallivm);
 314    LLVMAddTypeName(gallivm->module, "pipe_vertex_buffer", vb_type);
 315    llvm->vb_ptr_type = LLVMPointerType(vb_type, 0);
 316 }
 317
 318
 319 static LLVMTypeRef
 320 get_context_ptr_type(struct draw_llvm *llvm)
 321 {
 322    if (!llvm->context_ptr_type)
 323       create_jit_types(llvm);
 324    return llvm->context_ptr_type;
 325 }
 326
 327
 328 static LLVMTypeRef
 329 get_buffer_ptr_type(struct draw_llvm *llvm)
 330 {
 331    if (!llvm->buffer_ptr_type)
 332       create_jit_types(llvm);
 333    return llvm->buffer_ptr_type;
 334 }
 335
 336
 337 static LLVMTypeRef
 338 get_vb_ptr_type(struct draw_llvm *llvm)
 339 {
 340    if (!llvm->vb_ptr_type)
 341       create_jit_types(llvm);
 342    return llvm->vb_ptr_type;
 343 }
 344
 345 static LLVMTypeRef
 346 get_vertex_header_ptr_type(struct draw_llvm *llvm)
 347 {
 348    if (!llvm->vertex_header_ptr_type)
 349       create_jit_types(llvm);
 350    return llvm->vertex_header_ptr_type;
 351 }
 352
 353
 354 /**
 355  * Create per-context LLVM info.
 356  */
 357 struct draw_llvm *
 358 draw_llvm_create(struct draw_context *draw, struct gallivm_state *gallivm)
 359 {
 360    struct draw_llvm *llvm;
 361
 362    llvm = CALLOC_STRUCT( draw_llvm );
 363    if (!llvm)
 364       return NULL;
 365
 366    lp_build_init();
 367
 368    llvm->draw = draw;
 369    llvm->gallivm = gallivm;
 370
 371    if (gallivm_debug & GALLIVM_DEBUG_IR) {
 372       LLVMDumpModule(llvm->gallivm->module);
 373    }
 374
 375    llvm->nr_variants = 0;
 376    make_empty_list(&llvm->vs_variants_list);
 377
 378    gallivm_register_garbage_collector_callback(
 379                               draw_llvm_garbage_collect_callback, llvm);
 380
 381    return llvm;
 382 }
 383
 384
 385 /**
 386  * Free per-context LLVM info.
 387  */
 388 void
 389 draw_llvm_destroy(struct draw_llvm *llvm)
 390 {
 391    gallivm_remove_garbage_collector_callback(
 392                               draw_llvm_garbage_collect_callback, llvm);
 393
 394    /* XXX free other draw_llvm data? */
 395    FREE(llvm);
 396 }
 397
 398
 399 /**
 400  * Create LLVM-generated code for a vertex shader.
 401  */
 402 struct draw_llvm_variant *
 403 draw_llvm_create_variant(struct draw_llvm *llvm,
 404                          unsigned num_inputs,
 405                          const struct draw_llvm_variant_key *key)
 406 {
 407    struct draw_llvm_variant *variant;
 408    struct llvm_vertex_shader *shader =
 409       llvm_vertex_shader(llvm->draw->vs.vertex_shader);
 410    LLVMTypeRef vertex_header;
 411
 412    variant = MALLOC(sizeof *variant +
 413                     shader->variant_key_size -
 414                     sizeof variant->key);
 415    if (variant == NULL)
 416       return NULL;
 417
 418    variant->llvm = llvm;
 419
 420    memcpy(&variant->key, key, shader->variant_key_size);
 421
 422    vertex_header = create_jit_vertex_header(llvm->gallivm, num_inputs);
 423
 424    llvm->vertex_header_ptr_type = LLVMPointerType(vertex_header, 0);
 425
 426    draw_llvm_generate(llvm, variant);
 427    draw_llvm_generate_elts(llvm, variant);
 428
 429    variant->shader = shader;
 430    variant->list_item_global.base = variant;
 431    variant->list_item_local.base = variant;
 432    /*variant->no = */shader->variants_created++;
 433    variant->list_item_global.base = variant;
 434
 435    return variant;
 436 }
 437
 438
 439 static void
 440 generate_vs(struct draw_llvm *llvm,
 441             LLVMBuilderRef builder,
 442             LLVMValueRef (*outputs)[NUM_CHANNELS],
 443             const LLVMValueRef (*inputs)[NUM_CHANNELS],
 444             LLVMValueRef system_values_array,
 445             LLVMValueRef context_ptr,
 446             struct lp_build_sampler_soa *draw_sampler,
 447             boolean clamp_vertex_color)
 448 {
 449    const struct tgsi_token *tokens = llvm->draw->vs.vertex_shader->state.tokens;
 450    struct lp_type vs_type;
 451    LLVMValueRef consts_ptr = draw_jit_context_vs_constants(llvm->gallivm, context_ptr);
 452    struct lp_build_sampler_soa *sampler = 0;
 453
 454    memset(&vs_type, 0, sizeof vs_type);
 455    vs_type.floating = TRUE; /* floating point values */
 456    vs_type.sign = TRUE;     /* values are signed */
 457    vs_type.norm = FALSE;    /* values are not limited to [0,1] or [-1,1] */
 458    vs_type.width = 32;      /* 32-bit float */
 459    vs_type.length = 4;      /* 4 elements per vector */
 460 #if 0
 461    num_vs = 4;              /* number of vertices per block */
 462 #endif
 463
 464    if (gallivm_debug & GALLIVM_DEBUG_IR) {
 465       tgsi_dump(tokens, 0);
 466    }
 467
 468    if (llvm->draw->num_sampler_views && llvm->draw->num_samplers)
 469       sampler = draw_sampler;
 470
 471    lp_build_tgsi_soa(llvm->gallivm,
 472                      tokens,
 473                      vs_type,
 474                      NULL /*struct lp_build_mask_context *mask*/,
 475                      consts_ptr,
 476                      system_values_array,
 477                      NULL /*pos*/,
 478                      inputs,
 479                      outputs,
 480                      sampler,
 481                      &llvm->draw->vs.vertex_shader->info);
 482
 483    if (clamp_vertex_color) {
 484       LLVMValueRef out;
 485       unsigned chan, attrib;
 486       struct lp_build_context bld;
 487       struct tgsi_shader_info* info = &llvm->draw->vs.vertex_shader->info;
 488       lp_build_context_init(&bld, llvm->gallivm, vs_type);
 489
 490       for (attrib = 0; attrib < info->num_outputs; ++attrib) {
 491          for (chan = 0; chan < NUM_CHANNELS; ++chan) {
 492             if (outputs[attrib][chan]) {
 493                switch (info->output_semantic_name[attrib]) {
 494                case TGSI_SEMANTIC_COLOR:
 495                case TGSI_SEMANTIC_BCOLOR:
 496                   out = LLVMBuildLoad(builder, outputs[attrib][chan], "");
 497                   out = lp_build_clamp(&bld, out, bld.zero, bld.one);
 498                   LLVMBuildStore(builder, out, outputs[attrib][chan]);
 499                   break;
 500                }
 501             }
 502          }
 503       }
 504    }
 505 }
 506
 507
 508 #if DEBUG_STORE
 509 static void print_vectorf(LLVMBuilderRef builder,
 510                          LLVMValueRef vec)
 511 {
 512    LLVMValueRef val[4];
 513    val[0] = LLVMBuildExtractElement(builder, vec,
 514                                     lp_build_const_int32(gallivm, 0), "");
 515    val[1] = LLVMBuildExtractElement(builder, vec,
 516                                     lp_build_const_int32(gallivm, 1), "");
 517    val[2] = LLVMBuildExtractElement(builder, vec,
 518                                     lp_build_const_int32(gallivm, 2), "");
 519    val[3] = LLVMBuildExtractElement(builder, vec,
 520                                     lp_build_const_int32(gallivm, 3), "");
 521    lp_build_printf(builder, "vector = [%f, %f, %f, %f]\n",
 522                    val[0], val[1], val[2], val[3]);
 523 }
 524 #endif
 525
 526
 527 static void
 528 generate_fetch(struct gallivm_state *gallivm,
 529                LLVMValueRef vbuffers_ptr,
 530                LLVMValueRef *res,
 531                struct pipe_vertex_element *velem,
 532                LLVMValueRef vbuf,
 533                LLVMValueRef index,
 534                LLVMValueRef instance_id)
 535 {
 536    LLVMBuilderRef builder = gallivm->builder;
 537    LLVMValueRef indices =
 538       LLVMConstInt(LLVMInt64TypeInContext(gallivm->context),
 539                    velem->vertex_buffer_index, 0);
 540    LLVMValueRef vbuffer_ptr = LLVMBuildGEP(builder, vbuffers_ptr,
 541                                            &indices, 1, "");
 542    LLVMValueRef vb_stride = draw_jit_vbuffer_stride(gallivm, vbuf);
 543    LLVMValueRef vb_buffer_offset = draw_jit_vbuffer_offset(gallivm, vbuf);
 544    LLVMValueRef stride;
 545
 546    if (velem->instance_divisor) {
 547       /* array index = instance_id / instance_divisor */
 548       index = LLVMBuildUDiv(builder, instance_id,
 549                             lp_build_const_int32(gallivm, velem->instance_divisor),
 550                             "instance_divisor");
 551    }
 552
 553    stride = LLVMBuildMul(builder, vb_stride, index, "");
 554
 555    vbuffer_ptr = LLVMBuildLoad(builder, vbuffer_ptr, "vbuffer");
 556
 557    stride = LLVMBuildAdd(builder, stride,
 558                          vb_buffer_offset,
 559                          "");
 560    stride = LLVMBuildAdd(builder, stride,
 561                          lp_build_const_int32(gallivm, velem->src_offset),
 562                          "");
 563
 564    /*lp_build_printf(builder, "vbuf index = %d, stride is %d\n", indices, stride);*/
 565    vbuffer_ptr = LLVMBuildGEP(builder, vbuffer_ptr, &stride, 1, "");
 566
 567    *res = draw_llvm_translate_from(gallivm, vbuffer_ptr, velem->src_format);
 568 }
 569
 570
 571 static LLVMValueRef
 572 aos_to_soa(struct gallivm_state *gallivm,
 573            LLVMValueRef val0,
 574            LLVMValueRef val1,
 575            LLVMValueRef val2,
 576            LLVMValueRef val3,
 577            LLVMValueRef channel)
 578 {
 579    LLVMBuilderRef builder = gallivm->builder;
 580    LLVMValueRef ex, res;
 581
 582    ex = LLVMBuildExtractElement(builder, val0,
 583                                 channel, "");
 584    res = LLVMBuildInsertElement(builder,
 585                                 LLVMConstNull(LLVMTypeOf(val0)),
 586                                 ex,
 587                                 lp_build_const_int32(gallivm, 0),
 588                                 "");
 589
 590    ex = LLVMBuildExtractElement(builder, val1,
 591                                 channel, "");
 592    res = LLVMBuildInsertElement(builder,
 593                                 res, ex,
 594                                 lp_build_const_int32(gallivm, 1),
 595                                 "");
 596
 597    ex = LLVMBuildExtractElement(builder, val2,
 598                                 channel, "");
 599    res = LLVMBuildInsertElement(builder,
 600                                 res, ex,
 601                                 lp_build_const_int32(gallivm, 2),
 602                                 "");
 603
 604    ex = LLVMBuildExtractElement(builder, val3,
 605                                 channel, "");
 606    res = LLVMBuildInsertElement(builder,
 607                                 res, ex,
 608                                 lp_build_const_int32(gallivm, 3),
 609                                 "");
 610
 611    return res;
 612 }
 613
 614
 615 static void
 616 soa_to_aos(struct gallivm_state *gallivm,
 617            LLVMValueRef soa[NUM_CHANNELS],
 618            LLVMValueRef aos[NUM_CHANNELS])
 619 {
 620    LLVMBuilderRef builder = gallivm->builder;
 621    LLVMValueRef comp;
 622    int i = 0;
 623
 624    debug_assert(NUM_CHANNELS == 4);
 625
 626    aos[0] = LLVMConstNull(LLVMTypeOf(soa[0]));
 627    aos[1] = aos[2] = aos[3] = aos[0];
 628
 629    for (i = 0; i < NUM_CHANNELS; ++i) {
 630       LLVMValueRef channel = lp_build_const_int32(gallivm, i);
 631
 632       comp = LLVMBuildExtractElement(builder, soa[i],
 633                                      lp_build_const_int32(gallivm, 0), "");
 634       aos[0] = LLVMBuildInsertElement(builder, aos[0], comp, channel, "");
 635
 636       comp = LLVMBuildExtractElement(builder, soa[i],
 637                                      lp_build_const_int32(gallivm, 1), "");
 638       aos[1] = LLVMBuildInsertElement(builder, aos[1], comp, channel, "");
 639
 640       comp = LLVMBuildExtractElement(builder, soa[i],
 641                                      lp_build_const_int32(gallivm, 2), "");
 642       aos[2] = LLVMBuildInsertElement(builder, aos[2], comp, channel, "");
 643
 644       comp = LLVMBuildExtractElement(builder, soa[i],
 645                                      lp_build_const_int32(gallivm, 3), "");
 646       aos[3] = LLVMBuildInsertElement(builder, aos[3], comp, channel, "");
 647
 648    }
 649 }
 650
 651
 652 static void
 653 convert_to_soa(struct gallivm_state *gallivm,
 654                LLVMValueRef (*aos)[NUM_CHANNELS],
 655                LLVMValueRef (*soa)[NUM_CHANNELS],
 656                int num_attribs)
 657 {
 658    int i;
 659
 660    debug_assert(NUM_CHANNELS == 4);
 661
 662    for (i = 0; i < num_attribs; ++i) {
 663       LLVMValueRef val0 = aos[i][0];
 664       LLVMValueRef val1 = aos[i][1];
 665       LLVMValueRef val2 = aos[i][2];
 666       LLVMValueRef val3 = aos[i][3];
 667
 668       soa[i][0] = aos_to_soa(gallivm, val0, val1, val2, val3,
 669                              lp_build_const_int32(gallivm, 0));
 670       soa[i][1] = aos_to_soa(gallivm, val0, val1, val2, val3,
 671                              lp_build_const_int32(gallivm, 1));
 672       soa[i][2] = aos_to_soa(gallivm, val0, val1, val2, val3,
 673                              lp_build_const_int32(gallivm, 2));
 674       soa[i][3] = aos_to_soa(gallivm, val0, val1, val2, val3,
 675                              lp_build_const_int32(gallivm, 3));
 676    }
 677 }
 678
 679
 680 static void
 681 store_aos(struct gallivm_state *gallivm,
 682           LLVMValueRef io_ptr,
 683           LLVMValueRef index,
 684           LLVMValueRef value,
 685           LLVMValueRef clipmask)
 686 {
 687    LLVMBuilderRef builder = gallivm->builder;
 688    LLVMValueRef id_ptr = draw_jit_header_id(gallivm, io_ptr);
 689    LLVMValueRef data_ptr = draw_jit_header_data(gallivm, io_ptr);
 690    LLVMValueRef indices[3];
 691    LLVMValueRef val, shift;
 692
 693    indices[0] = lp_build_const_int32(gallivm, 0);
 694    indices[1] = index;
 695    indices[2] = lp_build_const_int32(gallivm, 0);
 696
 697    /* initialize vertex id:16 = 0xffff, pad:3 = 0, edgeflag:1 = 1 */
 698    val = lp_build_const_int32(gallivm, 0xffff1);
 699    shift = lp_build_const_int32(gallivm, 12);
 700    val = LLVMBuildShl(builder, val, shift, "");
 701    /* add clipmask:12 */
 702    val = LLVMBuildOr(builder, val, clipmask, "");
 703
 704    /* store vertex header */
 705    LLVMBuildStore(builder, val, id_ptr);
 706
 707
 708 #if DEBUG_STORE
 709    lp_build_printf(builder, "    ---- %p storing attribute %d (io = %p)\n", data_ptr, index, io_ptr);
 710 #endif
 711 #if 0
 712    /*lp_build_printf(builder, " ---- %p storing at %d (%p)  ", io_ptr, index, data_ptr);
 713      print_vectorf(builder, value);*/
 714    data_ptr = LLVMBuildBitCast(builder, data_ptr,
 715                                LLVMPointerType(LLVMArrayType(LLVMVectorType(LLVMFloatTypeInContext(gallivm->context), 4), 0), 0),
 716                                "datavec");
 717    data_ptr = LLVMBuildGEP(builder, data_ptr, indices, 2, "");
 718
 719    LLVMBuildStore(builder, value, data_ptr);
 720 #else
 721    {
 722       LLVMValueRef x, y, z, w;
 723       LLVMValueRef idx0, idx1, idx2, idx3;
 724       LLVMValueRef gep0, gep1, gep2, gep3;
 725       data_ptr = LLVMBuildGEP(builder, data_ptr, indices, 3, "");
 726
 727       idx0 = lp_build_const_int32(gallivm, 0);
 728       idx1 = lp_build_const_int32(gallivm, 1);
 729       idx2 = lp_build_const_int32(gallivm, 2);
 730       idx3 = lp_build_const_int32(gallivm, 3);
 731
 732       x = LLVMBuildExtractElement(builder, value,
 733                                   idx0, "");
 734       y = LLVMBuildExtractElement(builder, value,
 735                                   idx1, "");
 736       z = LLVMBuildExtractElement(builder, value,
 737                                   idx2, "");
 738       w = LLVMBuildExtractElement(builder, value,
 739                                   idx3, "");
 740
 741       gep0 = LLVMBuildGEP(builder, data_ptr, &idx0, 1, "");
 742       gep1 = LLVMBuildGEP(builder, data_ptr, &idx1, 1, "");
 743       gep2 = LLVMBuildGEP(builder, data_ptr, &idx2, 1, "");
 744       gep3 = LLVMBuildGEP(builder, data_ptr, &idx3, 1, "");
 745
 746       /*lp_build_printf(builder, "##### x = %f (%p), y = %f (%p), z = %f (%p), w = %f (%p)\n",
 747         x, gep0, y, gep1, z, gep2, w, gep3);*/
 748       LLVMBuildStore(builder, x, gep0);
 749       LLVMBuildStore(builder, y, gep1);
 750       LLVMBuildStore(builder, z, gep2);
 751       LLVMBuildStore(builder, w, gep3);
 752    }
 753 #endif
 754 }
 755
 756
 757 static void
 758 store_aos_array(struct gallivm_state *gallivm,
 759                 LLVMValueRef io_ptr,
 760                 LLVMValueRef aos[NUM_CHANNELS],
 761                 int attrib,
 762                 int num_outputs,
 763                 LLVMValueRef clipmask)
 764 {
 765    LLVMBuilderRef builder = gallivm->builder;
 766    LLVMValueRef attr_index = lp_build_const_int32(gallivm, attrib);
 767    LLVMValueRef ind0 = lp_build_const_int32(gallivm, 0);
 768    LLVMValueRef ind1 = lp_build_const_int32(gallivm, 1);
 769    LLVMValueRef ind2 = lp_build_const_int32(gallivm, 2);
 770    LLVMValueRef ind3 = lp_build_const_int32(gallivm, 3);
 771    LLVMValueRef io0_ptr, io1_ptr, io2_ptr, io3_ptr;
 772    LLVMValueRef clipmask0, clipmask1, clipmask2, clipmask3;
 773
 774    debug_assert(NUM_CHANNELS == 4);
 775
 776    io0_ptr = LLVMBuildGEP(builder, io_ptr,
 777                           &ind0, 1, "");
 778    io1_ptr = LLVMBuildGEP(builder, io_ptr,
 779                           &ind1, 1, "");
 780    io2_ptr = LLVMBuildGEP(builder, io_ptr,
 781                           &ind2, 1, "");
 782    io3_ptr = LLVMBuildGEP(builder, io_ptr,
 783                           &ind3, 1, "");
 784
 785    clipmask0 = LLVMBuildExtractElement(builder, clipmask,
 786                                        ind0, "");
 787    clipmask1 = LLVMBuildExtractElement(builder, clipmask,
 788                                        ind1, "");
 789    clipmask2 = LLVMBuildExtractElement(builder, clipmask,
 790                                        ind2, "");
 791    clipmask3 = LLVMBuildExtractElement(builder, clipmask,
 792                                        ind3, "");
 793
 794 #if DEBUG_STORE
 795    lp_build_printf(builder, "io = %p, indexes[%d, %d, %d, %d]\n, clipmask0 = %x, clipmask1 = %x, clipmask2 = %x, clipmask3 = %x\n",
 796                    io_ptr, ind0, ind1, ind2, ind3, clipmask0, clipmask1, clipmask2, clipmask3);
 797 #endif
 798    /* store for each of the 4 vertices */
 799    store_aos(gallivm, io0_ptr, attr_index, aos[0], clipmask0);
 800    store_aos(gallivm, io1_ptr, attr_index, aos[1], clipmask1);
 801    store_aos(gallivm, io2_ptr, attr_index, aos[2], clipmask2);
 802    store_aos(gallivm, io3_ptr, attr_index, aos[3], clipmask3);
 803 }
 804
 805
 806 static void
 807 convert_to_aos(struct gallivm_state *gallivm,
 808                LLVMValueRef io,
 809                LLVMValueRef (*outputs)[NUM_CHANNELS],
 810                LLVMValueRef clipmask,
 811                int num_outputs,
 812                int max_vertices)
 813 {
 814    LLVMBuilderRef builder = gallivm->builder;
 815    unsigned chan, attrib;
 816
 817 #if DEBUG_STORE
 818    lp_build_printf(builder, "   # storing begin\n");
 819 #endif
 820    for (attrib = 0; attrib < num_outputs; ++attrib) {
 821       LLVMValueRef soa[4];
 822       LLVMValueRef aos[4];
 823       for (chan = 0; chan < NUM_CHANNELS; ++chan) {
 824          if (outputs[attrib][chan]) {
 825             LLVMValueRef out = LLVMBuildLoad(builder, outputs[attrib][chan], "");
 826             lp_build_name(out, "output%u.%c", attrib, "xyzw"[chan]);
 827             /*lp_build_printf(builder, "output %d : %d ",
 828                             LLVMConstInt(LLVMInt32Type(), attrib, 0),
 829                             LLVMConstInt(LLVMInt32Type(), chan, 0));
 830               print_vectorf(builder, out);*/
 831             soa[chan] = out;
 832          }
 833          else {
 834             soa[chan] = 0;
 835          }
 836       }
 837       soa_to_aos(gallivm, soa, aos);
 838       store_aos_array(gallivm,
 839                       io,
 840                       aos,
 841                       attrib,
 842                       num_outputs,
 843                       clipmask);
 844    }
 845 #if DEBUG_STORE
 846    lp_build_printf(builder, "   # storing end\n");
 847 #endif
 848 }
 849
 850
 851 /**
 852  * Stores original vertex positions in clip coordinates
 853  * There is probably a more efficient way to do this, 4 floats at once
 854  * rather than extracting each element one by one.
 855  */
 856 static void
 857 store_clip(struct gallivm_state *gallivm,
 858            LLVMValueRef io_ptr,
 859            LLVMValueRef (*outputs)[NUM_CHANNELS])
 860 {
 861    LLVMBuilderRef builder = gallivm->builder;
 862    LLVMValueRef out[4];
 863    LLVMValueRef indices[2];
 864    LLVMValueRef io0_ptr, io1_ptr, io2_ptr, io3_ptr;
 865    LLVMValueRef clip_ptr0, clip_ptr1, clip_ptr2, clip_ptr3;
 866    LLVMValueRef clip0_ptr, clip1_ptr, clip2_ptr, clip3_ptr;
 867    LLVMValueRef out0elem, out1elem, out2elem, out3elem;
 868    int i;
 869
 870    LLVMValueRef ind0 = lp_build_const_int32(gallivm, 0);
 871    LLVMValueRef ind1 = lp_build_const_int32(gallivm, 1);
 872    LLVMValueRef ind2 = lp_build_const_int32(gallivm, 2);
 873    LLVMValueRef ind3 = lp_build_const_int32(gallivm, 3);
 874
 875    indices[0] =
 876    indices[1] = lp_build_const_int32(gallivm, 0);
 877
 878    out[0] = LLVMBuildLoad(builder, outputs[0][0], ""); /*x0 x1 x2 x3*/
 879    out[1] = LLVMBuildLoad(builder, outputs[0][1], ""); /*y0 y1 y2 y3*/
 880    out[2] = LLVMBuildLoad(builder, outputs[0][2], ""); /*z0 z1 z2 z3*/
 881    out[3] = LLVMBuildLoad(builder, outputs[0][3], ""); /*w0 w1 w2 w3*/
 882
 883    io0_ptr = LLVMBuildGEP(builder, io_ptr, &ind0, 1, "");
 884    io1_ptr = LLVMBuildGEP(builder, io_ptr, &ind1, 1, "");
 885    io2_ptr = LLVMBuildGEP(builder, io_ptr, &ind2, 1, "");
 886    io3_ptr = LLVMBuildGEP(builder, io_ptr, &ind3, 1, "");
 887
 888    clip_ptr0 = draw_jit_header_clip(gallivm, io0_ptr);
 889    clip_ptr1 = draw_jit_header_clip(gallivm, io1_ptr);
 890    clip_ptr2 = draw_jit_header_clip(gallivm, io2_ptr);
 891    clip_ptr3 = draw_jit_header_clip(gallivm, io3_ptr);
 892
 893    for (i = 0; i<4; i++) {
 894       clip0_ptr = LLVMBuildGEP(builder, clip_ptr0, indices, 2, ""); /* x0 */
 895       clip1_ptr = LLVMBuildGEP(builder, clip_ptr1, indices, 2, ""); /* x1 */
 896       clip2_ptr = LLVMBuildGEP(builder, clip_ptr2, indices, 2, ""); /* x2 */
 897       clip3_ptr = LLVMBuildGEP(builder, clip_ptr3, indices, 2, ""); /* x3 */
 898
 899       out0elem = LLVMBuildExtractElement(builder, out[i], ind0, ""); /* x0 */
 900       out1elem = LLVMBuildExtractElement(builder, out[i], ind1, ""); /* x1 */
 901       out2elem = LLVMBuildExtractElement(builder, out[i], ind2, ""); /* x2 */
 902       out3elem = LLVMBuildExtractElement(builder, out[i], ind3, ""); /* x3 */
 903
 904       LLVMBuildStore(builder, out0elem, clip0_ptr);
 905       LLVMBuildStore(builder, out1elem, clip1_ptr);
 906       LLVMBuildStore(builder, out2elem, clip2_ptr);
 907       LLVMBuildStore(builder, out3elem, clip3_ptr);
 908
 909       indices[1]= LLVMBuildAdd(builder, indices[1], ind1, "");
 910    }
 911
 912 }
 913
 914
 915 /**
 916  * Equivalent of _mm_set1_ps(a)
 917  */
 918 static LLVMValueRef
 919 vec4f_from_scalar(struct gallivm_state *gallivm,
 920                   LLVMValueRef a,
 921                   const char *name)
 922 {
 923    LLVMTypeRef float_type = LLVMFloatTypeInContext(gallivm->context);
 924    LLVMValueRef res = LLVMGetUndef(LLVMVectorType(float_type, 4));
 925    int i;
 926
 927    for (i = 0; i < 4; ++i) {
 928       LLVMValueRef index = lp_build_const_int32(gallivm, i);
 929       res = LLVMBuildInsertElement(gallivm->builder, res, a,
 930                                    index, i == 3 ? name : "");
 931    }
 932
 933    return res;
 934 }
 935
 936
 937 /**
 938  * Transforms the outputs for viewport mapping
 939  */
 940 static void
 941 generate_viewport(struct draw_llvm *llvm,
 942                   LLVMBuilderRef builder,
 943                   LLVMValueRef (*outputs)[NUM_CHANNELS],
 944                   LLVMValueRef context_ptr)
 945 {
 946    int i;
 947    struct gallivm_state *gallivm = llvm->gallivm;
 948    struct lp_type f32_type = lp_type_float_vec(32);
 949    LLVMValueRef out3 = LLVMBuildLoad(builder, outputs[0][3], ""); /*w0 w1 w2 w3*/
 950    LLVMValueRef const1 = lp_build_const_vec(gallivm, f32_type, 1.0);       /*1.0 1.0 1.0 1.0*/
 951    LLVMValueRef vp_ptr = draw_jit_context_viewport(gallivm, context_ptr);
 952
 953    /* for 1/w convention*/
 954    out3 = LLVMBuildFDiv(builder, const1, out3, "");
 955    LLVMBuildStore(builder, out3, outputs[0][3]);
 956
 957    /* Viewport Mapping */
 958    for (i=0; i<3; i++) {
 959       LLVMValueRef out = LLVMBuildLoad(builder, outputs[0][i], ""); /*x0 x1 x2 x3*/
 960       LLVMValueRef scale;
 961       LLVMValueRef trans;
 962       LLVMValueRef scale_i;
 963       LLVMValueRef trans_i;
 964       LLVMValueRef index;
 965
 966       index = lp_build_const_int32(gallivm, i);
 967       scale_i = LLVMBuildGEP(builder, vp_ptr, &index, 1, "");
 968
 969       index = lp_build_const_int32(gallivm, i+4);
 970       trans_i = LLVMBuildGEP(builder, vp_ptr, &index, 1, "");
 971
 972       scale = vec4f_from_scalar(gallivm, LLVMBuildLoad(builder, scale_i, ""), "scale");
 973       trans = vec4f_from_scalar(gallivm, LLVMBuildLoad(builder, trans_i, ""), "trans");
 974
 975       /* divide by w */
 976       out = LLVMBuildFMul(builder, out, out3, "");
 977       /* mult by scale */
 978       out = LLVMBuildFMul(builder, out, scale, "");
 979       /* add translation */
 980       out = LLVMBuildFAdd(builder, out, trans, "");
 981
 982       /* store transformed outputs */
 983       LLVMBuildStore(builder, out, outputs[0][i]);
 984    }
 985
 986 }
 987
 988
 989 /**
 990  * Returns clipmask as 4xi32 bitmask for the 4 vertices
 991  */
 992 static LLVMValueRef
 993 generate_clipmask(struct gallivm_state *gallivm,
 994                   LLVMValueRef (*outputs)[NUM_CHANNELS],
 995                   boolean clip_xy,
 996                   boolean clip_z,
 997                   boolean clip_user,
 998                   boolean clip_halfz,
 999                   unsigned nr,
1000                   LLVMValueRef context_ptr)
1001 {
1002    LLVMBuilderRef builder = gallivm->builder;
1003    LLVMValueRef mask; /* stores the <4xi32> clipmasks */
1004    LLVMValueRef test, temp;
1005    LLVMValueRef zero, shift;
1006    LLVMValueRef pos_x, pos_y, pos_z, pos_w;
1007    LLVMValueRef plane1, planes, plane_ptr, sum;
1008    unsigned i;
1009    struct lp_type f32_type = lp_type_float_vec(32);
1010
1011    mask = lp_build_const_int_vec(gallivm, lp_type_int_vec(32), 0);
1012    temp = lp_build_const_int_vec(gallivm, lp_type_int_vec(32), 0);
1013    zero = lp_build_const_vec(gallivm, f32_type, 0);                    /* 0.0f 0.0f 0.0f 0.0f */
1014    shift = lp_build_const_int_vec(gallivm, lp_type_int_vec(32), 1);    /* 1 1 1 1 */
1015
1016    /* Assuming position stored at output[0] */
1017    pos_x = LLVMBuildLoad(builder, outputs[0][0], ""); /*x0 x1 x2 x3*/
1018    pos_y = LLVMBuildLoad(builder, outputs[0][1], ""); /*y0 y1 y2 y3*/
1019    pos_z = LLVMBuildLoad(builder, outputs[0][2], ""); /*z0 z1 z2 z3*/
1020    pos_w = LLVMBuildLoad(builder, outputs[0][3], ""); /*w0 w1 w2 w3*/
1021
1022    /* Cliptest, for hardwired planes */
1023    if (clip_xy) {
1024       /* plane 1 */
1025       test = lp_build_compare(gallivm, f32_type, PIPE_FUNC_GREATER, pos_x , pos_w);
1026       temp = shift;
1027       test = LLVMBuildAnd(builder, test, temp, "");
1028       mask = test;
1029
1030       /* plane 2 */
1031       test = LLVMBuildFAdd(builder, pos_x, pos_w, "");
1032       test = lp_build_compare(gallivm, f32_type, PIPE_FUNC_GREATER, zero, test);
1033       temp = LLVMBuildShl(builder, temp, shift, "");
1034       test = LLVMBuildAnd(builder, test, temp, "");
1035       mask = LLVMBuildOr(builder, mask, test, "");
1036
1037       /* plane 3 */
1038       test = lp_build_compare(gallivm, f32_type, PIPE_FUNC_GREATER, pos_y, pos_w);
1039       temp = LLVMBuildShl(builder, temp, shift, "");
1040       test = LLVMBuildAnd(builder, test, temp, "");
1041       mask = LLVMBuildOr(builder, mask, test, "");
1042
1043       /* plane 4 */
1044       test = LLVMBuildFAdd(builder, pos_y, pos_w, "");
1045       test = lp_build_compare(gallivm, f32_type, PIPE_FUNC_GREATER, zero, test);
1046       temp = LLVMBuildShl(builder, temp, shift, "");
1047       test = LLVMBuildAnd(builder, test, temp, "");
1048       mask = LLVMBuildOr(builder, mask, test, "");
1049    }
1050
1051    if (clip_z) {
1052       temp = lp_build_const_int_vec(gallivm, lp_type_int_vec(32), 16);
1053       if (clip_halfz) {
1054          /* plane 5 */
1055          test = lp_build_compare(gallivm, f32_type, PIPE_FUNC_GREATER, zero, pos_z);
1056          test = LLVMBuildAnd(builder, test, temp, "");
1057          mask = LLVMBuildOr(builder, mask, test, "");
1058       }
1059       else {
1060          /* plane 5 */
1061          test = LLVMBuildFAdd(builder, pos_z, pos_w, "");
1062          test = lp_build_compare(gallivm, f32_type, PIPE_FUNC_GREATER, zero, test);
1063          test = LLVMBuildAnd(builder, test, temp, "");
1064          mask = LLVMBuildOr(builder, mask, test, "");
1065       }
1066       /* plane 6 */
1067       test = lp_build_compare(gallivm, f32_type, PIPE_FUNC_GREATER, pos_z, pos_w);
1068       temp = LLVMBuildShl(builder, temp, shift, "");
1069       test = LLVMBuildAnd(builder, test, temp, "");
1070       mask = LLVMBuildOr(builder, mask, test, "");
1071    }
1072
1073    if (clip_user) {
1074       LLVMValueRef planes_ptr = draw_jit_context_planes(gallivm, context_ptr);
1075       LLVMValueRef indices[3];
1076       temp = lp_build_const_int_vec(gallivm, lp_type_int_vec(32), 32);
1077
1078       /* userclip planes */
1079       for (i = 6; i < nr; i++) {
1080          indices[0] = lp_build_const_int32(gallivm, 0);
1081          indices[1] = lp_build_const_int32(gallivm, i);
1082
1083          indices[2] = lp_build_const_int32(gallivm, 0);
1084          plane_ptr = LLVMBuildGEP(builder, planes_ptr, indices, 3, "");
1085          plane1 = LLVMBuildLoad(builder, plane_ptr, "plane_x");
1086          planes = vec4f_from_scalar(gallivm, plane1, "plane4_x");
1087          sum = LLVMBuildFMul(builder, planes, pos_x, "");
1088
1089          indices[2] = lp_build_const_int32(gallivm, 1);
1090          plane_ptr = LLVMBuildGEP(builder, planes_ptr, indices, 3, "");
1091          plane1 = LLVMBuildLoad(builder, plane_ptr, "plane_y");
1092          planes = vec4f_from_scalar(gallivm, plane1, "plane4_y");
1093          test = LLVMBuildFMul(builder, planes, pos_y, "");
1094          sum = LLVMBuildFAdd(builder, sum, test, "");
1095
1096          indices[2] = lp_build_const_int32(gallivm, 2);
1097          plane_ptr = LLVMBuildGEP(builder, planes_ptr, indices, 3, "");
1098          plane1 = LLVMBuildLoad(builder, plane_ptr, "plane_z");
1099          planes = vec4f_from_scalar(gallivm, plane1, "plane4_z");
1100          test = LLVMBuildFMul(builder, planes, pos_z, "");
1101          sum = LLVMBuildFAdd(builder, sum, test, "");
1102
1103          indices[2] = lp_build_const_int32(gallivm, 3);
1104          plane_ptr = LLVMBuildGEP(builder, planes_ptr, indices, 3, "");
1105          plane1 = LLVMBuildLoad(builder, plane_ptr, "plane_w");
1106          planes = vec4f_from_scalar(gallivm, plane1, "plane4_w");
1107          test = LLVMBuildFMul(builder, planes, pos_w, "");
1108          sum = LLVMBuildFAdd(builder, sum, test, "");
1109
1110          test = lp_build_compare(gallivm, f32_type, PIPE_FUNC_GREATER, zero, sum);
1111          temp = LLVMBuildShl(builder, temp, shift, "");
1112          test = LLVMBuildAnd(builder, test, temp, "");
1113          mask = LLVMBuildOr(builder, mask, test, "");
1114       }
1115    }
1116    return mask;
1117 }
1118
1119
1120 /**
1121  * Returns boolean if any clipping has occurred
1122  * Used zero/non-zero i32 value to represent boolean
1123  */
1124 static void
1125 clipmask_bool(struct gallivm_state *gallivm,
1126               LLVMValueRef clipmask,
1127               LLVMValueRef ret_ptr)
1128 {
1129    LLVMBuilderRef builder = gallivm->builder;
1130    LLVMValueRef ret = LLVMBuildLoad(builder, ret_ptr, "");
1131    LLVMValueRef temp;
1132    int i;
1133
1134    for (i=0; i<4; i++) {
1135       temp = LLVMBuildExtractElement(builder, clipmask,
1136                                      lp_build_const_int32(gallivm, i) , "");
1137       ret = LLVMBuildOr(builder, ret, temp, "");
1138    }
1139
1140    LLVMBuildStore(builder, ret, ret_ptr);
1141 }
1142
1143
1144 static void
1145 draw_llvm_generate(struct draw_llvm *llvm, struct draw_llvm_variant *variant)
1146 {
1147    struct gallivm_state *gallivm = llvm->gallivm;
1148    LLVMContextRef context = gallivm->context;
1149    LLVMTypeRef int32_type = LLVMInt32TypeInContext(context);
1150    LLVMTypeRef arg_types[8];
1151    LLVMTypeRef func_type;
1152    LLVMValueRef context_ptr;
1153    LLVMBasicBlockRef block;
1154    LLVMBuilderRef builder;
1155    LLVMValueRef start, end, count, stride, step, io_itr;
1156    LLVMValueRef io_ptr, vbuffers_ptr, vb_ptr;
1157    LLVMValueRef instance_id;
1158    LLVMValueRef system_values_array;
1159    struct draw_context *draw = llvm->draw;
1160    const struct tgsi_shader_info *vs_info = &draw->vs.vertex_shader->info;
1161    unsigned i, j;
1162    struct lp_build_context bld;
1163    struct lp_build_loop_state lp_loop;
1164    const int max_vertices = 4;
1165    LLVMValueRef outputs[PIPE_MAX_SHADER_OUTPUTS][NUM_CHANNELS];
1166    void *code;
1167    struct lp_build_sampler_soa *sampler = 0;
1168    LLVMValueRef ret, ret_ptr;
1169    boolean bypass_viewport = variant->key.bypass_viewport;
1170    boolean enable_cliptest = variant->key.clip_xy ||
1171                              variant->key.clip_z  ||
1172                              variant->key.clip_user;
1173
1174    arg_types[0] = get_context_ptr_type(llvm);       /* context */
1175    arg_types[1] = get_vertex_header_ptr_type(llvm); /* vertex_header */
1176    arg_types[2] = get_buffer_ptr_type(llvm);        /* vbuffers */
1177    arg_types[3] = int32_type;                       /* start */
1178    arg_types[4] = int32_type;                       /* count */
1179    arg_types[5] = int32_type;                       /* stride */
1180    arg_types[6] = get_vb_ptr_type(llvm);            /* pipe_vertex_buffer's */
1181    arg_types[7] = int32_type;                       /* instance_id */
1182
1183    func_type = LLVMFunctionType(int32_type, arg_types, Elements(arg_types), 0);
1184
1185    variant->function = LLVMAddFunction(gallivm->module, "draw_llvm_shader",
1186                                        func_type);
1187    LLVMSetFunctionCallConv(variant->function, LLVMCCallConv);
1188    for (i = 0; i < Elements(arg_types); ++i)
1189       if (LLVMGetTypeKind(arg_types[i]) == LLVMPointerTypeKind)
1190          LLVMAddAttribute(LLVMGetParam(variant->function, i), LLVMNoAliasAttribute);
1191
1192    context_ptr  = LLVMGetParam(variant->function, 0);
1193    io_ptr       = LLVMGetParam(variant->function, 1);
1194    vbuffers_ptr = LLVMGetParam(variant->function, 2);
1195    start        = LLVMGetParam(variant->function, 3);
1196    count        = LLVMGetParam(variant->function, 4);
1197    stride       = LLVMGetParam(variant->function, 5);
1198    vb_ptr       = LLVMGetParam(variant->function, 6);
1199    instance_id  = LLVMGetParam(variant->function, 7);
1200
1201    lp_build_name(context_ptr, "context");
1202    lp_build_name(io_ptr, "io");
1203    lp_build_name(vbuffers_ptr, "vbuffers");
1204    lp_build_name(start, "start");
1205    lp_build_name(count, "count");
1206    lp_build_name(stride, "stride");
1207    lp_build_name(vb_ptr, "vb");
1208    lp_build_name(instance_id, "instance_id");
1209
1210    /*
1211     * Function body
1212     */
1213
1214    block = LLVMAppendBasicBlockInContext(gallivm->context, variant->function, "entry");
1215    builder = gallivm->builder;
1216    assert(builder);
1217    LLVMPositionBuilderAtEnd(builder, block);
1218
1219    lp_build_context_init(&bld, llvm->gallivm, lp_type_int(32));
1220
1221    system_values_array = lp_build_system_values_array(gallivm, vs_info,
1222                                                       instance_id, NULL);
1223
1224    end = lp_build_add(&bld, start, count);
1225
1226    step = lp_build_const_int32(gallivm, max_vertices);
1227
1228    /* function will return non-zero i32 value if any clipped vertices */
1229    ret_ptr = lp_build_alloca(gallivm, int32_type, "");
1230    LLVMBuildStore(builder, lp_build_const_int32(gallivm, 0), ret_ptr);
1231
1232    /* code generated texture sampling */
1233    sampler = draw_llvm_sampler_soa_create(
1234       draw_llvm_variant_key_samplers(&variant->key),
1235       context_ptr);
1236
1237 #if DEBUG_STORE
1238    lp_build_printf(builder, "start = %d, end = %d, step = %d\n",
1239                    start, end, step);
1240 #endif
1241    lp_build_loop_begin(&lp_loop, llvm->gallivm, start);
1242    {
1243       LLVMValueRef inputs[PIPE_MAX_SHADER_INPUTS][NUM_CHANNELS];
1244       LLVMValueRef aos_attribs[PIPE_MAX_SHADER_INPUTS][NUM_CHANNELS] = { { 0 } };
1245       LLVMValueRef io;
1246       LLVMValueRef clipmask;   /* holds the clipmask value */
1247       const LLVMValueRef (*ptr_aos)[NUM_CHANNELS];
1248
1249       io_itr = LLVMBuildSub(builder, lp_loop.counter, start, "");
1250       io = LLVMBuildGEP(builder, io_ptr, &io_itr, 1, "");
1251 #if DEBUG_STORE
1252       lp_build_printf(builder, " --- io %d = %p, loop counter %d\n",
1253                       io_itr, io, lp_loop.counter);
1254 #endif
1255       for (i = 0; i < NUM_CHANNELS; ++i) {
1256          LLVMValueRef true_index = LLVMBuildAdd(
1257             builder,
1258             lp_loop.counter,
1259             lp_build_const_int32(gallivm, i), "");
1260          for (j = 0; j < draw->pt.nr_vertex_elements; ++j) {
1261             struct pipe_vertex_element *velem = &draw->pt.vertex_element[j];
1262             LLVMValueRef vb_index = lp_build_const_int32(gallivm, velem->vertex_buffer_index);
1263             LLVMValueRef vb = LLVMBuildGEP(builder, vb_ptr,
1264                                            &vb_index, 1, "");
1265             generate_fetch(llvm->gallivm, vbuffers_ptr,
1266                            &aos_attribs[j][i], velem, vb, true_index,
1267                            instance_id);
1268          }
1269       }
1270       convert_to_soa(gallivm, aos_attribs, inputs,
1271                      draw->pt.nr_vertex_elements);
1272
1273       ptr_aos = (const LLVMValueRef (*)[NUM_CHANNELS]) inputs;
1274       generate_vs(llvm,
1275                   builder,
1276                   outputs,
1277                   ptr_aos,
1278                   system_values_array,
1279                   context_ptr,
1280                   sampler,
1281                   variant->key.clamp_vertex_color);
1282
1283       /* store original positions in clip before further manipulation */
1284       store_clip(gallivm, io, outputs);
1285
1286       /* do cliptest */
1287       if (enable_cliptest) {
1288          /* allocate clipmask, assign it integer type */
1289          clipmask = generate_clipmask(gallivm, outputs,
1290                                       variant->key.clip_xy,
1291                                       variant->key.clip_z,
1292                                       variant->key.clip_user,
1293                                       variant->key.clip_halfz,
1294                                       variant->key.nr_planes,
1295                                       context_ptr);
1296          /* return clipping boolean value for function */
1297          clipmask_bool(gallivm, clipmask, ret_ptr);
1298       }
1299       else {
1300          clipmask = lp_build_const_int_vec(gallivm, lp_type_int_vec(32), 0);
1301       }
1302
1303       /* do viewport mapping */
1304       if (!bypass_viewport) {
1305          generate_viewport(llvm, builder, outputs, context_ptr);
1306       }
1307
1308       /* store clipmask in vertex header and positions in data */
1309       convert_to_aos(gallivm, io, outputs, clipmask,
1310                      vs_info->num_outputs, max_vertices);
1311    }
1312
1313    lp_build_loop_end_cond(&lp_loop, end, step, LLVMIntUGE);
1314
1315    sampler->destroy(sampler);
1316
1317    ret = LLVMBuildLoad(builder, ret_ptr,"");
1318    LLVMBuildRet(builder, ret);
1319
1320    /*
1321     * Translate the LLVM IR into machine code.
1322     */
1323 #ifdef DEBUG
1324    if (LLVMVerifyFunction(variant->function, LLVMPrintMessageAction)) {
1325       lp_debug_dump_value(variant->function);
1326       assert(0);
1327    }
1328 #endif
1329
1330    LLVMRunFunctionPassManager(gallivm->passmgr, variant->function);
1331
1332    if (gallivm_debug & GALLIVM_DEBUG_IR) {
1333       lp_debug_dump_value(variant->function);
1334       debug_printf("\n");
1335    }
1336
1337    code = LLVMGetPointerToGlobal(gallivm->engine, variant->function);
1338    variant->jit_func = (draw_jit_vert_func)pointer_to_func(code);
1339
1340    if (gallivm_debug & GALLIVM_DEBUG_ASM) {
1341       lp_disassemble(code);
1342    }
1343    lp_func_delete_body(variant->function);
1344 }
1345
1346
1347 static void
1348 draw_llvm_generate_elts(struct draw_llvm *llvm, struct draw_llvm_variant *variant)
1349 {
1350    struct gallivm_state *gallivm = llvm->gallivm;
1351    LLVMContextRef context = gallivm->context;
1352    LLVMTypeRef int32_type = LLVMInt32TypeInContext(context);
1353    LLVMTypeRef arg_types[8];
1354    LLVMTypeRef func_type;
1355    LLVMValueRef context_ptr;
1356    LLVMBasicBlockRef block;
1357    LLVMBuilderRef builder;
1358    LLVMValueRef fetch_elts, fetch_count, stride, step, io_itr;
1359    LLVMValueRef io_ptr, vbuffers_ptr, vb_ptr;
1360    LLVMValueRef instance_id;
1361    LLVMValueRef system_values_array;
1362    struct draw_context *draw = llvm->draw;
1363    const struct tgsi_shader_info *vs_info = &draw->vs.vertex_shader->info;
1364    unsigned i, j;
1365    struct lp_build_context bld;
1366    struct lp_build_loop_state lp_loop;
1367    const int max_vertices = 4;
1368    LLVMValueRef outputs[PIPE_MAX_SHADER_OUTPUTS][NUM_CHANNELS];
1369    LLVMValueRef fetch_max;
1370    void *code;
1371    struct lp_build_sampler_soa *sampler = 0;
1372    LLVMValueRef ret, ret_ptr;
1373    boolean bypass_viewport = variant->key.bypass_viewport;
1374    boolean enable_cliptest = variant->key.clip_xy ||
1375                              variant->key.clip_z  ||
1376                              variant->key.clip_user;
1377
1378    arg_types[0] = get_context_ptr_type(llvm);           /* context */
1379    arg_types[1] = get_vertex_header_ptr_type(llvm);     /* vertex_header */
1380    arg_types[2] = get_buffer_ptr_type(llvm);            /* vbuffers */
1381    arg_types[3] = LLVMPointerType(int32_type, 0);       /* fetch_elts * */
1382    arg_types[4] = int32_type;                           /* fetch_count */
1383    arg_types[5] = int32_type;                           /* stride */
1384    arg_types[6] = get_vb_ptr_type(llvm);                /* pipe_vertex_buffer's */
1385    arg_types[7] = int32_type;                           /* instance_id */
1386
1387    func_type = LLVMFunctionType(int32_type, arg_types, Elements(arg_types), 0);
1388
1389    variant->function_elts = LLVMAddFunction(gallivm->module, "draw_llvm_shader_elts", func_type);
1390    LLVMSetFunctionCallConv(variant->function_elts, LLVMCCallConv);
1391    for (i = 0; i < Elements(arg_types); ++i)
1392       if (LLVMGetTypeKind(arg_types[i]) == LLVMPointerTypeKind)
1393          LLVMAddAttribute(LLVMGetParam(variant->function_elts, i),
1394                           LLVMNoAliasAttribute);
1395
1396    context_ptr  = LLVMGetParam(variant->function_elts, 0);
1397    io_ptr       = LLVMGetParam(variant->function_elts, 1);
1398    vbuffers_ptr = LLVMGetParam(variant->function_elts, 2);
1399    fetch_elts   = LLVMGetParam(variant->function_elts, 3);
1400    fetch_count  = LLVMGetParam(variant->function_elts, 4);
1401    stride       = LLVMGetParam(variant->function_elts, 5);
1402    vb_ptr       = LLVMGetParam(variant->function_elts, 6);
1403    instance_id  = LLVMGetParam(variant->function_elts, 7);
1404
1405    lp_build_name(context_ptr, "context");
1406    lp_build_name(io_ptr, "io");
1407    lp_build_name(vbuffers_ptr, "vbuffers");
1408    lp_build_name(fetch_elts, "fetch_elts");
1409    lp_build_name(fetch_count, "fetch_count");
1410    lp_build_name(stride, "stride");
1411    lp_build_name(vb_ptr, "vb");
1412    lp_build_name(instance_id, "instance_id");
1413
1414    /*
1415     * Function body
1416     */
1417
1418    block = LLVMAppendBasicBlockInContext(gallivm->context, variant->function_elts, "entry");
1419    builder = gallivm->builder;
1420    LLVMPositionBuilderAtEnd(builder, block);
1421
1422    lp_build_context_init(&bld, gallivm, lp_type_int(32));
1423
1424    system_values_array = lp_build_system_values_array(gallivm, vs_info,
1425                                                       instance_id, NULL);
1426
1427
1428    step = lp_build_const_int32(gallivm, max_vertices);
1429
1430    /* code generated texture sampling */
1431    sampler = draw_llvm_sampler_soa_create(
1432       draw_llvm_variant_key_samplers(&variant->key),
1433       context_ptr);
1434
1435    fetch_max = LLVMBuildSub(builder, fetch_count,
1436                             lp_build_const_int32(gallivm, 1),
1437                             "fetch_max");
1438
1439    /* function returns non-zero i32 value if any clipped vertices */
1440    ret_ptr = lp_build_alloca(gallivm, int32_type, "");
1441    LLVMBuildStore(builder, lp_build_const_int32(gallivm, 0), ret_ptr);
1442
1443    lp_build_loop_begin(&lp_loop, gallivm, lp_build_const_int32(gallivm, 0));
1444    {
1445       LLVMValueRef inputs[PIPE_MAX_SHADER_INPUTS][NUM_CHANNELS];
1446       LLVMValueRef aos_attribs[PIPE_MAX_SHADER_INPUTS][NUM_CHANNELS] = { { 0 } };
1447       LLVMValueRef io;
1448       LLVMValueRef clipmask;   /* holds the clipmask value */
1449       const LLVMValueRef (*ptr_aos)[NUM_CHANNELS];
1450
1451       io_itr = lp_loop.counter;
1452       io = LLVMBuildGEP(builder, io_ptr, &io_itr, 1, "");
1453 #if DEBUG_STORE
1454       lp_build_printf(builder, " --- io %d = %p, loop counter %d\n",
1455                       io_itr, io, lp_loop.counter);
1456 #endif
1457       for (i = 0; i < NUM_CHANNELS; ++i) {
1458          LLVMValueRef true_index = LLVMBuildAdd(
1459             builder,
1460             lp_loop.counter,
1461             lp_build_const_int32(gallivm, i), "");
1462          LLVMValueRef fetch_ptr;
1463
1464          /* make sure we're not out of bounds which can happen
1465           * if fetch_count % 4 != 0, because on the last iteration
1466           * a few of the 4 vertex fetches will be out of bounds */
1467          true_index = lp_build_min(&bld, true_index, fetch_max);
1468
1469          fetch_ptr = LLVMBuildGEP(builder, fetch_elts,
1470                                   &true_index, 1, "");
1471          true_index = LLVMBuildLoad(builder, fetch_ptr, "fetch_elt");
1472          for (j = 0; j < draw->pt.nr_vertex_elements; ++j) {
1473             struct pipe_vertex_element *velem = &draw->pt.vertex_element[j];
1474             LLVMValueRef vb_index = lp_build_const_int32(gallivm, velem->vertex_buffer_index);
1475             LLVMValueRef vb = LLVMBuildGEP(builder, vb_ptr,
1476                                            &vb_index, 1, "");
1477             generate_fetch(gallivm, vbuffers_ptr,
1478                            &aos_attribs[j][i], velem, vb, true_index,
1479                            instance_id);
1480          }
1481       }
1482       convert_to_soa(gallivm, aos_attribs, inputs,
1483                      draw->pt.nr_vertex_elements);
1484
1485       ptr_aos = (const LLVMValueRef (*)[NUM_CHANNELS]) inputs;
1486       generate_vs(llvm,
1487                   builder,
1488                   outputs,
1489                   ptr_aos,
1490                   system_values_array,
1491                   context_ptr,
1492                   sampler,
1493                   variant->key.clamp_vertex_color);
1494
1495       /* store original positions in clip before further manipulation */
1496       store_clip(gallivm, io, outputs);
1497
1498       /* do cliptest */
1499       if (enable_cliptest) {
1500          /* allocate clipmask, assign it integer type */
1501          clipmask = generate_clipmask(gallivm, outputs,
1502                                       variant->key.clip_xy,
1503                                       variant->key.clip_z,
1504                                       variant->key.clip_user,
1505                                       variant->key.clip_halfz,
1506                                       variant->key.nr_planes,
1507                                       context_ptr);
1508          /* return clipping boolean value for function */
1509          clipmask_bool(gallivm, clipmask, ret_ptr);
1510       }
1511       else {
1512          clipmask = lp_build_const_int_vec(gallivm, lp_type_int_vec(32), 0);
1513       }
1514
1515       /* do viewport mapping */
1516       if (!bypass_viewport) {
1517          generate_viewport(llvm, builder, outputs, context_ptr);
1518       }
1519
1520       /* store clipmask in vertex header,
1521        * original positions in clip
1522        * and transformed positions in data
1523        */
1524       convert_to_aos(gallivm, io, outputs, clipmask,
1525                      vs_info->num_outputs, max_vertices);
1526    }
1527
1528    lp_build_loop_end_cond(&lp_loop, fetch_count, step, LLVMIntUGE);
1529
1530    sampler->destroy(sampler);
1531
1532    ret = LLVMBuildLoad(builder, ret_ptr,"");
1533    LLVMBuildRet(builder, ret);
1534
1535    /*
1536     * Translate the LLVM IR into machine code.
1537     */
1538 #ifdef DEBUG
1539    if (LLVMVerifyFunction(variant->function_elts, LLVMPrintMessageAction)) {
1540       lp_debug_dump_value(variant->function_elts);
1541       assert(0);
1542    }
1543 #endif
1544
1545    LLVMRunFunctionPassManager(gallivm->passmgr, variant->function_elts);
1546
1547    if (gallivm_debug & GALLIVM_DEBUG_IR) {
1548       lp_debug_dump_value(variant->function_elts);
1549       debug_printf("\n");
1550    }
1551
1552    code = LLVMGetPointerToGlobal(gallivm->engine, variant->function_elts);
1553    variant->jit_func_elts = (draw_jit_vert_func_elts)pointer_to_func(code);
1554
1555    if (gallivm_debug & GALLIVM_DEBUG_ASM) {
1556       lp_disassemble(code);
1557    }
1558    lp_func_delete_body(variant->function_elts);
1559 }
1560
1561
1562 struct draw_llvm_variant_key *
1563 draw_llvm_make_variant_key(struct draw_llvm *llvm, char *store)
1564 {
1565    unsigned i;
1566    struct draw_llvm_variant_key *key;
1567    struct lp_sampler_static_state *sampler;
1568
1569    key = (struct draw_llvm_variant_key *)store;
1570
1571    key->clamp_vertex_color = llvm->draw->rasterizer->clamp_vertex_color; /**/
1572
1573    /* Presumably all variants of the shader should have the same
1574     * number of vertex elements - ie the number of shader inputs.
1575     */
1576    key->nr_vertex_elements = llvm->draw->pt.nr_vertex_elements;
1577
1578    /* will have to rig this up properly later */
1579    key->clip_xy = llvm->draw->clip_xy;
1580    key->clip_z = llvm->draw->clip_z;
1581    key->clip_user = llvm->draw->clip_user;
1582    key->bypass_viewport = llvm->draw->identity_viewport;
1583    key->clip_halfz = !llvm->draw->rasterizer->gl_rasterization_rules;
1584    key->need_edgeflags = (llvm->draw->vs.edgeflag_output ? TRUE : FALSE);
1585    key->nr_planes = llvm->draw->nr_planes;
1586    key->pad = 0;
1587
1588    /* All variants of this shader will have the same value for
1589     * nr_samplers.  Not yet trying to compact away holes in the
1590     * sampler array.
1591     */
1592    key->nr_samplers = llvm->draw->vs.vertex_shader->info.file_max[TGSI_FILE_SAMPLER] + 1;
1593
1594    sampler = draw_llvm_variant_key_samplers(key);
1595
1596    memcpy(key->vertex_element,
1597           llvm->draw->pt.vertex_element,
1598           sizeof(struct pipe_vertex_element) * key->nr_vertex_elements);
1599
1600    memset(sampler, 0, key->nr_samplers * sizeof *sampler);
1601
1602    for (i = 0 ; i < key->nr_samplers; i++) {
1603       lp_sampler_static_state(&sampler[i],
1604                               llvm->draw->sampler_views[i],
1605                               llvm->draw->samplers[i]);
1606    }
1607
1608    return key;
1609 }
1610
1611
1612 void
1613 draw_llvm_set_mapped_texture(struct draw_context *draw,
1614                              unsigned sampler_idx,
1615                              uint32_t width, uint32_t height, uint32_t depth,
1616                              uint32_t first_level, uint32_t last_level,
1617                              uint32_t row_stride[PIPE_MAX_TEXTURE_LEVELS],
1618                              uint32_t img_stride[PIPE_MAX_TEXTURE_LEVELS],
1619                              const void *data[PIPE_MAX_TEXTURE_LEVELS])
1620 {
1621    unsigned j;
1622    struct draw_jit_texture *jit_tex;
1623
1624    assert(sampler_idx < PIPE_MAX_VERTEX_SAMPLERS);
1625
1626    jit_tex = &draw->llvm->jit_context.textures[sampler_idx];
1627
1628    jit_tex->width = width;
1629    jit_tex->height = height;
1630    jit_tex->depth = depth;
1631    jit_tex->first_level = first_level;
1632    jit_tex->last_level = last_level;
1633
1634    for (j = first_level; j <= last_level; j++) {
1635       jit_tex->data[j] = data[j];
1636       jit_tex->row_stride[j] = row_stride[j];
1637       jit_tex->img_stride[j] = img_stride[j];
1638    }
1639 }
1640
1641
1642 void
1643 draw_llvm_set_sampler_state(struct draw_context *draw)
1644 {
1645    unsigned i;
1646
1647    for (i = 0; i < draw->num_samplers; i++) {
1648       struct draw_jit_texture *jit_tex = &draw->llvm->jit_context.textures[i];
1649
1650       if (draw->samplers[i]) {
1651          jit_tex->min_lod = draw->samplers[i]->min_lod;
1652          jit_tex->max_lod = draw->samplers[i]->max_lod;
1653          jit_tex->lod_bias = draw->samplers[i]->lod_bias;
1654          COPY_4V(jit_tex->border_color, draw->samplers[i]->border_color);
1655       }
1656    }
1657 }
1658
1659
1660 void
1661 draw_llvm_destroy_variant(struct draw_llvm_variant *variant)
1662 {
1663    struct draw_llvm *llvm = variant->llvm;
1664
1665    if (variant->function_elts) {
1666       LLVMFreeMachineCodeForFunction(llvm->gallivm->engine,
1667                                      variant->function_elts);
1668       LLVMDeleteFunction(variant->function_elts);
1669    }
1670
1671    if (variant->function) {
1672       LLVMFreeMachineCodeForFunction(llvm->gallivm->engine,
1673                                      variant->function);
1674       LLVMDeleteFunction(variant->function);
1675    }
1676
1677    remove_from_list(&variant->list_item_local);
1678    variant->shader->variants_cached--;
1679    remove_from_list(&variant->list_item_global);
1680    llvm->nr_variants--;
1681    FREE(variant);
1682 }