src/gallium/auxiliary/gallivm/lp_bld_sample_soa.c

   1 /**************************************************************************
   2  *
   3  * Copyright 2009 VMware, Inc.
   4  * All Rights Reserved.
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a
   7  * copy of this software and associated documentation files (the
   8  * "Software"), to deal in the Software without restriction, including
   9  * without limitation the rights to use, copy, modify, merge, publish,
  10  * distribute, sub license, and/or sell copies of the Software, and to
  11  * permit persons to whom the Software is furnished to do so, subject to
  12  * the following conditions:
  13  *
  14  * The above copyright notice and this permission notice (including the
  15  * next paragraph) shall be included in all copies or substantial portions
  16  * of the Software.
  17  *
  18  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
  19  * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  20  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
  21  * IN NO EVENT SHALL VMWARE AND/OR ITS SUPPLIERS BE LIABLE FOR
  22  * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
  23  * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
  24  * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  25  *
  26  **************************************************************************/
  27
  28 /**
  29  * @file
  30  * Texture sampling -- SoA.
  31  *
  32  * @author Jose Fonseca <jfonseca@vmware.com>
  33  * @author Brian Paul <brianp@vmware.com>
  34  */
  35
  36 #include "pipe/p_defines.h"
  37 #include "pipe/p_state.h"
  38 #include "pipe/p_shader_tokens.h"
  39 #include "util/u_debug.h"
  40 #include "util/u_dump.h"
  41 #include "util/u_memory.h"
  42 #include "util/u_math.h"
  43 #include "util/u_format.h"
  44 #include "util/u_cpu_detect.h"
  45 #include "util/u_format_rgb9e5.h"
  46 #include "lp_bld_debug.h"
  47 #include "lp_bld_type.h"
  48 #include "lp_bld_const.h"
  49 #include "lp_bld_conv.h"
  50 #include "lp_bld_arit.h"
  51 #include "lp_bld_bitarit.h"
  52 #include "lp_bld_logic.h"
  53 #include "lp_bld_printf.h"
  54 #include "lp_bld_swizzle.h"
  55 #include "lp_bld_flow.h"
  56 #include "lp_bld_gather.h"
  57 #include "lp_bld_format.h"
  58 #include "lp_bld_sample.h"
  59 #include "lp_bld_sample_aos.h"
  60 #include "lp_bld_struct.h"
  61 #include "lp_bld_quad.h"
  62 #include "lp_bld_pack.h"
  63
  64
  65 /**
  66  * Generate code to fetch a texel from a texture at int coords (x, y, z).
  67  * The computation depends on whether the texture is 1D, 2D or 3D.
  68  * The result, texel, will be float vectors:
  69  *   texel[0] = red values
  70  *   texel[1] = green values
  71  *   texel[2] = blue values
  72  *   texel[3] = alpha values
  73  */
  74 static void
  75 lp_build_sample_texel_soa(struct lp_build_sample_context *bld,
  76                           LLVMValueRef width,
  77                           LLVMValueRef height,
  78                           LLVMValueRef depth,
  79                           LLVMValueRef x,
  80                           LLVMValueRef y,
  81                           LLVMValueRef z,
  82                           LLVMValueRef y_stride,
  83                           LLVMValueRef z_stride,
  84                           LLVMValueRef data_ptr,
  85                           LLVMValueRef mipoffsets,
  86                           LLVMValueRef texel_out[4])
  87 {
  88    const struct lp_static_sampler_state *static_state = bld->static_sampler_state;
  89    const unsigned dims = bld->dims;
  90    struct lp_build_context *int_coord_bld = &bld->int_coord_bld;
  91    LLVMBuilderRef builder = bld->gallivm->builder;
  92    LLVMValueRef offset;
  93    LLVMValueRef i, j;
  94    LLVMValueRef use_border = NULL;
  95
  96    /* use_border = x < 0 || x >= width || y < 0 || y >= height */
  97    if (lp_sampler_wrap_mode_uses_border_color(static_state->wrap_s,
  98                                               static_state->min_img_filter,
  99                                               static_state->mag_img_filter)) {
 100       LLVMValueRef b1, b2;
 101       b1 = lp_build_cmp(int_coord_bld, PIPE_FUNC_LESS, x, int_coord_bld->zero);
 102       b2 = lp_build_cmp(int_coord_bld, PIPE_FUNC_GEQUAL, x, width);
 103       use_border = LLVMBuildOr(builder, b1, b2, "b1_or_b2");
 104    }
 105
 106    if (dims >= 2 &&
 107        lp_sampler_wrap_mode_uses_border_color(static_state->wrap_t,
 108                                               static_state->min_img_filter,
 109                                               static_state->mag_img_filter)) {
 110       LLVMValueRef b1, b2;
 111       b1 = lp_build_cmp(int_coord_bld, PIPE_FUNC_LESS, y, int_coord_bld->zero);
 112       b2 = lp_build_cmp(int_coord_bld, PIPE_FUNC_GEQUAL, y, height);
 113       if (use_border) {
 114          use_border = LLVMBuildOr(builder, use_border, b1, "ub_or_b1");
 115          use_border = LLVMBuildOr(builder, use_border, b2, "ub_or_b2");
 116       }
 117       else {
 118          use_border = LLVMBuildOr(builder, b1, b2, "b1_or_b2");
 119       }
 120    }
 121
 122    if (dims == 3 &&
 123        lp_sampler_wrap_mode_uses_border_color(static_state->wrap_r,
 124                                               static_state->min_img_filter,
 125                                               static_state->mag_img_filter)) {
 126       LLVMValueRef b1, b2;
 127       b1 = lp_build_cmp(int_coord_bld, PIPE_FUNC_LESS, z, int_coord_bld->zero);
 128       b2 = lp_build_cmp(int_coord_bld, PIPE_FUNC_GEQUAL, z, depth);
 129       if (use_border) {
 130          use_border = LLVMBuildOr(builder, use_border, b1, "ub_or_b1");
 131          use_border = LLVMBuildOr(builder, use_border, b2, "ub_or_b2");
 132       }
 133       else {
 134          use_border = LLVMBuildOr(builder, b1, b2, "b1_or_b2");
 135       }
 136    }
 137
 138    /* convert x,y,z coords to linear offset from start of texture, in bytes */
 139    lp_build_sample_offset(&bld->int_coord_bld,
 140                           bld->format_desc,
 141                           x, y, z, y_stride, z_stride,
 142                           &offset, &i, &j);
 143    if (mipoffsets) {
 144       offset = lp_build_add(&bld->int_coord_bld, offset, mipoffsets);
 145    }
 146
 147    if (use_border) {
 148       /* If we can sample the border color, it means that texcoords may
 149        * lie outside the bounds of the texture image.  We need to do
 150        * something to prevent reading out of bounds and causing a segfault.
 151        *
 152        * Simply AND the texture coords with !use_border.  This will cause
 153        * coords which are out of bounds to become zero.  Zero's guaranteed
 154        * to be inside the texture image.
 155        */
 156       offset = lp_build_andnot(&bld->int_coord_bld, offset, use_border);
 157    }
 158
 159    lp_build_fetch_rgba_soa(bld->gallivm,
 160                            bld->format_desc,
 161                            bld->texel_type,
 162                            data_ptr, offset,
 163                            i, j,
 164                            texel_out);
 165
 166    /*
 167     * Note: if we find an app which frequently samples the texture border
 168     * we might want to implement a true conditional here to avoid sampling
 169     * the texture whenever possible (since that's quite a bit of code).
 170     * Ex:
 171     *   if (use_border) {
 172     *      texel = border_color;
 173     *   }
 174     *   else {
 175     *      texel = sample_texture(coord);
 176     *   }
 177     * As it is now, we always sample the texture, then selectively replace
 178     * the texel color results with the border color.
 179     */
 180
 181    if (use_border) {
 182       /* select texel color or border color depending on use_border. */
 183       const struct util_format_description *format_desc = bld->format_desc;
 184       int chan;
 185       struct lp_type border_type = bld->texel_type;
 186       border_type.length = 4;
 187       /*
 188        * Only replace channels which are actually present. The others should
 189        * get optimized away eventually by sampler_view swizzle anyway but it's
 190        * easier too.
 191        */
 192       for (chan = 0; chan < 4; chan++) {
 193          unsigned chan_s;
 194          /* reverse-map channel... */
 195          for (chan_s = 0; chan_s < 4; chan_s++) {
 196             if (chan_s == format_desc->swizzle[chan]) {
 197                break;
 198             }
 199          }
 200          if (chan_s <= 3) {
 201             /* use the already clamped color */
 202             LLVMValueRef idx = lp_build_const_int32(bld->gallivm, chan);
 203             LLVMValueRef border_chan;
 204
 205             border_chan = lp_build_extract_broadcast(bld->gallivm,
 206                                                      border_type,
 207                                                      bld->texel_type,
 208                                                      bld->border_color_clamped,
 209                                                      idx);
 210             texel_out[chan] = lp_build_select(&bld->texel_bld, use_border,
 211                                               border_chan, texel_out[chan]);
 212          }
 213       }
 214    }
 215 }
 216
 217
 218 /**
 219  * Helper to compute the mirror function for the PIPE_WRAP_MIRROR modes.
 220  */
 221 static LLVMValueRef
 222 lp_build_coord_mirror(struct lp_build_sample_context *bld,
 223                       LLVMValueRef coord)
 224 {
 225    struct lp_build_context *coord_bld = &bld->coord_bld;
 226    struct lp_build_context *int_coord_bld = &bld->int_coord_bld;
 227    LLVMValueRef fract, flr, isOdd;
 228
 229    lp_build_ifloor_fract(coord_bld, coord, &flr, &fract);
 230
 231    /* isOdd = flr & 1 */
 232    isOdd = LLVMBuildAnd(bld->gallivm->builder, flr, int_coord_bld->one, "");
 233
 234    /* make coord positive or negative depending on isOdd */
 235    coord = lp_build_set_sign(coord_bld, fract, isOdd);
 236
 237    /* convert isOdd to float */
 238    isOdd = lp_build_int_to_float(coord_bld, isOdd);
 239
 240    /* add isOdd to coord */
 241    coord = lp_build_add(coord_bld, coord, isOdd);
 242
 243    return coord;
 244 }
 245
 246
 247 /**
 248  * Helper to compute the first coord and the weight for
 249  * linear wrap repeat npot textures
 250  */
 251 void
 252 lp_build_coord_repeat_npot_linear(struct lp_build_sample_context *bld,
 253                                   LLVMValueRef coord_f,
 254                                   LLVMValueRef length_i,
 255                                   LLVMValueRef length_f,
 256                                   LLVMValueRef *coord0_i,
 257                                   LLVMValueRef *weight_f)
 258 {
 259    struct lp_build_context *coord_bld = &bld->coord_bld;
 260    struct lp_build_context *int_coord_bld = &bld->int_coord_bld;
 261    LLVMValueRef half = lp_build_const_vec(bld->gallivm, coord_bld->type, 0.5);
 262    LLVMValueRef length_minus_one = lp_build_sub(int_coord_bld, length_i,
 263                                                 int_coord_bld->one);
 264    LLVMValueRef mask;
 265    /* wrap with normalized floats is just fract */
 266    coord_f = lp_build_fract(coord_bld, coord_f);
 267    /* mul by size and subtract 0.5 */
 268    coord_f = lp_build_mul(coord_bld, coord_f, length_f);
 269    coord_f = lp_build_sub(coord_bld, coord_f, half);
 270    /*
 271     * we avoided the 0.5/length division before the repeat wrap,
 272     * now need to fix up edge cases with selects
 273     */
 274    /* convert to int, compute lerp weight */
 275    lp_build_ifloor_fract(coord_bld, coord_f, coord0_i, weight_f);
 276    mask = lp_build_compare(int_coord_bld->gallivm, int_coord_bld->type,
 277                            PIPE_FUNC_LESS, *coord0_i, int_coord_bld->zero);
 278    *coord0_i = lp_build_select(int_coord_bld, mask, length_minus_one, *coord0_i);
 279 }
 280
 281
 282 /**
 283  * Build LLVM code for texture wrap mode for linear filtering.
 284  * \param x0_out  returns first integer texcoord
 285  * \param x1_out  returns second integer texcoord
 286  * \param weight_out  returns linear interpolation weight
 287  */
 288 static void
 289 lp_build_sample_wrap_linear(struct lp_build_sample_context *bld,
 290                             LLVMValueRef coord,
 291                             LLVMValueRef length,
 292                             LLVMValueRef length_f,
 293                             LLVMValueRef offset,
 294                             boolean is_pot,
 295                             unsigned wrap_mode,
 296                             LLVMValueRef *x0_out,
 297                             LLVMValueRef *x1_out,
 298                             LLVMValueRef *weight_out)
 299 {
 300    struct lp_build_context *coord_bld = &bld->coord_bld;
 301    struct lp_build_context *int_coord_bld = &bld->int_coord_bld;
 302    LLVMBuilderRef builder = bld->gallivm->builder;
 303    LLVMValueRef half = lp_build_const_vec(bld->gallivm, coord_bld->type, 0.5);
 304    LLVMValueRef length_minus_one = lp_build_sub(int_coord_bld, length, int_coord_bld->one);
 305    LLVMValueRef coord0, coord1, weight;
 306
 307    switch(wrap_mode) {
 308    case PIPE_TEX_WRAP_REPEAT:
 309       if (is_pot) {
 310          /* mul by size and subtract 0.5 */
 311          coord = lp_build_mul(coord_bld, coord, length_f);
 312          coord = lp_build_sub(coord_bld, coord, half);
 313          if (offset) {
 314             offset = lp_build_int_to_float(coord_bld, offset);
 315             coord = lp_build_add(coord_bld, coord, offset);
 316          }
 317          /* convert to int, compute lerp weight */
 318          lp_build_ifloor_fract(coord_bld, coord, &coord0, &weight);
 319          coord1 = lp_build_add(int_coord_bld, coord0, int_coord_bld->one);
 320          /* repeat wrap */
 321          coord0 = LLVMBuildAnd(builder, coord0, length_minus_one, "");
 322          coord1 = LLVMBuildAnd(builder, coord1, length_minus_one, "");
 323       }
 324       else {
 325          LLVMValueRef mask;
 326          if (offset) {
 327             offset = lp_build_int_to_float(coord_bld, offset);
 328             offset = lp_build_div(coord_bld, offset, length_f);
 329             coord = lp_build_add(coord_bld, coord, offset);
 330          }
 331          lp_build_coord_repeat_npot_linear(bld, coord,
 332                                            length, length_f,
 333                                            &coord0, &weight);
 334          mask = lp_build_compare(int_coord_bld->gallivm, int_coord_bld->type,
 335                                  PIPE_FUNC_NOTEQUAL, coord0, length_minus_one);
 336          coord1 = LLVMBuildAnd(builder,
 337                                lp_build_add(int_coord_bld, coord0, int_coord_bld->one),
 338                                mask, "");
 339       }
 340       break;
 341
 342    case PIPE_TEX_WRAP_CLAMP:
 343       if (bld->static_sampler_state->normalized_coords) {
 344          /* scale coord to length */
 345          coord = lp_build_mul(coord_bld, coord, length_f);
 346       }
 347       if (offset) {
 348          offset = lp_build_int_to_float(coord_bld, offset);
 349          coord = lp_build_add(coord_bld, coord, offset);
 350       }
 351
 352       /* clamp to [0, length] */
 353       coord = lp_build_clamp(coord_bld, coord, coord_bld->zero, length_f);
 354
 355       coord = lp_build_sub(coord_bld, coord, half);
 356
 357       /* convert to int, compute lerp weight */
 358       lp_build_ifloor_fract(coord_bld, coord, &coord0, &weight);
 359       coord1 = lp_build_add(int_coord_bld, coord0, int_coord_bld->one);
 360       break;
 361
 362    case PIPE_TEX_WRAP_CLAMP_TO_EDGE:
 363       {
 364          struct lp_build_context abs_coord_bld = bld->coord_bld;
 365          abs_coord_bld.type.sign = FALSE;
 366
 367          if (bld->static_sampler_state->normalized_coords) {
 368             /* mul by tex size */
 369             coord = lp_build_mul(coord_bld, coord, length_f);
 370          }
 371          if (offset) {
 372             offset = lp_build_int_to_float(coord_bld, offset);
 373             coord = lp_build_add(coord_bld, coord, offset);
 374          }
 375
 376          /* clamp to length max */
 377          coord = lp_build_min(coord_bld, coord, length_f);
 378          /* subtract 0.5 */
 379          coord = lp_build_sub(coord_bld, coord, half);
 380          /* clamp to [0, length - 0.5] */
 381          coord = lp_build_max(coord_bld, coord, coord_bld->zero);
 382          /* convert to int, compute lerp weight */
 383          lp_build_ifloor_fract(&abs_coord_bld, coord, &coord0, &weight);
 384          coord1 = lp_build_add(int_coord_bld, coord0, int_coord_bld->one);
 385          /* coord1 = min(coord1, length-1) */
 386          coord1 = lp_build_min(int_coord_bld, coord1, length_minus_one);
 387          break;
 388       }
 389
 390    case PIPE_TEX_WRAP_CLAMP_TO_BORDER:
 391       if (bld->static_sampler_state->normalized_coords) {
 392          /* scale coord to length */
 393          coord = lp_build_mul(coord_bld, coord, length_f);
 394       }
 395       if (offset) {
 396          offset = lp_build_int_to_float(coord_bld, offset);
 397          coord = lp_build_add(coord_bld, coord, offset);
 398       }
 399       /* was: clamp to [-0.5, length + 0.5], then sub 0.5 */
 400       /* can skip clamp (though might not work for very large coord values */
 401       coord = lp_build_sub(coord_bld, coord, half);
 402       /* convert to int, compute lerp weight */
 403       lp_build_ifloor_fract(coord_bld, coord, &coord0, &weight);
 404       coord1 = lp_build_add(int_coord_bld, coord0, int_coord_bld->one);
 405       break;
 406
 407    case PIPE_TEX_WRAP_MIRROR_REPEAT:
 408       if (offset) {
 409          offset = lp_build_int_to_float(coord_bld, offset);
 410          offset = lp_build_div(coord_bld, offset, length_f);
 411          coord = lp_build_add(coord_bld, coord, offset);
 412       }
 413       /* compute mirror function */
 414       coord = lp_build_coord_mirror(bld, coord);
 415
 416       /* scale coord to length */
 417       coord = lp_build_mul(coord_bld, coord, length_f);
 418       coord = lp_build_sub(coord_bld, coord, half);
 419
 420       /* convert to int, compute lerp weight */
 421       lp_build_ifloor_fract(coord_bld, coord, &coord0, &weight);
 422       coord1 = lp_build_add(int_coord_bld, coord0, int_coord_bld->one);
 423
 424       /* coord0 = max(coord0, 0) */
 425       coord0 = lp_build_max(int_coord_bld, coord0, int_coord_bld->zero);
 426       /* coord1 = min(coord1, length-1) */
 427       coord1 = lp_build_min(int_coord_bld, coord1, length_minus_one);
 428       break;
 429
 430    case PIPE_TEX_WRAP_MIRROR_CLAMP:
 431       if (bld->static_sampler_state->normalized_coords) {
 432          /* scale coord to length */
 433          coord = lp_build_mul(coord_bld, coord, length_f);
 434       }
 435       if (offset) {
 436          offset = lp_build_int_to_float(coord_bld, offset);
 437          coord = lp_build_add(coord_bld, coord, offset);
 438       }
 439       coord = lp_build_abs(coord_bld, coord);
 440
 441       /* clamp to [0, length] */
 442       coord = lp_build_min(coord_bld, coord, length_f);
 443
 444       coord = lp_build_sub(coord_bld, coord, half);
 445
 446       /* convert to int, compute lerp weight */
 447       lp_build_ifloor_fract(coord_bld, coord, &coord0, &weight);
 448       coord1 = lp_build_add(int_coord_bld, coord0, int_coord_bld->one);
 449       break;
 450
 451    case PIPE_TEX_WRAP_MIRROR_CLAMP_TO_EDGE:
 452       {
 453          struct lp_build_context abs_coord_bld = bld->coord_bld;
 454          abs_coord_bld.type.sign = FALSE;
 455
 456          if (bld->static_sampler_state->normalized_coords) {
 457             /* scale coord to length */
 458             coord = lp_build_mul(coord_bld, coord, length_f);
 459          }
 460          if (offset) {
 461             offset = lp_build_int_to_float(coord_bld, offset);
 462             coord = lp_build_add(coord_bld, coord, offset);
 463          }
 464          coord = lp_build_abs(coord_bld, coord);
 465
 466          /* clamp to length max */
 467          coord = lp_build_min(coord_bld, coord, length_f);
 468          /* subtract 0.5 */
 469          coord = lp_build_sub(coord_bld, coord, half);
 470          /* clamp to [0, length - 0.5] */
 471          coord = lp_build_max(coord_bld, coord, coord_bld->zero);
 472
 473          /* convert to int, compute lerp weight */
 474          lp_build_ifloor_fract(&abs_coord_bld, coord, &coord0, &weight);
 475          coord1 = lp_build_add(int_coord_bld, coord0, int_coord_bld->one);
 476          /* coord1 = min(coord1, length-1) */
 477          coord1 = lp_build_min(int_coord_bld, coord1, length_minus_one);
 478       }
 479       break;
 480
 481    case PIPE_TEX_WRAP_MIRROR_CLAMP_TO_BORDER:
 482       {
 483          if (bld->static_sampler_state->normalized_coords) {
 484             /* scale coord to length */
 485             coord = lp_build_mul(coord_bld, coord, length_f);
 486          }
 487          if (offset) {
 488             offset = lp_build_int_to_float(coord_bld, offset);
 489             coord = lp_build_add(coord_bld, coord, offset);
 490          }
 491          coord = lp_build_abs(coord_bld, coord);
 492
 493          /* was: clamp to [-0.5, length + 0.5] then sub 0.5 */
 494          /* skip clamp - always positive, and other side
 495             only potentially matters for very large coords */
 496          coord = lp_build_sub(coord_bld, coord, half);
 497
 498          /* convert to int, compute lerp weight */
 499          lp_build_ifloor_fract(coord_bld, coord, &coord0, &weight);
 500          coord1 = lp_build_add(int_coord_bld, coord0, int_coord_bld->one);
 501       }
 502       break;
 503
 504    default:
 505       assert(0);
 506       coord0 = NULL;
 507       coord1 = NULL;
 508       weight = NULL;
 509    }
 510
 511    *x0_out = coord0;
 512    *x1_out = coord1;
 513    *weight_out = weight;
 514 }
 515
 516
 517 /**
 518  * Build LLVM code for texture wrap mode for nearest filtering.
 519  * \param coord  the incoming texcoord (nominally in [0,1])
 520  * \param length  the texture size along one dimension, as int vector
 521  * \param length_f  the texture size along one dimension, as float vector
 522  * \param offset  texel offset along one dimension (as int vector)
 523  * \param is_pot  if TRUE, length is a power of two
 524  * \param wrap_mode  one of PIPE_TEX_WRAP_x
 525  */
 526 static LLVMValueRef
 527 lp_build_sample_wrap_nearest(struct lp_build_sample_context *bld,
 528                              LLVMValueRef coord,
 529                              LLVMValueRef length,
 530                              LLVMValueRef length_f,
 531                              LLVMValueRef offset,
 532                              boolean is_pot,
 533                              unsigned wrap_mode)
 534 {
 535    struct lp_build_context *coord_bld = &bld->coord_bld;
 536    struct lp_build_context *int_coord_bld = &bld->int_coord_bld;
 537    LLVMBuilderRef builder = bld->gallivm->builder;
 538    LLVMValueRef length_minus_one = lp_build_sub(int_coord_bld, length, int_coord_bld->one);
 539    LLVMValueRef icoord;
 540
 541    switch(wrap_mode) {
 542    case PIPE_TEX_WRAP_REPEAT:
 543       if (is_pot) {
 544          coord = lp_build_mul(coord_bld, coord, length_f);
 545          icoord = lp_build_ifloor(coord_bld, coord);
 546          if (offset) {
 547             icoord = lp_build_add(int_coord_bld, icoord, offset);
 548          }
 549          icoord = LLVMBuildAnd(builder, icoord, length_minus_one, "");
 550       }
 551       else {
 552           if (offset) {
 553              offset = lp_build_int_to_float(coord_bld, offset);
 554              offset = lp_build_div(coord_bld, offset, length_f);
 555              coord = lp_build_add(coord_bld, coord, offset);
 556           }
 557           /* take fraction, unnormalize */
 558           coord = lp_build_fract_safe(coord_bld, coord);
 559           coord = lp_build_mul(coord_bld, coord, length_f);
 560           icoord = lp_build_itrunc(coord_bld, coord);
 561       }
 562       break;
 563
 564    case PIPE_TEX_WRAP_CLAMP:
 565    case PIPE_TEX_WRAP_CLAMP_TO_EDGE:
 566       if (bld->static_sampler_state->normalized_coords) {
 567          /* scale coord to length */
 568          coord = lp_build_mul(coord_bld, coord, length_f);
 569       }
 570
 571       if (offset) {
 572          offset = lp_build_int_to_float(coord_bld, offset);
 573          coord = lp_build_add(coord_bld, coord, offset);
 574       }
 575       /* floor */
 576       /* use itrunc instead since we clamp to 0 anyway */
 577       icoord = lp_build_itrunc(coord_bld, coord);
 578
 579       /* clamp to [0, length - 1]. */
 580       icoord = lp_build_clamp(int_coord_bld, icoord, int_coord_bld->zero,
 581                               length_minus_one);
 582       break;
 583
 584    case PIPE_TEX_WRAP_CLAMP_TO_BORDER:
 585       if (bld->static_sampler_state->normalized_coords) {
 586          /* scale coord to length */
 587          coord = lp_build_mul(coord_bld, coord, length_f);
 588       }
 589       /* no clamp necessary, border masking will handle this */
 590       icoord = lp_build_ifloor(coord_bld, coord);
 591       if (offset) {
 592          icoord = lp_build_add(int_coord_bld, icoord, offset);
 593       }
 594       break;
 595
 596    case PIPE_TEX_WRAP_MIRROR_REPEAT:
 597       if (offset) {
 598          offset = lp_build_int_to_float(coord_bld, offset);
 599          offset = lp_build_div(coord_bld, offset, length_f);
 600          coord = lp_build_add(coord_bld, coord, offset);
 601       }
 602       /* compute mirror function */
 603       coord = lp_build_coord_mirror(bld, coord);
 604
 605       /* scale coord to length */
 606       assert(bld->static_sampler_state->normalized_coords);
 607       coord = lp_build_mul(coord_bld, coord, length_f);
 608
 609       /* itrunc == ifloor here */
 610       icoord = lp_build_itrunc(coord_bld, coord);
 611
 612       /* clamp to [0, length - 1] */
 613       icoord = lp_build_min(int_coord_bld, icoord, length_minus_one);
 614       break;
 615
 616    case PIPE_TEX_WRAP_MIRROR_CLAMP:
 617    case PIPE_TEX_WRAP_MIRROR_CLAMP_TO_EDGE:
 618       if (bld->static_sampler_state->normalized_coords) {
 619          /* scale coord to length */
 620          coord = lp_build_mul(coord_bld, coord, length_f);
 621       }
 622       if (offset) {
 623          offset = lp_build_int_to_float(coord_bld, offset);
 624          coord = lp_build_add(coord_bld, coord, offset);
 625       }
 626       coord = lp_build_abs(coord_bld, coord);
 627
 628       /* itrunc == ifloor here */
 629       icoord = lp_build_itrunc(coord_bld, coord);
 630
 631       /* clamp to [0, length - 1] */
 632       icoord = lp_build_min(int_coord_bld, icoord, length_minus_one);
 633       break;
 634
 635    case PIPE_TEX_WRAP_MIRROR_CLAMP_TO_BORDER:
 636       if (bld->static_sampler_state->normalized_coords) {
 637          /* scale coord to length */
 638          coord = lp_build_mul(coord_bld, coord, length_f);
 639       }
 640       if (offset) {
 641          offset = lp_build_int_to_float(coord_bld, offset);
 642          coord = lp_build_add(coord_bld, coord, offset);
 643       }
 644       coord = lp_build_abs(coord_bld, coord);
 645
 646       /* itrunc == ifloor here */
 647       icoord = lp_build_itrunc(coord_bld, coord);
 648       break;
 649
 650    default:
 651       assert(0);
 652       icoord = NULL;
 653    }
 654
 655    return icoord;
 656 }
 657
 658
 659 /**
 660  * Do shadow test/comparison.
 661  * \param p shadow ref value
 662  * \param texel  the texel to compare against
 663  */
 664 static LLVMValueRef
 665 lp_build_sample_comparefunc(struct lp_build_sample_context *bld,
 666                             LLVMValueRef p,
 667                             LLVMValueRef texel)
 668 {
 669    struct lp_build_context *texel_bld = &bld->texel_bld;
 670    LLVMValueRef res;
 671
 672    if (0) {
 673       //lp_build_print_value(bld->gallivm, "shadow cmp coord", p);
 674       lp_build_print_value(bld->gallivm, "shadow cmp texel", texel);
 675    }
 676
 677    /* result = (p FUNC texel) ? 1 : 0 */
 678    /*
 679     * honor d3d10 floating point rules here, which state that comparisons
 680     * are ordered except NOT_EQUAL which is unordered.
 681     */
 682    if (bld->static_sampler_state->compare_func != PIPE_FUNC_NOTEQUAL) {
 683       res = lp_build_cmp_ordered(texel_bld, bld->static_sampler_state->compare_func,
 684                                  p, texel);
 685    }
 686    else {
 687       res = lp_build_cmp(texel_bld, bld->static_sampler_state->compare_func,
 688                          p, texel);
 689    }
 690    return res;
 691 }
 692
 693
 694 /**
 695  * Generate code to sample a mipmap level with nearest filtering.
 696  * If sampling a cube texture, r = cube face in [0,5].
 697  */
 698 static void
 699 lp_build_sample_image_nearest(struct lp_build_sample_context *bld,
 700                               LLVMValueRef size,
 701                               LLVMValueRef row_stride_vec,
 702                               LLVMValueRef img_stride_vec,
 703                               LLVMValueRef data_ptr,
 704                               LLVMValueRef mipoffsets,
 705                               LLVMValueRef *coords,
 706                               const LLVMValueRef *offsets,
 707                               LLVMValueRef colors_out[4])
 708 {
 709    const unsigned dims = bld->dims;
 710    LLVMValueRef width_vec;
 711    LLVMValueRef height_vec;
 712    LLVMValueRef depth_vec;
 713    LLVMValueRef flt_size;
 714    LLVMValueRef flt_width_vec;
 715    LLVMValueRef flt_height_vec;
 716    LLVMValueRef flt_depth_vec;
 717    LLVMValueRef x, y = NULL, z = NULL;
 718
 719    lp_build_extract_image_sizes(bld,
 720                                 &bld->int_size_bld,
 721                                 bld->int_coord_type,
 722                                 size,
 723                                 &width_vec, &height_vec, &depth_vec);
 724
 725    flt_size = lp_build_int_to_float(&bld->float_size_bld, size);
 726
 727    lp_build_extract_image_sizes(bld,
 728                                 &bld->float_size_bld,
 729                                 bld->coord_type,
 730                                 flt_size,
 731                                 &flt_width_vec, &flt_height_vec, &flt_depth_vec);
 732
 733    /*
 734     * Compute integer texcoords.
 735     */
 736    x = lp_build_sample_wrap_nearest(bld, coords[0], width_vec,
 737                                     flt_width_vec, offsets[0],
 738                                     bld->static_texture_state->pot_width,
 739                                     bld->static_sampler_state->wrap_s);
 740    lp_build_name(x, "tex.x.wrapped");
 741
 742    if (dims >= 2) {
 743       y = lp_build_sample_wrap_nearest(bld, coords[1], height_vec,
 744                                        flt_height_vec, offsets[1],
 745                                        bld->static_texture_state->pot_height,
 746                                        bld->static_sampler_state->wrap_t);
 747       lp_build_name(y, "tex.y.wrapped");
 748
 749       if (dims == 3) {
 750          z = lp_build_sample_wrap_nearest(bld, coords[2], depth_vec,
 751                                           flt_depth_vec, offsets[2],
 752                                           bld->static_texture_state->pot_depth,
 753                                           bld->static_sampler_state->wrap_r);
 754          lp_build_name(z, "tex.z.wrapped");
 755       }
 756    }
 757    if (has_layer_coord(bld->static_texture_state->target)) {
 758       if (bld->static_texture_state->target == PIPE_TEXTURE_CUBE_ARRAY) {
 759          /* add cube layer to face */
 760          z = lp_build_add(&bld->int_coord_bld, coords[2], coords[3]);
 761       }
 762       else {
 763          z = coords[2];
 764       }
 765       lp_build_name(z, "tex.z.layer");
 766    }
 767
 768    /*
 769     * Get texture colors.
 770     */
 771    lp_build_sample_texel_soa(bld,
 772                              width_vec, height_vec, depth_vec,
 773                              x, y, z,
 774                              row_stride_vec, img_stride_vec,
 775                              data_ptr, mipoffsets, colors_out);
 776
 777    if (bld->static_sampler_state->compare_mode != PIPE_TEX_COMPARE_NONE) {
 778       LLVMValueRef cmpval;
 779       cmpval = lp_build_sample_comparefunc(bld, coords[4], colors_out[0]);
 780       /* this is really just a AND 1.0, cmpval but llvm is clever enough */
 781       colors_out[0] = lp_build_select(&bld->texel_bld, cmpval,
 782                                       bld->texel_bld.one, bld->texel_bld.zero);
 783       colors_out[1] = colors_out[2] = colors_out[3] = colors_out[0];
 784    }
 785
 786 }
 787
 788
 789 /**
 790  * Like a lerp, but inputs are 0/~0 masks, so can simplify slightly.
 791  */
 792 static LLVMValueRef
 793 lp_build_masklerp(struct lp_build_context *bld,
 794                  LLVMValueRef weight,
 795                  LLVMValueRef mask0,
 796                  LLVMValueRef mask1)
 797 {
 798    struct gallivm_state *gallivm = bld->gallivm;
 799    LLVMBuilderRef builder = gallivm->builder;
 800    LLVMValueRef weight2;
 801
 802    weight2 = lp_build_sub(bld, bld->one, weight);
 803    weight = LLVMBuildBitCast(builder, weight,
 804                               lp_build_int_vec_type(gallivm, bld->type), "");
 805    weight2 = LLVMBuildBitCast(builder, weight2,
 806                               lp_build_int_vec_type(gallivm, bld->type), "");
 807    weight = LLVMBuildAnd(builder, weight, mask1, "");
 808    weight2 = LLVMBuildAnd(builder, weight2, mask0, "");
 809    weight = LLVMBuildBitCast(builder, weight, bld->vec_type, "");
 810    weight2 = LLVMBuildBitCast(builder, weight2, bld->vec_type, "");
 811    return lp_build_add(bld, weight, weight2);
 812 }
 813
 814 /**
 815  * Like a 2d lerp, but inputs are 0/~0 masks, so can simplify slightly.
 816  */
 817 static LLVMValueRef
 818 lp_build_masklerp2d(struct lp_build_context *bld,
 819                     LLVMValueRef weight0,
 820                     LLVMValueRef weight1,
 821                     LLVMValueRef mask00,
 822                     LLVMValueRef mask01,
 823                     LLVMValueRef mask10,
 824                     LLVMValueRef mask11)
 825 {
 826    LLVMValueRef val0 = lp_build_masklerp(bld, weight0, mask00, mask01);
 827    LLVMValueRef val1 = lp_build_masklerp(bld, weight0, mask10, mask11);
 828    return lp_build_lerp(bld, weight1, val0, val1, 0);
 829 }
 830
 831 /*
 832  * this is a bit excessive code for something OpenGL just recommends
 833  * but does not require.
 834  */
 835 #define ACCURATE_CUBE_CORNERS 1
 836
 837 /**
 838  * Generate code to sample a mipmap level with linear filtering.
 839  * If sampling a cube texture, r = cube face in [0,5].
 840  * If linear_mask is present, only pixels having their mask set
 841  * will receive linear filtering, the rest will use nearest.
 842  */
 843 static void
 844 lp_build_sample_image_linear(struct lp_build_sample_context *bld,
 845                              boolean is_gather,
 846                              LLVMValueRef size,
 847                              LLVMValueRef linear_mask,
 848                              LLVMValueRef row_stride_vec,
 849                              LLVMValueRef img_stride_vec,
 850                              LLVMValueRef data_ptr,
 851                              LLVMValueRef mipoffsets,
 852                              LLVMValueRef *coords,
 853                              const LLVMValueRef *offsets,
 854                              LLVMValueRef colors_out[4])
 855 {
 856    LLVMBuilderRef builder = bld->gallivm->builder;
 857    struct lp_build_context *ivec_bld = &bld->int_coord_bld;
 858    struct lp_build_context *coord_bld = &bld->coord_bld;
 859    struct lp_build_context *texel_bld = &bld->texel_bld;
 860    const unsigned dims = bld->dims;
 861    LLVMValueRef width_vec;
 862    LLVMValueRef height_vec;
 863    LLVMValueRef depth_vec;
 864    LLVMValueRef flt_size;
 865    LLVMValueRef flt_width_vec;
 866    LLVMValueRef flt_height_vec;
 867    LLVMValueRef flt_depth_vec;
 868    LLVMValueRef fall_off[4], have_corners;
 869    LLVMValueRef z1 = NULL;
 870    LLVMValueRef z00 = NULL, z01 = NULL, z10 = NULL, z11 = NULL;
 871    LLVMValueRef x00 = NULL, x01 = NULL, x10 = NULL, x11 = NULL;
 872    LLVMValueRef y00 = NULL, y01 = NULL, y10 = NULL, y11 = NULL;
 873    LLVMValueRef s_fpart, t_fpart = NULL, r_fpart = NULL;
 874    LLVMValueRef xs[4], ys[4], zs[4];
 875    LLVMValueRef neighbors[2][2][4];
 876    int chan, texel_index;
 877    boolean seamless_cube_filter, accurate_cube_corners;
 878
 879    seamless_cube_filter = (bld->static_texture_state->target == PIPE_TEXTURE_CUBE ||
 880                            bld->static_texture_state->target == PIPE_TEXTURE_CUBE_ARRAY) &&
 881                           bld->static_sampler_state->seamless_cube_map;
 882    /*
 883     * XXX I don't know how this is really supposed to work with gather. From GL
 884     * spec wording (not gather specific) it sounds like the 4th missing texel
 885     * should be an average of the other 3, hence for gather could return this.
 886     * This is however NOT how the code here works, which just fixes up the
 887     * weights used for filtering instead. And of course for gather there is
 888     * no filter to tweak...
 889     */
 890    accurate_cube_corners = ACCURATE_CUBE_CORNERS && seamless_cube_filter &&
 891                            !is_gather;
 892
 893    lp_build_extract_image_sizes(bld,
 894                                 &bld->int_size_bld,
 895                                 bld->int_coord_type,
 896                                 size,
 897                                 &width_vec, &height_vec, &depth_vec);
 898
 899    flt_size = lp_build_int_to_float(&bld->float_size_bld, size);
 900
 901    lp_build_extract_image_sizes(bld,
 902                                 &bld->float_size_bld,
 903                                 bld->coord_type,
 904                                 flt_size,
 905                                 &flt_width_vec, &flt_height_vec, &flt_depth_vec);
 906
 907    /*
 908     * Compute integer texcoords.
 909     */
 910
 911    if (!seamless_cube_filter) {
 912       lp_build_sample_wrap_linear(bld, coords[0], width_vec,
 913                                   flt_width_vec, offsets[0],
 914                                   bld->static_texture_state->pot_width,
 915                                   bld->static_sampler_state->wrap_s,
 916                                   &x00, &x01, &s_fpart);
 917       lp_build_name(x00, "tex.x0.wrapped");
 918       lp_build_name(x01, "tex.x1.wrapped");
 919       x10 = x00;
 920       x11 = x01;
 921
 922       if (dims >= 2) {
 923          lp_build_sample_wrap_linear(bld, coords[1], height_vec,
 924                                      flt_height_vec, offsets[1],
 925                                      bld->static_texture_state->pot_height,
 926                                      bld->static_sampler_state->wrap_t,
 927                                      &y00, &y10, &t_fpart);
 928          lp_build_name(y00, "tex.y0.wrapped");
 929          lp_build_name(y10, "tex.y1.wrapped");
 930          y01 = y00;
 931          y11 = y10;
 932
 933          if (dims == 3) {
 934             lp_build_sample_wrap_linear(bld, coords[2], depth_vec,
 935                                         flt_depth_vec, offsets[2],
 936                                         bld->static_texture_state->pot_depth,
 937                                         bld->static_sampler_state->wrap_r,
 938                                         &z00, &z1, &r_fpart);
 939             z01 = z10 = z11 = z00;
 940             lp_build_name(z00, "tex.z0.wrapped");
 941             lp_build_name(z1, "tex.z1.wrapped");
 942          }
 943       }
 944       if (has_layer_coord(bld->static_texture_state->target)) {
 945          if (bld->static_texture_state->target == PIPE_TEXTURE_CUBE_ARRAY) {
 946             /* add cube layer to face */
 947             z00 = z01 = z10 = z11 = z1 =
 948                lp_build_add(&bld->int_coord_bld, coords[2], coords[3]);
 949          }
 950          else {
 951             z00 = z01 = z10 = z11 = z1 = coords[2];  /* cube face or layer */
 952          }
 953          lp_build_name(z00, "tex.z0.layer");
 954          lp_build_name(z1, "tex.z1.layer");
 955       }
 956    }
 957    else {
 958       struct lp_build_if_state edge_if;
 959       LLVMTypeRef int1t;
 960       LLVMValueRef new_faces[4], new_xcoords[4][2], new_ycoords[4][2];
 961       LLVMValueRef coord, have_edge, have_corner;
 962       LLVMValueRef fall_off_ym_notxm, fall_off_ym_notxp, fall_off_x, fall_off_y;
 963       LLVMValueRef fall_off_yp_notxm, fall_off_yp_notxp;
 964       LLVMValueRef x0, x1, y0, y1, y0_clamped, y1_clamped;
 965       LLVMValueRef face = coords[2];
 966       LLVMValueRef half = lp_build_const_vec(bld->gallivm, coord_bld->type, 0.5f);
 967       LLVMValueRef length_minus_one = lp_build_sub(ivec_bld, width_vec, ivec_bld->one);
 968       /* XXX drop height calcs. Could (should) do this without seamless filtering too */
 969       height_vec = width_vec;
 970       flt_height_vec = flt_width_vec;
 971
 972       /* XXX the overflow logic is actually sort of duplicated with trilinear,
 973        * since an overflow in one mip should also have a corresponding overflow
 974        * in another.
 975        */
 976       /* should always have normalized coords, and offsets are undefined */
 977       assert(bld->static_sampler_state->normalized_coords);
 978       coord = lp_build_mul(coord_bld, coords[0], flt_width_vec);
 979       /* instead of clamp, build mask if overflowed */
 980       coord = lp_build_sub(coord_bld, coord, half);
 981       /* convert to int, compute lerp weight */
 982       /* not ideal with AVX (and no AVX2) */
 983       lp_build_ifloor_fract(coord_bld, coord, &x0, &s_fpart);
 984       x1 = lp_build_add(ivec_bld, x0, ivec_bld->one);
 985       coord = lp_build_mul(coord_bld, coords[1], flt_height_vec);
 986       coord = lp_build_sub(coord_bld, coord, half);
 987       lp_build_ifloor_fract(coord_bld, coord, &y0, &t_fpart);
 988       y1 = lp_build_add(ivec_bld, y0, ivec_bld->one);
 989
 990       fall_off[0] = lp_build_cmp(ivec_bld, PIPE_FUNC_LESS, x0, ivec_bld->zero);
 991       fall_off[1] = lp_build_cmp(ivec_bld, PIPE_FUNC_GREATER, x1, length_minus_one);
 992       fall_off[2] = lp_build_cmp(ivec_bld, PIPE_FUNC_LESS, y0, ivec_bld->zero);
 993       fall_off[3] = lp_build_cmp(ivec_bld, PIPE_FUNC_GREATER, y1, length_minus_one);
 994
 995       fall_off_x = lp_build_or(ivec_bld, fall_off[0], fall_off[1]);
 996       fall_off_y = lp_build_or(ivec_bld, fall_off[2], fall_off[3]);
 997       have_edge = lp_build_or(ivec_bld, fall_off_x, fall_off_y);
 998       have_edge = lp_build_any_true_range(ivec_bld, ivec_bld->type.length, have_edge);
 999
1000       /* needed for accurate corner filtering branch later, rely on 0 init */
1001       int1t = LLVMInt1TypeInContext(bld->gallivm->context);
1002       have_corners = lp_build_alloca(bld->gallivm, int1t, "have_corner");
1003
1004       for (texel_index = 0; texel_index < 4; texel_index++) {
1005          xs[texel_index] = lp_build_alloca(bld->gallivm, ivec_bld->vec_type, "xs");
1006          ys[texel_index] = lp_build_alloca(bld->gallivm, ivec_bld->vec_type, "ys");
1007          zs[texel_index] = lp_build_alloca(bld->gallivm, ivec_bld->vec_type, "zs");
1008       }
1009
1010       lp_build_if(&edge_if, bld->gallivm, have_edge);
1011
1012       have_corner = lp_build_and(ivec_bld, fall_off_x, fall_off_y);
1013       have_corner = lp_build_any_true_range(ivec_bld, ivec_bld->type.length, have_corner);
1014       LLVMBuildStore(builder, have_corner, have_corners);
1015
1016       /*
1017        * Need to feed clamped values here for cheap corner handling,
1018        * but only for y coord (as when falling off both edges we only
1019        * fall off the x one) - this should be sufficient.
1020        */
1021       y0_clamped = lp_build_max(ivec_bld, y0, ivec_bld->zero);
1022       y1_clamped = lp_build_min(ivec_bld, y1, length_minus_one);
1023
1024       /*
1025        * Get all possible new coords.
1026        */
1027       lp_build_cube_new_coords(ivec_bld, face,
1028                                x0, x1, y0_clamped, y1_clamped,
1029                                length_minus_one,
1030                                new_faces, new_xcoords, new_ycoords);
1031
1032       /* handle fall off x-, x+ direction */
1033       /* determine new coords, face (not both fall_off vars can be true at same time) */
1034       x00 = lp_build_select(ivec_bld, fall_off[0], new_xcoords[0][0], x0);
1035       y00 = lp_build_select(ivec_bld, fall_off[0], new_ycoords[0][0], y0_clamped);
1036       x10 = lp_build_select(ivec_bld, fall_off[0], new_xcoords[0][1], x0);
1037       y10 = lp_build_select(ivec_bld, fall_off[0], new_ycoords[0][1], y1_clamped);
1038       x01 = lp_build_select(ivec_bld, fall_off[1], new_xcoords[1][0], x1);
1039       y01 = lp_build_select(ivec_bld, fall_off[1], new_ycoords[1][0], y0_clamped);
1040       x11 = lp_build_select(ivec_bld, fall_off[1], new_xcoords[1][1], x1);
1041       y11 = lp_build_select(ivec_bld, fall_off[1], new_ycoords[1][1], y1_clamped);
1042
1043       z00 = z10 = lp_build_select(ivec_bld, fall_off[0], new_faces[0], face);
1044       z01 = z11 = lp_build_select(ivec_bld, fall_off[1], new_faces[1], face);
1045
1046       /* handle fall off y-, y+ direction */
1047       /*
1048        * Cheap corner logic: just hack up things so a texel doesn't fall
1049        * off both sides (which means filter weights will be wrong but we'll only
1050        * use valid texels in the filter).
1051        * This means however (y) coords must additionally be clamped (see above).
1052        * This corner handling should be fully OpenGL (but not d3d10) compliant.
1053        */
1054       fall_off_ym_notxm = lp_build_andnot(ivec_bld, fall_off[2], fall_off[0]);
1055       fall_off_ym_notxp = lp_build_andnot(ivec_bld, fall_off[2], fall_off[1]);
1056       fall_off_yp_notxm = lp_build_andnot(ivec_bld, fall_off[3], fall_off[0]);
1057       fall_off_yp_notxp = lp_build_andnot(ivec_bld, fall_off[3], fall_off[1]);
1058
1059       x00 = lp_build_select(ivec_bld, fall_off_ym_notxm, new_xcoords[2][0], x00);
1060       y00 = lp_build_select(ivec_bld, fall_off_ym_notxm, new_ycoords[2][0], y00);
1061       x01 = lp_build_select(ivec_bld, fall_off_ym_notxp, new_xcoords[2][1], x01);
1062       y01 = lp_build_select(ivec_bld, fall_off_ym_notxp, new_ycoords[2][1], y01);
1063       x10 = lp_build_select(ivec_bld, fall_off_yp_notxm, new_xcoords[3][0], x10);
1064       y10 = lp_build_select(ivec_bld, fall_off_yp_notxm, new_ycoords[3][0], y10);
1065       x11 = lp_build_select(ivec_bld, fall_off_yp_notxp, new_xcoords[3][1], x11);
1066       y11 = lp_build_select(ivec_bld, fall_off_yp_notxp, new_ycoords[3][1], y11);
1067
1068       z00 = lp_build_select(ivec_bld, fall_off_ym_notxm, new_faces[2], z00);
1069       z01 = lp_build_select(ivec_bld, fall_off_ym_notxp, new_faces[2], z01);
1070       z10 = lp_build_select(ivec_bld, fall_off_yp_notxm, new_faces[3], z10);
1071       z11 = lp_build_select(ivec_bld, fall_off_yp_notxp, new_faces[3], z11);
1072
1073       if (bld->static_texture_state->target == PIPE_TEXTURE_CUBE_ARRAY) {
1074          /* now can add cube layer to face (per sample) */
1075          z00 = lp_build_add(ivec_bld, z00, coords[3]);
1076          z01 = lp_build_add(ivec_bld, z01, coords[3]);
1077          z10 = lp_build_add(ivec_bld, z10, coords[3]);
1078          z11 = lp_build_add(ivec_bld, z11, coords[3]);
1079       }
1080
1081       LLVMBuildStore(builder, x00, xs[0]);
1082       LLVMBuildStore(builder, x01, xs[1]);
1083       LLVMBuildStore(builder, x10, xs[2]);
1084       LLVMBuildStore(builder, x11, xs[3]);
1085       LLVMBuildStore(builder, y00, ys[0]);
1086       LLVMBuildStore(builder, y01, ys[1]);
1087       LLVMBuildStore(builder, y10, ys[2]);
1088       LLVMBuildStore(builder, y11, ys[3]);
1089       LLVMBuildStore(builder, z00, zs[0]);
1090       LLVMBuildStore(builder, z01, zs[1]);
1091       LLVMBuildStore(builder, z10, zs[2]);
1092       LLVMBuildStore(builder, z11, zs[3]);
1093
1094       lp_build_else(&edge_if);
1095
1096       LLVMBuildStore(builder, x0, xs[0]);
1097       LLVMBuildStore(builder, x1, xs[1]);
1098       LLVMBuildStore(builder, x0, xs[2]);
1099       LLVMBuildStore(builder, x1, xs[3]);
1100       LLVMBuildStore(builder, y0, ys[0]);
1101       LLVMBuildStore(builder, y0, ys[1]);
1102       LLVMBuildStore(builder, y1, ys[2]);
1103       LLVMBuildStore(builder, y1, ys[3]);
1104       if (bld->static_texture_state->target == PIPE_TEXTURE_CUBE_ARRAY) {
1105          LLVMValueRef cube_layer = lp_build_add(ivec_bld, face, coords[3]);
1106          LLVMBuildStore(builder, cube_layer, zs[0]);
1107          LLVMBuildStore(builder, cube_layer, zs[1]);
1108          LLVMBuildStore(builder, cube_layer, zs[2]);
1109          LLVMBuildStore(builder, cube_layer, zs[3]);
1110       }
1111       else {
1112          LLVMBuildStore(builder, face, zs[0]);
1113          LLVMBuildStore(builder, face, zs[1]);
1114          LLVMBuildStore(builder, face, zs[2]);
1115          LLVMBuildStore(builder, face, zs[3]);
1116       }
1117
1118       lp_build_endif(&edge_if);
1119
1120       x00 = LLVMBuildLoad(builder, xs[0], "");
1121       x01 = LLVMBuildLoad(builder, xs[1], "");
1122       x10 = LLVMBuildLoad(builder, xs[2], "");
1123       x11 = LLVMBuildLoad(builder, xs[3], "");
1124       y00 = LLVMBuildLoad(builder, ys[0], "");
1125       y01 = LLVMBuildLoad(builder, ys[1], "");
1126       y10 = LLVMBuildLoad(builder, ys[2], "");
1127       y11 = LLVMBuildLoad(builder, ys[3], "");
1128       z00 = LLVMBuildLoad(builder, zs[0], "");
1129       z01 = LLVMBuildLoad(builder, zs[1], "");
1130       z10 = LLVMBuildLoad(builder, zs[2], "");
1131       z11 = LLVMBuildLoad(builder, zs[3], "");
1132    }
1133
1134    if (linear_mask) {
1135       /*
1136        * Whack filter weights into place. Whatever texel had more weight is
1137        * the one which should have been selected by nearest filtering hence
1138        * just use 100% weight for it.
1139        */
1140       struct lp_build_context *c_bld = &bld->coord_bld;
1141       LLVMValueRef w1_mask, w1_weight;
1142       LLVMValueRef half = lp_build_const_vec(bld->gallivm, c_bld->type, 0.5f);
1143
1144       w1_mask = lp_build_cmp(c_bld, PIPE_FUNC_GREATER, s_fpart, half);
1145       /* this select is really just a "and" */
1146       w1_weight = lp_build_select(c_bld, w1_mask, c_bld->one, c_bld->zero);
1147       s_fpart = lp_build_select(c_bld, linear_mask, s_fpart, w1_weight);
1148       if (dims >= 2) {
1149          w1_mask = lp_build_cmp(c_bld, PIPE_FUNC_GREATER, t_fpart, half);
1150          w1_weight = lp_build_select(c_bld, w1_mask, c_bld->one, c_bld->zero);
1151          t_fpart = lp_build_select(c_bld, linear_mask, t_fpart, w1_weight);
1152          if (dims == 3) {
1153             w1_mask = lp_build_cmp(c_bld, PIPE_FUNC_GREATER, r_fpart, half);
1154             w1_weight = lp_build_select(c_bld, w1_mask, c_bld->one, c_bld->zero);
1155             r_fpart = lp_build_select(c_bld, linear_mask, r_fpart, w1_weight);
1156          }
1157       }
1158    }
1159
1160    /*
1161     * Get texture colors.
1162     */
1163    /* get x0/x1 texels */
1164    lp_build_sample_texel_soa(bld,
1165                              width_vec, height_vec, depth_vec,
1166                              x00, y00, z00,
1167                              row_stride_vec, img_stride_vec,
1168                              data_ptr, mipoffsets, neighbors[0][0]);
1169    lp_build_sample_texel_soa(bld,
1170                              width_vec, height_vec, depth_vec,
1171                              x01, y01, z01,
1172                              row_stride_vec, img_stride_vec,
1173                              data_ptr, mipoffsets, neighbors[0][1]);
1174
1175    if (dims == 1) {
1176       assert(!is_gather);
1177       if (bld->static_sampler_state->compare_mode == PIPE_TEX_COMPARE_NONE) {
1178          /* Interpolate two samples from 1D image to produce one color */
1179          for (chan = 0; chan < 4; chan++) {
1180             colors_out[chan] = lp_build_lerp(texel_bld, s_fpart,
1181                                              neighbors[0][0][chan],
1182                                              neighbors[0][1][chan],
1183                                              0);
1184          }
1185       }
1186       else {
1187          LLVMValueRef cmpval0, cmpval1;
1188          cmpval0 = lp_build_sample_comparefunc(bld, coords[4], neighbors[0][0][0]);
1189          cmpval1 = lp_build_sample_comparefunc(bld, coords[4], neighbors[0][1][0]);
1190          /* simplified lerp, AND mask with weight and add */
1191          colors_out[0] = lp_build_masklerp(texel_bld, s_fpart,
1192                                            cmpval0, cmpval1);
1193          colors_out[1] = colors_out[2] = colors_out[3] = colors_out[0];
1194       }
1195    }
1196    else {
1197       /* 2D/3D texture */
1198       struct lp_build_if_state corner_if;
1199       LLVMValueRef colors0[4], colorss[4];
1200
1201       /* get x0/x1 texels at y1 */
1202       lp_build_sample_texel_soa(bld,
1203                                 width_vec, height_vec, depth_vec,
1204                                 x10, y10, z10,
1205                                 row_stride_vec, img_stride_vec,
1206                                 data_ptr, mipoffsets, neighbors[1][0]);
1207       lp_build_sample_texel_soa(bld,
1208                                 width_vec, height_vec, depth_vec,
1209                                 x11, y11, z11,
1210                                 row_stride_vec, img_stride_vec,
1211                                 data_ptr, mipoffsets, neighbors[1][1]);
1212
1213       /*
1214        * To avoid having to duplicate linear_mask / fetch code use
1215        * another branch (with corner condition though edge would work
1216        * as well) here.
1217        */
1218       if (accurate_cube_corners) {
1219          LLVMValueRef w00, w01, w10, w11, wx0, wy0;
1220          LLVMValueRef c_weight, c00, c01, c10, c11;
1221          LLVMValueRef have_corner, one_third, tmp;
1222
1223          colorss[0] = lp_build_alloca(bld->gallivm, coord_bld->vec_type, "cs");
1224          colorss[1] = lp_build_alloca(bld->gallivm, coord_bld->vec_type, "cs");
1225          colorss[2] = lp_build_alloca(bld->gallivm, coord_bld->vec_type, "cs");
1226          colorss[3] = lp_build_alloca(bld->gallivm, coord_bld->vec_type, "cs");
1227
1228          have_corner = LLVMBuildLoad(builder, have_corners, "");
1229
1230          lp_build_if(&corner_if, bld->gallivm, have_corner);
1231
1232          /*
1233           * we can't use standard 2d lerp as we need per-element weight
1234           * in case of corners, so just calculate bilinear result as
1235           * w00*s00 + w01*s01 + w10*s10 + w11*s11.
1236           * (This is actually less work than using 2d lerp, 7 vs. 9 instructions,
1237           * however calculating the weights needs another 6, so actually probably
1238           * not slower than 2d lerp only for 4 channels as weights only need
1239           * to be calculated once - of course fixing the weights has additional cost.)
1240           */
1241          wx0 = lp_build_sub(coord_bld, coord_bld->one, s_fpart);
1242          wy0 = lp_build_sub(coord_bld, coord_bld->one, t_fpart);
1243          w00 = lp_build_mul(coord_bld, wx0, wy0);
1244          w01 = lp_build_mul(coord_bld, s_fpart, wy0);
1245          w10 = lp_build_mul(coord_bld, wx0, t_fpart);
1246          w11 = lp_build_mul(coord_bld, s_fpart, t_fpart);
1247
1248          /* find corner weight */
1249          c00 = lp_build_and(ivec_bld, fall_off[0], fall_off[2]);
1250          c_weight = lp_build_select(coord_bld, c00, w00, coord_bld->zero);
1251          c01 = lp_build_and(ivec_bld, fall_off[1], fall_off[2]);
1252          c_weight = lp_build_select(coord_bld, c01, w01, c_weight);
1253          c10 = lp_build_and(ivec_bld, fall_off[0], fall_off[3]);
1254          c_weight = lp_build_select(coord_bld, c10, w10, c_weight);
1255          c11 = lp_build_and(ivec_bld, fall_off[1], fall_off[3]);
1256          c_weight = lp_build_select(coord_bld, c11, w11, c_weight);
1257
1258          /*
1259           * add 1/3 of the corner weight to each of the 3 other samples
1260           * and null out corner weight
1261           */
1262          one_third = lp_build_const_vec(bld->gallivm, coord_bld->type, 1.0f/3.0f);
1263          c_weight = lp_build_mul(coord_bld, c_weight, one_third);
1264          w00 = lp_build_add(coord_bld, w00, c_weight);
1265          c00 = LLVMBuildBitCast(builder, c00, coord_bld->vec_type, "");
1266          w00 = lp_build_andnot(coord_bld, w00, c00);
1267          w01 = lp_build_add(coord_bld, w01, c_weight);
1268          c01 = LLVMBuildBitCast(builder, c01, coord_bld->vec_type, "");
1269          w01 = lp_build_andnot(coord_bld, w01, c01);
1270          w10 = lp_build_add(coord_bld, w10, c_weight);
1271          c10 = LLVMBuildBitCast(builder, c10, coord_bld->vec_type, "");
1272          w10 = lp_build_andnot(coord_bld, w10, c10);
1273          w11 = lp_build_add(coord_bld, w11, c_weight);
1274          c11 = LLVMBuildBitCast(builder, c11, coord_bld->vec_type, "");
1275          w11 = lp_build_andnot(coord_bld, w11, c11);
1276
1277          if (bld->static_sampler_state->compare_mode == PIPE_TEX_COMPARE_NONE) {
1278             for (chan = 0; chan < 4; chan++) {
1279                colors0[chan] = lp_build_mul(coord_bld, w00, neighbors[0][0][chan]);
1280                tmp = lp_build_mul(coord_bld, w01, neighbors[0][1][chan]);
1281                colors0[chan] = lp_build_add(coord_bld, tmp, colors0[chan]);
1282                tmp = lp_build_mul(coord_bld, w10, neighbors[1][0][chan]);
1283                colors0[chan] = lp_build_add(coord_bld, tmp, colors0[chan]);
1284                tmp = lp_build_mul(coord_bld, w11, neighbors[1][1][chan]);
1285                colors0[chan] = lp_build_add(coord_bld, tmp, colors0[chan]);
1286             }
1287          }
1288          else {
1289             LLVMValueRef cmpval00, cmpval01, cmpval10, cmpval11;
1290             cmpval00 = lp_build_sample_comparefunc(bld, coords[4], neighbors[0][0][0]);
1291             cmpval01 = lp_build_sample_comparefunc(bld, coords[4], neighbors[0][1][0]);
1292             cmpval10 = lp_build_sample_comparefunc(bld, coords[4], neighbors[1][0][0]);
1293             cmpval11 = lp_build_sample_comparefunc(bld, coords[4], neighbors[1][1][0]);
1294             /* inputs to interpolation are just masks so just add masked weights together */
1295             cmpval00 = LLVMBuildBitCast(builder, cmpval00, coord_bld->vec_type, "");
1296             cmpval01 = LLVMBuildBitCast(builder, cmpval01, coord_bld->vec_type, "");
1297             cmpval10 = LLVMBuildBitCast(builder, cmpval10, coord_bld->vec_type, "");
1298             cmpval11 = LLVMBuildBitCast(builder, cmpval11, coord_bld->vec_type, "");
1299             colors0[0] = lp_build_and(coord_bld, w00, cmpval00);
1300             tmp = lp_build_and(coord_bld, w01, cmpval01);
1301             colors0[0] = lp_build_add(coord_bld, tmp, colors0[0]);
1302             tmp = lp_build_and(coord_bld, w10, cmpval10);
1303             colors0[0] = lp_build_add(coord_bld, tmp, colors0[0]);
1304             tmp = lp_build_and(coord_bld, w11, cmpval11);
1305             colors0[0] = lp_build_add(coord_bld, tmp, colors0[0]);
1306             colors0[1] = colors0[2] = colors0[3] = colors0[0];
1307          }
1308
1309          LLVMBuildStore(builder, colors0[0], colorss[0]);
1310          LLVMBuildStore(builder, colors0[1], colorss[1]);
1311          LLVMBuildStore(builder, colors0[2], colorss[2]);
1312          LLVMBuildStore(builder, colors0[3], colorss[3]);
1313
1314          lp_build_else(&corner_if);
1315       }
1316
1317       if (bld->static_sampler_state->compare_mode == PIPE_TEX_COMPARE_NONE) {
1318          if (is_gather) {
1319             /*
1320              * Just assign the red channel (no component selection yet).
1321              * This is a bit hackish, we usually do the swizzle at the
1322              * end of sampling (much less values to swizzle), but this
1323              * obviously cannot work when using gather.
1324              */
1325             unsigned chan_swiz = bld->static_texture_state->swizzle_r;
1326             colors0[0] = lp_build_swizzle_soa_channel(texel_bld,
1327                                                       neighbors[1][0],
1328                                                       chan_swiz);
1329             colors0[1] = lp_build_swizzle_soa_channel(texel_bld,
1330                                                       neighbors[1][1],
1331                                                       chan_swiz);
1332             colors0[2] = lp_build_swizzle_soa_channel(texel_bld,
1333                                                       neighbors[0][1],
1334                                                       chan_swiz);
1335             colors0[3] = lp_build_swizzle_soa_channel(texel_bld,
1336                                                       neighbors[0][0],
1337                                                       chan_swiz);
1338          }
1339          else {
1340             /* Bilinear interpolate the four samples from the 2D image / 3D slice */
1341             for (chan = 0; chan < 4; chan++) {
1342                colors0[chan] = lp_build_lerp_2d(texel_bld,
1343                                                 s_fpart, t_fpart,
1344                                                 neighbors[0][0][chan],
1345                                                 neighbors[0][1][chan],
1346                                                 neighbors[1][0][chan],
1347                                                 neighbors[1][1][chan],
1348                                                 0);
1349             }
1350          }
1351       }
1352       else {
1353          LLVMValueRef cmpval00, cmpval01, cmpval10, cmpval11;
1354          cmpval00 = lp_build_sample_comparefunc(bld, coords[4], neighbors[0][0][0]);
1355          cmpval01 = lp_build_sample_comparefunc(bld, coords[4], neighbors[0][1][0]);
1356          cmpval10 = lp_build_sample_comparefunc(bld, coords[4], neighbors[1][0][0]);
1357          cmpval11 = lp_build_sample_comparefunc(bld, coords[4], neighbors[1][1][0]);
1358
1359          if (is_gather) {
1360             /* more hacks for swizzling, should be X, ONE or ZERO... */
1361             unsigned chan_swiz = bld->static_texture_state->swizzle_r;
1362             if (chan_swiz <= PIPE_SWIZZLE_ALPHA) {
1363                colors0[0] = lp_build_select(texel_bld, cmpval10,
1364                                             texel_bld->one, texel_bld->zero);
1365                colors0[1] = lp_build_select(texel_bld, cmpval11,
1366                                             texel_bld->one, texel_bld->zero);
1367                colors0[2] = lp_build_select(texel_bld, cmpval01,
1368                                             texel_bld->one, texel_bld->zero);
1369                colors0[3] = lp_build_select(texel_bld, cmpval00,
1370                                             texel_bld->one, texel_bld->zero);
1371             }
1372             else if (chan_swiz == PIPE_SWIZZLE_ZERO) {
1373                colors0[0] = colors0[1] = colors0[2] = colors0[3] =
1374                             texel_bld->zero;
1375             }
1376             else {
1377                colors0[0] = colors0[1] = colors0[2] = colors0[3] =
1378                             texel_bld->one;
1379             }
1380          }
1381          else {
1382             colors0[0] = lp_build_masklerp2d(texel_bld, s_fpart, t_fpart,
1383                                              cmpval00, cmpval01, cmpval10, cmpval11);
1384             colors0[1] = colors0[2] = colors0[3] = colors0[0];
1385          }
1386       }
1387
1388       if (accurate_cube_corners) {
1389          LLVMBuildStore(builder, colors0[0], colorss[0]);
1390          LLVMBuildStore(builder, colors0[1], colorss[1]);
1391          LLVMBuildStore(builder, colors0[2], colorss[2]);
1392          LLVMBuildStore(builder, colors0[3], colorss[3]);
1393
1394          lp_build_endif(&corner_if);
1395
1396          colors0[0] = LLVMBuildLoad(builder, colorss[0], "");
1397          colors0[1] = LLVMBuildLoad(builder, colorss[1], "");
1398          colors0[2] = LLVMBuildLoad(builder, colorss[2], "");
1399          colors0[3] = LLVMBuildLoad(builder, colorss[3], "");
1400       }
1401
1402       if (dims == 3) {
1403          LLVMValueRef neighbors1[2][2][4];
1404          LLVMValueRef colors1[4];
1405
1406          assert(!is_gather);
1407
1408          /* get x0/x1/y0/y1 texels at z1 */
1409          lp_build_sample_texel_soa(bld,
1410                                    width_vec, height_vec, depth_vec,
1411                                    x00, y00, z1,
1412                                    row_stride_vec, img_stride_vec,
1413                                    data_ptr, mipoffsets, neighbors1[0][0]);
1414          lp_build_sample_texel_soa(bld,
1415                                    width_vec, height_vec, depth_vec,
1416                                    x01, y01, z1,
1417                                    row_stride_vec, img_stride_vec,
1418                                    data_ptr, mipoffsets, neighbors1[0][1]);
1419          lp_build_sample_texel_soa(bld,
1420                                    width_vec, height_vec, depth_vec,
1421                                    x10, y10, z1,
1422                                    row_stride_vec, img_stride_vec,
1423                                    data_ptr, mipoffsets, neighbors1[1][0]);
1424          lp_build_sample_texel_soa(bld,
1425                                    width_vec, height_vec, depth_vec,
1426                                    x11, y11, z1,
1427                                    row_stride_vec, img_stride_vec,
1428                                    data_ptr, mipoffsets, neighbors1[1][1]);
1429
1430          if (bld->static_sampler_state->compare_mode == PIPE_TEX_COMPARE_NONE) {
1431             /* Bilinear interpolate the four samples from the second Z slice */
1432             for (chan = 0; chan < 4; chan++) {
1433                colors1[chan] = lp_build_lerp_2d(texel_bld,
1434                                                 s_fpart, t_fpart,
1435                                                 neighbors1[0][0][chan],
1436                                                 neighbors1[0][1][chan],
1437                                                 neighbors1[1][0][chan],
1438                                                 neighbors1[1][1][chan],
1439                                                 0);
1440             }
1441             /* Linearly interpolate the two samples from the two 3D slices */
1442             for (chan = 0; chan < 4; chan++) {
1443                colors_out[chan] = lp_build_lerp(texel_bld,
1444                                                 r_fpart,
1445                                                 colors0[chan], colors1[chan],
1446                                                 0);
1447             }
1448          }
1449          else {
1450             LLVMValueRef cmpval00, cmpval01, cmpval10, cmpval11;
1451             cmpval00 = lp_build_sample_comparefunc(bld, coords[4], neighbors[0][0][0]);
1452             cmpval01 = lp_build_sample_comparefunc(bld, coords[4], neighbors[0][1][0]);
1453             cmpval10 = lp_build_sample_comparefunc(bld, coords[4], neighbors[1][0][0]);
1454             cmpval11 = lp_build_sample_comparefunc(bld, coords[4], neighbors[1][1][0]);
1455             colors1[0] = lp_build_masklerp2d(texel_bld, s_fpart, t_fpart,
1456                                              cmpval00, cmpval01, cmpval10, cmpval11);
1457             /* Linearly interpolate the two samples from the two 3D slices */
1458             colors_out[0] = lp_build_lerp(texel_bld,
1459                                           r_fpart,
1460                                           colors0[0], colors1[0],
1461                                           0);
1462             colors_out[1] = colors_out[2] = colors_out[3] = colors_out[0];
1463          }
1464       }
1465       else {
1466          /* 2D tex */
1467          for (chan = 0; chan < 4; chan++) {
1468             colors_out[chan] = colors0[chan];
1469          }
1470       }
1471    }
1472 }
1473
1474
1475 /**
1476  * Sample the texture/mipmap using given image filter and mip filter.
1477  * ilevel0 and ilevel1 indicate the two mipmap levels to sample
1478  * from (vectors or scalars).
1479  * If we're using nearest miplevel sampling the '1' values will be null/unused.
1480  */
1481 static void
1482 lp_build_sample_mipmap(struct lp_build_sample_context *bld,
1483                        unsigned img_filter,
1484                        unsigned mip_filter,
1485                        boolean is_gather,
1486                        LLVMValueRef *coords,
1487                        const LLVMValueRef *offsets,
1488                        LLVMValueRef ilevel0,
1489                        LLVMValueRef ilevel1,
1490                        LLVMValueRef lod_fpart,
1491                        LLVMValueRef *colors_out)
1492 {
1493    LLVMBuilderRef builder = bld->gallivm->builder;
1494    LLVMValueRef size0 = NULL;
1495    LLVMValueRef size1 = NULL;
1496    LLVMValueRef row_stride0_vec = NULL;
1497    LLVMValueRef row_stride1_vec = NULL;
1498    LLVMValueRef img_stride0_vec = NULL;
1499    LLVMValueRef img_stride1_vec = NULL;
1500    LLVMValueRef data_ptr0 = NULL;
1501    LLVMValueRef data_ptr1 = NULL;
1502    LLVMValueRef mipoff0 = NULL;
1503    LLVMValueRef mipoff1 = NULL;
1504    LLVMValueRef colors0[4], colors1[4];
1505    unsigned chan;
1506
1507    /* sample the first mipmap level */
1508    lp_build_mipmap_level_sizes(bld, ilevel0,
1509                                &size0,
1510                                &row_stride0_vec, &img_stride0_vec);
1511    if (bld->num_mips == 1) {
1512       data_ptr0 = lp_build_get_mipmap_level(bld, ilevel0);
1513    }
1514    else {
1515       /* This path should work for num_lods 1 too but slightly less efficient */
1516       data_ptr0 = bld->base_ptr;
1517       mipoff0 = lp_build_get_mip_offsets(bld, ilevel0);
1518    }
1519    if (img_filter == PIPE_TEX_FILTER_NEAREST) {
1520       lp_build_sample_image_nearest(bld, size0,
1521                                     row_stride0_vec, img_stride0_vec,
1522                                     data_ptr0, mipoff0, coords, offsets,
1523                                     colors0);
1524    }
1525    else {
1526       assert(img_filter == PIPE_TEX_FILTER_LINEAR);
1527       lp_build_sample_image_linear(bld, is_gather, size0, NULL,
1528                                    row_stride0_vec, img_stride0_vec,
1529                                    data_ptr0, mipoff0, coords, offsets,
1530                                    colors0);
1531    }
1532
1533    /* Store the first level's colors in the output variables */
1534    for (chan = 0; chan < 4; chan++) {
1535        LLVMBuildStore(builder, colors0[chan], colors_out[chan]);
1536    }
1537
1538    if (mip_filter == PIPE_TEX_MIPFILTER_LINEAR) {
1539       struct lp_build_if_state if_ctx;
1540       LLVMValueRef need_lerp;
1541
1542       /* need_lerp = lod_fpart > 0 */
1543       if (bld->num_lods == 1) {
1544          need_lerp = LLVMBuildFCmp(builder, LLVMRealUGT,
1545                                    lod_fpart, bld->lodf_bld.zero,
1546                                    "need_lerp");
1547       }
1548       else {
1549          /*
1550           * We'll do mip filtering if any of the quads (or individual
1551           * pixel in case of per-pixel lod) need it.
1552           * It might be better to split the vectors here and only fetch/filter
1553           * quads which need it (if there's one lod per quad).
1554           */
1555          need_lerp = lp_build_compare(bld->gallivm, bld->lodf_bld.type,
1556                                       PIPE_FUNC_GREATER,
1557                                       lod_fpart, bld->lodf_bld.zero);
1558          need_lerp = lp_build_any_true_range(&bld->lodi_bld, bld->num_lods, need_lerp);
1559       }
1560
1561       lp_build_if(&if_ctx, bld->gallivm, need_lerp);
1562       {
1563          /*
1564           * We unfortunately need to clamp lod_fpart here since we can get
1565           * negative values which would screw up filtering if not all
1566           * lod_fpart values have same sign.
1567           */
1568          lod_fpart = lp_build_max(&bld->lodf_bld, lod_fpart,
1569                                   bld->lodf_bld.zero);
1570          /* sample the second mipmap level */
1571          lp_build_mipmap_level_sizes(bld, ilevel1,
1572                                      &size1,
1573                                      &row_stride1_vec, &img_stride1_vec);
1574          if (bld->num_mips == 1) {
1575             data_ptr1 = lp_build_get_mipmap_level(bld, ilevel1);
1576          }
1577          else {
1578             data_ptr1 = bld->base_ptr;
1579             mipoff1 = lp_build_get_mip_offsets(bld, ilevel1);
1580          }
1581          if (img_filter == PIPE_TEX_FILTER_NEAREST) {
1582             lp_build_sample_image_nearest(bld, size1,
1583                                           row_stride1_vec, img_stride1_vec,
1584                                           data_ptr1, mipoff1, coords, offsets,
1585                                           colors1);
1586          }
1587          else {
1588             lp_build_sample_image_linear(bld, FALSE, size1, NULL,
1589                                          row_stride1_vec, img_stride1_vec,
1590                                          data_ptr1, mipoff1, coords, offsets,
1591                                          colors1);
1592          }
1593
1594          /* interpolate samples from the two mipmap levels */
1595
1596          if (bld->num_lods != bld->coord_type.length)
1597             lod_fpart = lp_build_unpack_broadcast_aos_scalars(bld->gallivm,
1598                                                               bld->lodf_bld.type,
1599                                                               bld->texel_bld.type,
1600                                                               lod_fpart);
1601
1602          for (chan = 0; chan < 4; chan++) {
1603             colors0[chan] = lp_build_lerp(&bld->texel_bld, lod_fpart,
1604                                           colors0[chan], colors1[chan],
1605                                           0);
1606             LLVMBuildStore(builder, colors0[chan], colors_out[chan]);
1607          }
1608       }
1609       lp_build_endif(&if_ctx);
1610    }
1611 }
1612
1613
1614 /**
1615  * Sample the texture/mipmap using given mip filter, and using
1616  * both nearest and linear filtering at the same time depending
1617  * on linear_mask.
1618  * lod can be per quad but linear_mask is always per pixel.
1619  * ilevel0 and ilevel1 indicate the two mipmap levels to sample
1620  * from (vectors or scalars).
1621  * If we're using nearest miplevel sampling the '1' values will be null/unused.
1622  */
1623 static void
1624 lp_build_sample_mipmap_both(struct lp_build_sample_context *bld,
1625                             LLVMValueRef linear_mask,
1626                             unsigned mip_filter,
1627                             LLVMValueRef *coords,
1628                             const LLVMValueRef *offsets,
1629                             LLVMValueRef ilevel0,
1630                             LLVMValueRef ilevel1,
1631                             LLVMValueRef lod_fpart,
1632                             LLVMValueRef lod_positive,
1633                             LLVMValueRef *colors_out)
1634 {
1635    LLVMBuilderRef builder = bld->gallivm->builder;
1636    LLVMValueRef size0 = NULL;
1637    LLVMValueRef size1 = NULL;
1638    LLVMValueRef row_stride0_vec = NULL;
1639    LLVMValueRef row_stride1_vec = NULL;
1640    LLVMValueRef img_stride0_vec = NULL;
1641    LLVMValueRef img_stride1_vec = NULL;
1642    LLVMValueRef data_ptr0 = NULL;
1643    LLVMValueRef data_ptr1 = NULL;
1644    LLVMValueRef mipoff0 = NULL;
1645    LLVMValueRef mipoff1 = NULL;
1646    LLVMValueRef colors0[4], colors1[4];
1647    unsigned chan;
1648
1649    /* sample the first mipmap level */
1650    lp_build_mipmap_level_sizes(bld, ilevel0,
1651                                &size0,
1652                                &row_stride0_vec, &img_stride0_vec);
1653    if (bld->num_mips == 1) {
1654       data_ptr0 = lp_build_get_mipmap_level(bld, ilevel0);
1655    }
1656    else {
1657       /* This path should work for num_lods 1 too but slightly less efficient */
1658       data_ptr0 = bld->base_ptr;
1659       mipoff0 = lp_build_get_mip_offsets(bld, ilevel0);
1660    }
1661
1662    lp_build_sample_image_linear(bld, FALSE, size0, linear_mask,
1663                                 row_stride0_vec, img_stride0_vec,
1664                                 data_ptr0, mipoff0, coords, offsets,
1665                                 colors0);
1666
1667    /* Store the first level's colors in the output variables */
1668    for (chan = 0; chan < 4; chan++) {
1669        LLVMBuildStore(builder, colors0[chan], colors_out[chan]);
1670    }
1671
1672    if (mip_filter == PIPE_TEX_MIPFILTER_LINEAR) {
1673       struct lp_build_if_state if_ctx;
1674       LLVMValueRef need_lerp;
1675
1676       /*
1677        * We'll do mip filtering if any of the quads (or individual
1678        * pixel in case of per-pixel lod) need it.
1679        * Note using lod_positive here not lod_fpart since it may be the same
1680        * condition as that used in the outer "if" in the caller hence llvm
1681        * should be able to merge the branches in this case.
1682        */
1683       need_lerp = lp_build_any_true_range(&bld->lodi_bld, bld->num_lods, lod_positive);
1684
1685       lp_build_if(&if_ctx, bld->gallivm, need_lerp);
1686       {
1687          /*
1688           * We unfortunately need to clamp lod_fpart here since we can get
1689           * negative values which would screw up filtering if not all
1690           * lod_fpart values have same sign.
1691           */
1692          lod_fpart = lp_build_max(&bld->lodf_bld, lod_fpart,
1693                                   bld->lodf_bld.zero);
1694          /* sample the second mipmap level */
1695          lp_build_mipmap_level_sizes(bld, ilevel1,
1696                                      &size1,
1697                                      &row_stride1_vec, &img_stride1_vec);
1698          if (bld->num_mips == 1) {
1699             data_ptr1 = lp_build_get_mipmap_level(bld, ilevel1);
1700          }
1701          else {
1702             data_ptr1 = bld->base_ptr;
1703             mipoff1 = lp_build_get_mip_offsets(bld, ilevel1);
1704          }
1705
1706          lp_build_sample_image_linear(bld, FALSE, size1, linear_mask,
1707                                       row_stride1_vec, img_stride1_vec,
1708                                       data_ptr1, mipoff1, coords, offsets,
1709                                       colors1);
1710
1711          /* interpolate samples from the two mipmap levels */
1712
1713          if (bld->num_lods != bld->coord_type.length)
1714             lod_fpart = lp_build_unpack_broadcast_aos_scalars(bld->gallivm,
1715                                                               bld->lodf_bld.type,
1716                                                               bld->texel_bld.type,
1717                                                               lod_fpart);
1718
1719          for (chan = 0; chan < 4; chan++) {
1720             colors0[chan] = lp_build_lerp(&bld->texel_bld, lod_fpart,
1721                                           colors0[chan], colors1[chan],
1722                                           0);
1723             LLVMBuildStore(builder, colors0[chan], colors_out[chan]);
1724          }
1725       }
1726       lp_build_endif(&if_ctx);
1727    }
1728 }
1729
1730
1731 /**
1732  * Build (per-coord) layer value.
1733  * Either clamp layer to valid values or fill in optional out_of_bounds
1734  * value and just return value unclamped.
1735  */
1736 static LLVMValueRef
1737 lp_build_layer_coord(struct lp_build_sample_context *bld,
1738                      unsigned texture_unit,
1739                      boolean is_cube_array,
1740                      LLVMValueRef layer,
1741                      LLVMValueRef *out_of_bounds)
1742 {
1743    LLVMValueRef num_layers;
1744    struct lp_build_context *int_coord_bld = &bld->int_coord_bld;
1745
1746    num_layers = bld->dynamic_state->depth(bld->dynamic_state, bld->gallivm,
1747                                           bld->context_ptr, texture_unit);
1748
1749    if (out_of_bounds) {
1750       LLVMValueRef out1, out;
1751       assert(!is_cube_array);
1752       num_layers = lp_build_broadcast_scalar(int_coord_bld, num_layers);
1753       out = lp_build_cmp(int_coord_bld, PIPE_FUNC_LESS, layer, int_coord_bld->zero);
1754       out1 = lp_build_cmp(int_coord_bld, PIPE_FUNC_GEQUAL, layer, num_layers);
1755       *out_of_bounds = lp_build_or(int_coord_bld, out, out1);
1756       return layer;
1757    }
1758    else {
1759       LLVMValueRef maxlayer;
1760       LLVMValueRef s = is_cube_array ? lp_build_const_int32(bld->gallivm, 6) :
1761                                        bld->int_bld.one;
1762       maxlayer = lp_build_sub(&bld->int_bld, num_layers, s);
1763       maxlayer = lp_build_broadcast_scalar(int_coord_bld, maxlayer);
1764       return lp_build_clamp(int_coord_bld, layer, int_coord_bld->zero, maxlayer);
1765    }
1766 }
1767
1768
1769 /**
1770  * Calculate cube face, lod, mip levels.
1771  */
1772 static void
1773 lp_build_sample_common(struct lp_build_sample_context *bld,
1774                        unsigned texture_index,
1775                        unsigned sampler_index,
1776                        LLVMValueRef *coords,
1777                        const struct lp_derivatives *derivs, /* optional */
1778                        LLVMValueRef lod_bias, /* optional */
1779                        LLVMValueRef explicit_lod, /* optional */
1780                        LLVMValueRef *lod_pos_or_zero,
1781                        LLVMValueRef *lod_fpart,
1782                        LLVMValueRef *ilevel0,
1783                        LLVMValueRef *ilevel1)
1784 {
1785    const unsigned mip_filter = bld->static_sampler_state->min_mip_filter;
1786    const unsigned min_filter = bld->static_sampler_state->min_img_filter;
1787    const unsigned mag_filter = bld->static_sampler_state->mag_img_filter;
1788    const unsigned target = bld->static_texture_state->target;
1789    LLVMValueRef first_level, cube_rho = NULL;
1790    LLVMValueRef lod_ipart = NULL;
1791    struct lp_derivatives cube_derivs;
1792
1793    /*
1794    printf("%s mip %d  min %d  mag %d\n", __FUNCTION__,
1795           mip_filter, min_filter, mag_filter);
1796    */
1797
1798    /*
1799     * Choose cube face, recompute texcoords for the chosen face and
1800     * compute rho here too (as it requires transform of derivatives).
1801     */
1802    if (target == PIPE_TEXTURE_CUBE || target == PIPE_TEXTURE_CUBE_ARRAY) {
1803       boolean need_derivs;
1804       need_derivs = ((min_filter != mag_filter ||
1805                       mip_filter != PIPE_TEX_MIPFILTER_NONE) &&
1806                       !bld->static_sampler_state->min_max_lod_equal &&
1807                       !explicit_lod);
1808       lp_build_cube_lookup(bld, coords, derivs, &cube_rho, &cube_derivs, need_derivs);
1809       derivs = &cube_derivs;
1810       if (target == PIPE_TEXTURE_CUBE_ARRAY) {
1811          /* calculate cube layer coord now */
1812          LLVMValueRef layer = lp_build_iround(&bld->coord_bld, coords[3]);
1813          LLVMValueRef six = lp_build_const_int_vec(bld->gallivm, bld->int_coord_type, 6);
1814          layer = lp_build_mul(&bld->int_coord_bld, layer, six);
1815          coords[3] = lp_build_layer_coord(bld, texture_index, TRUE, layer, NULL);
1816          /* because of seamless filtering can't add it to face (coords[2]) here. */
1817       }
1818    }
1819    else if (target == PIPE_TEXTURE_1D_ARRAY ||
1820             target == PIPE_TEXTURE_2D_ARRAY) {
1821       coords[2] = lp_build_iround(&bld->coord_bld, coords[2]);
1822       coords[2] = lp_build_layer_coord(bld, texture_index, FALSE, coords[2], NULL);
1823    }
1824
1825    if (bld->static_sampler_state->compare_mode != PIPE_TEX_COMPARE_NONE) {
1826       /*
1827        * Clamp p coords to [0,1] for fixed function depth texture format here.
1828        * Technically this is not entirely correct for unorm depth as the ref value
1829        * should be converted to the depth format (quantization!) and comparison
1830        * then done in texture format. This would actually help performance (since
1831        * only need to do it once and could save the per-sample conversion of texels
1832        * to floats instead), but it would need more messy code (would need to push
1833        * at least some bits down to actual fetch so conversion could be skipped,
1834        * and would have ugly interaction with border color, would need to convert
1835        * border color to that format too or do some other tricks to make it work).
1836        */
1837       const struct util_format_description *format_desc = bld->format_desc;
1838       unsigned chan_type;
1839       /* not entirely sure we couldn't end up with non-valid swizzle here */
1840       chan_type = format_desc->swizzle[0] <= UTIL_FORMAT_SWIZZLE_W ?
1841                      format_desc->channel[format_desc->swizzle[0]].type :
1842                      UTIL_FORMAT_TYPE_FLOAT;
1843       if (chan_type != UTIL_FORMAT_TYPE_FLOAT) {
1844          coords[4] = lp_build_clamp(&bld->coord_bld, coords[4],
1845                                     bld->coord_bld.zero, bld->coord_bld.one);
1846       }
1847    }
1848
1849    /*
1850     * Compute the level of detail (float).
1851     */
1852    if (min_filter != mag_filter ||
1853        mip_filter != PIPE_TEX_MIPFILTER_NONE) {
1854       /* Need to compute lod either to choose mipmap levels or to
1855        * distinguish between minification/magnification with one mipmap level.
1856        */
1857       lp_build_lod_selector(bld, texture_index, sampler_index,
1858                             coords[0], coords[1], coords[2], cube_rho,
1859                             derivs, lod_bias, explicit_lod,
1860                             mip_filter,
1861                             &lod_ipart, lod_fpart, lod_pos_or_zero);
1862    } else {
1863       lod_ipart = bld->lodi_bld.zero;
1864       *lod_pos_or_zero = bld->lodi_bld.zero;
1865    }
1866
1867    if (bld->num_lods != bld->num_mips) {
1868       /* only makes sense if there's just a single mip level */
1869       assert(bld->num_mips == 1);
1870       lod_ipart = lp_build_extract_range(bld->gallivm, lod_ipart, 0, 1);
1871    }
1872
1873    /*
1874     * Compute integer mipmap level(s) to fetch texels from: ilevel0, ilevel1
1875     */
1876    switch (mip_filter) {
1877    default:
1878       assert(0 && "bad mip_filter value in lp_build_sample_soa()");
1879       /* fall-through */
1880    case PIPE_TEX_MIPFILTER_NONE:
1881       /* always use mip level 0 */
1882       first_level = bld->dynamic_state->first_level(bld->dynamic_state,
1883                                                     bld->gallivm, bld->context_ptr,
1884                                                     texture_index);
1885       first_level = lp_build_broadcast_scalar(&bld->leveli_bld, first_level);
1886       *ilevel0 = first_level;
1887       break;
1888    case PIPE_TEX_MIPFILTER_NEAREST:
1889       assert(lod_ipart);
1890       lp_build_nearest_mip_level(bld, texture_index, lod_ipart, ilevel0, NULL);
1891       break;
1892    case PIPE_TEX_MIPFILTER_LINEAR:
1893       assert(lod_ipart);
1894       assert(*lod_fpart);
1895       lp_build_linear_mip_levels(bld, texture_index,
1896                                  lod_ipart, lod_fpart,
1897                                  ilevel0, ilevel1);
1898       break;
1899    }
1900 }
1901
1902 static void
1903 lp_build_clamp_border_color(struct lp_build_sample_context *bld,
1904                             unsigned sampler_unit)
1905 {
1906    struct gallivm_state *gallivm = bld->gallivm;
1907    LLVMBuilderRef builder = gallivm->builder;
1908    LLVMValueRef border_color_ptr =
1909       bld->dynamic_state->border_color(bld->dynamic_state, gallivm,
1910                                        bld->context_ptr, sampler_unit);
1911    LLVMValueRef border_color;
1912    const struct util_format_description *format_desc = bld->format_desc;
1913    struct lp_type vec4_type = bld->texel_type;
1914    struct lp_build_context vec4_bld;
1915    LLVMValueRef min_clamp = NULL;
1916    LLVMValueRef max_clamp = NULL;
1917
1918    /*
1919     * For normalized format need to clamp border color (technically
1920     * probably should also quantize the data). Really sucks doing this
1921     * here but can't avoid at least for now since this is part of
1922     * sampler state and texture format is part of sampler_view state.
1923     * GL expects also expects clamping for uint/sint formats too so
1924     * do that as well (d3d10 can't end up here with uint/sint since it
1925     * only supports them with ld).
1926     */
1927    vec4_type.length = 4;
1928    lp_build_context_init(&vec4_bld, gallivm, vec4_type);
1929
1930    /*
1931     * Vectorized clamping of border color. Loading is a bit of a hack since
1932     * we just cast the pointer to float array to pointer to vec4
1933     * (int or float).
1934     */
1935    border_color_ptr = lp_build_array_get_ptr(gallivm, border_color_ptr,
1936                                              lp_build_const_int32(gallivm, 0));
1937    border_color_ptr = LLVMBuildBitCast(builder, border_color_ptr,
1938                                        LLVMPointerType(vec4_bld.vec_type, 0), "");
1939    border_color = LLVMBuildLoad(builder, border_color_ptr, "");
1940    /* we don't have aligned type in the dynamic state unfortunately */
1941    lp_set_load_alignment(border_color, 4);
1942
1943    /*
1944     * Instead of having some incredibly complex logic which will try to figure out
1945     * clamping necessary for each channel, simply use the first channel, and treat
1946     * mixed signed/unsigned normalized formats specially.
1947     * (Mixed non-normalized, which wouldn't work at all here, do not exist for a
1948     * good reason.)
1949     */
1950    if (format_desc->layout == UTIL_FORMAT_LAYOUT_PLAIN) {
1951       int chan;
1952       /* d/s needs special handling because both present means just sampling depth */
1953       if (util_format_is_depth_and_stencil(format_desc->format)) {
1954          chan = format_desc->swizzle[0];
1955       }
1956       else {
1957          chan = util_format_get_first_non_void_channel(format_desc->format);
1958       }
1959       if (chan >= 0 && chan <= UTIL_FORMAT_SWIZZLE_W) {
1960          unsigned chan_type = format_desc->channel[chan].type;
1961          unsigned chan_norm = format_desc->channel[chan].normalized;
1962          unsigned chan_pure = format_desc->channel[chan].pure_integer;
1963          if (chan_type == UTIL_FORMAT_TYPE_SIGNED) {
1964             if (chan_norm) {
1965                min_clamp = lp_build_const_vec(gallivm, vec4_type, -1.0F);
1966                max_clamp = vec4_bld.one;
1967             }
1968             else if (chan_pure) {
1969                /*
1970                 * Border color was stored as int, hence need min/max clamp
1971                 * only if chan has less than 32 bits..
1972                 */
1973                unsigned chan_size = format_desc->channel[chan].size;
1974                if (chan_size < 32) {
1975                   min_clamp = lp_build_const_int_vec(gallivm, vec4_type,
1976                                                      0 - (1 << (chan_size - 1)));
1977                   max_clamp = lp_build_const_int_vec(gallivm, vec4_type,
1978                                                      (1 << (chan_size - 1)) - 1);
1979                }
1980             }
1981             /* TODO: no idea about non-pure, non-normalized! */
1982          }
1983          else if (chan_type == UTIL_FORMAT_TYPE_UNSIGNED) {
1984             if (chan_norm) {
1985                min_clamp = vec4_bld.zero;
1986                max_clamp = vec4_bld.one;
1987             }
1988             /*
1989              * Need a ugly hack here, because we don't have Z32_FLOAT_X8X24
1990              * we use Z32_FLOAT_S8X24 to imply sampling depth component
1991              * and ignoring stencil, which will blow up here if we try to
1992              * do a uint clamp in a float texel build...
1993              * And even if we had that format, mesa st also thinks using z24s8
1994              * means depth sampling ignoring stencil.
1995              */
1996             else if (chan_pure) {
1997                /*
1998                 * Border color was stored as uint, hence never need min
1999                 * clamp, and only need max clamp if chan has less than 32 bits.
2000                 */
2001                unsigned chan_size = format_desc->channel[chan].size;
2002                if (chan_size < 32) {
2003                   max_clamp = lp_build_const_int_vec(gallivm, vec4_type,
2004                                                      (1 << chan_size) - 1);
2005                }
2006                /* TODO: no idea about non-pure, non-normalized! */
2007             }
2008          }
2009          else if (chan_type == UTIL_FORMAT_TYPE_FIXED) {
2010             /* TODO: I have no idea what clamp this would need if any! */
2011          }
2012       }
2013       /* mixed plain formats (or different pure size) */
2014       switch (format_desc->format) {
2015       case PIPE_FORMAT_B10G10R10A2_UINT:
2016       case PIPE_FORMAT_R10G10B10A2_UINT:
2017       {
2018          unsigned max10 = (1 << 10) - 1;
2019          max_clamp = lp_build_const_aos(gallivm, vec4_type, max10, max10,
2020                                         max10, (1 << 2) - 1, NULL);
2021       }
2022          break;
2023       case PIPE_FORMAT_R10SG10SB10SA2U_NORM:
2024          min_clamp = lp_build_const_aos(gallivm, vec4_type, -1.0F, -1.0F,
2025                                         -1.0F, 0.0F, NULL);
2026          max_clamp = vec4_bld.one;
2027          break;
2028       case PIPE_FORMAT_R8SG8SB8UX8U_NORM:
2029       case PIPE_FORMAT_R5SG5SB6U_NORM:
2030          min_clamp = lp_build_const_aos(gallivm, vec4_type, -1.0F, -1.0F,
2031                                         0.0F, 0.0F, NULL);
2032          max_clamp = vec4_bld.one;
2033          break;
2034       default:
2035          break;
2036       }
2037    }
2038    else {
2039       /* cannot figure this out from format description */
2040       if (format_desc->layout == UTIL_FORMAT_LAYOUT_S3TC) {
2041          /* s3tc formats are always unorm */
2042          min_clamp = vec4_bld.zero;
2043          max_clamp = vec4_bld.one;
2044       }
2045       else if (format_desc->layout == UTIL_FORMAT_LAYOUT_RGTC ||
2046                format_desc->layout == UTIL_FORMAT_LAYOUT_ETC) {
2047          switch (format_desc->format) {
2048          case PIPE_FORMAT_RGTC1_UNORM:
2049          case PIPE_FORMAT_RGTC2_UNORM:
2050          case PIPE_FORMAT_LATC1_UNORM:
2051          case PIPE_FORMAT_LATC2_UNORM:
2052          case PIPE_FORMAT_ETC1_RGB8:
2053             min_clamp = vec4_bld.zero;
2054             max_clamp = vec4_bld.one;
2055             break;
2056          case PIPE_FORMAT_RGTC1_SNORM:
2057          case PIPE_FORMAT_RGTC2_SNORM:
2058          case PIPE_FORMAT_LATC1_SNORM:
2059          case PIPE_FORMAT_LATC2_SNORM:
2060             min_clamp = lp_build_const_vec(gallivm, vec4_type, -1.0F);
2061             max_clamp = vec4_bld.one;
2062             break;
2063          default:
2064             assert(0);
2065             break;
2066          }
2067       }
2068       /*
2069        * all others from subsampled/other group, though we don't care
2070        * about yuv (and should not have any from zs here)
2071        */
2072       else if (format_desc->colorspace != UTIL_FORMAT_COLORSPACE_YUV){
2073          switch (format_desc->format) {
2074          case PIPE_FORMAT_R8G8_B8G8_UNORM:
2075          case PIPE_FORMAT_G8R8_G8B8_UNORM:
2076          case PIPE_FORMAT_G8R8_B8R8_UNORM:
2077          case PIPE_FORMAT_R8G8_R8B8_UNORM:
2078          case PIPE_FORMAT_R1_UNORM: /* doesn't make sense but ah well */
2079             min_clamp = vec4_bld.zero;
2080             max_clamp = vec4_bld.one;
2081             break;
2082          case PIPE_FORMAT_R8G8Bx_SNORM:
2083             min_clamp = lp_build_const_vec(gallivm, vec4_type, -1.0F);
2084             max_clamp = vec4_bld.one;
2085             break;
2086             /*
2087              * Note smallfloat formats usually don't need clamping
2088              * (they still have infinite range) however this is not
2089              * true for r11g11b10 and r9g9b9e5, which can't represent
2090              * negative numbers (and additionally r9g9b9e5 can't represent
2091              * very large numbers). d3d10 seems happy without clamping in
2092              * this case, but gl spec is pretty clear: "for floating
2093              * point and integer formats, border values are clamped to
2094              * the representable range of the format" so do that here.
2095              */
2096          case PIPE_FORMAT_R11G11B10_FLOAT:
2097             min_clamp = vec4_bld.zero;
2098             break;
2099          case PIPE_FORMAT_R9G9B9E5_FLOAT:
2100             min_clamp = vec4_bld.zero;
2101             max_clamp = lp_build_const_vec(gallivm, vec4_type, MAX_RGB9E5);
2102             break;
2103          default:
2104             assert(0);
2105             break;
2106          }
2107       }
2108    }
2109
2110    if (min_clamp) {
2111       border_color = lp_build_max(&vec4_bld, border_color, min_clamp);
2112    }
2113    if (max_clamp) {
2114       border_color = lp_build_min(&vec4_bld, border_color, max_clamp);
2115    }
2116
2117    bld->border_color_clamped = border_color;
2118 }
2119
2120
2121 /**
2122  * General texture sampling codegen.
2123  * This function handles texture sampling for all texture targets (1D,
2124  * 2D, 3D, cube) and all filtering modes.
2125  */
2126 static void
2127 lp_build_sample_general(struct lp_build_sample_context *bld,
2128                         unsigned sampler_unit,
2129                         boolean is_gather,
2130                         LLVMValueRef *coords,
2131                         const LLVMValueRef *offsets,
2132                         LLVMValueRef lod_positive,
2133                         LLVMValueRef lod_fpart,
2134                         LLVMValueRef ilevel0,
2135                         LLVMValueRef ilevel1,
2136                         LLVMValueRef *colors_out)
2137 {
2138    LLVMBuilderRef builder = bld->gallivm->builder;
2139    const struct lp_static_sampler_state *sampler_state = bld->static_sampler_state;
2140    const unsigned mip_filter = sampler_state->min_mip_filter;
2141    const unsigned min_filter = sampler_state->min_img_filter;
2142    const unsigned mag_filter = sampler_state->mag_img_filter;
2143    LLVMValueRef texels[4];
2144    unsigned chan;
2145
2146    /* if we need border color, (potentially) clamp it now */
2147    if (lp_sampler_wrap_mode_uses_border_color(sampler_state->wrap_s,
2148                                               min_filter,
2149                                               mag_filter) ||
2150        (bld->dims > 1 &&
2151            lp_sampler_wrap_mode_uses_border_color(sampler_state->wrap_t,
2152                                                   min_filter,
2153                                                   mag_filter)) ||
2154        (bld->dims > 2 &&
2155            lp_sampler_wrap_mode_uses_border_color(sampler_state->wrap_r,
2156                                                   min_filter,
2157                                                   mag_filter))) {
2158       lp_build_clamp_border_color(bld, sampler_unit);
2159    }
2160
2161
2162    /*
2163     * Get/interpolate texture colors.
2164     */
2165
2166    for (chan = 0; chan < 4; ++chan) {
2167      texels[chan] = lp_build_alloca(bld->gallivm, bld->texel_bld.vec_type, "");
2168      lp_build_name(texels[chan], "sampler%u_texel_%c_var", sampler_unit, "xyzw"[chan]);
2169    }
2170
2171    if (min_filter == mag_filter) {
2172       /* no need to distinguish between minification and magnification */
2173       lp_build_sample_mipmap(bld, min_filter, mip_filter,
2174                              is_gather,
2175                              coords, offsets,
2176                              ilevel0, ilevel1, lod_fpart,
2177                              texels);
2178    }
2179    else {
2180       /*
2181        * Could also get rid of the if-logic and always use mipmap_both, both
2182        * for the single lod and multi-lod case if nothing really uses this.
2183        */
2184       if (bld->num_lods == 1) {
2185          /* Emit conditional to choose min image filter or mag image filter
2186           * depending on the lod being > 0 or <= 0, respectively.
2187           */
2188          struct lp_build_if_state if_ctx;
2189
2190          lod_positive = LLVMBuildTrunc(builder, lod_positive,
2191                                        LLVMInt1TypeInContext(bld->gallivm->context), "");
2192
2193          lp_build_if(&if_ctx, bld->gallivm, lod_positive);
2194          {
2195             /* Use the minification filter */
2196             lp_build_sample_mipmap(bld, min_filter, mip_filter, FALSE,
2197                                    coords, offsets,
2198                                    ilevel0, ilevel1, lod_fpart,
2199                                    texels);
2200          }
2201          lp_build_else(&if_ctx);
2202          {
2203             /* Use the magnification filter */
2204             lp_build_sample_mipmap(bld, mag_filter, PIPE_TEX_MIPFILTER_NONE,
2205                                    FALSE,
2206                                    coords, offsets,
2207                                    ilevel0, NULL, NULL,
2208                                    texels);
2209          }
2210          lp_build_endif(&if_ctx);
2211       }
2212       else {
2213          LLVMValueRef need_linear, linear_mask;
2214          unsigned mip_filter_for_nearest;
2215          struct lp_build_if_state if_ctx;
2216
2217          if (min_filter == PIPE_TEX_FILTER_LINEAR) {
2218             linear_mask = lod_positive;
2219             mip_filter_for_nearest = PIPE_TEX_MIPFILTER_NONE;
2220          }
2221          else {
2222             linear_mask = lp_build_not(&bld->lodi_bld, lod_positive);
2223             mip_filter_for_nearest = mip_filter;
2224          }
2225          need_linear = lp_build_any_true_range(&bld->lodi_bld, bld->num_lods,
2226                                                linear_mask);
2227
2228          if (bld->num_lods != bld->coord_type.length) {
2229             linear_mask = lp_build_unpack_broadcast_aos_scalars(bld->gallivm,
2230                                                                 bld->lodi_type,
2231                                                                 bld->int_coord_type,
2232                                                                 linear_mask);
2233          }
2234
2235          lp_build_if(&if_ctx, bld->gallivm, need_linear);
2236          {
2237             /*
2238              * Do sampling with both filters simultaneously. This means using
2239              * a linear filter and doing some tricks (with weights) for the pixels
2240              * which need nearest filter.
2241              * Note that it's probably rare some pixels need nearest and some
2242              * linear filter but the fixups required for the nearest pixels
2243              * aren't all that complicated so just always run a combined path
2244              * if at least some pixels require linear.
2245              */
2246             lp_build_sample_mipmap_both(bld, linear_mask, mip_filter,
2247                                         coords, offsets,
2248                                         ilevel0, ilevel1,
2249                                         lod_fpart, lod_positive,
2250                                         texels);
2251          }
2252          lp_build_else(&if_ctx);
2253          {
2254             /*
2255              * All pixels require just nearest filtering, which is way
2256              * cheaper than linear, hence do a separate path for that.
2257              */
2258             lp_build_sample_mipmap(bld, PIPE_TEX_FILTER_NEAREST, FALSE,
2259                                    mip_filter_for_nearest,
2260                                    coords, offsets,
2261                                    ilevel0, ilevel1, lod_fpart,
2262                                    texels);
2263          }
2264          lp_build_endif(&if_ctx);
2265       }
2266    }
2267
2268    for (chan = 0; chan < 4; ++chan) {
2269      colors_out[chan] = LLVMBuildLoad(builder, texels[chan], "");
2270      lp_build_name(colors_out[chan], "sampler%u_texel_%c", sampler_unit, "xyzw"[chan]);
2271    }
2272 }
2273
2274
2275 /**
2276  * Texel fetch function.
2277  * In contrast to general sampling there is no filtering, no coord minification,
2278  * lod (if any) is always explicit uint, coords are uints (in terms of texel units)
2279  * directly to be applied to the selected mip level (after adding texel offsets).
2280  * This function handles texel fetch for all targets where texel fetch is supported
2281  * (no cube maps, but 1d, 2d, 3d are supported, arrays and buffers should be too).
2282  */
2283 static void
2284 lp_build_fetch_texel(struct lp_build_sample_context *bld,
2285                      unsigned texture_unit,
2286                      const LLVMValueRef *coords,
2287                      LLVMValueRef explicit_lod,
2288                      const LLVMValueRef *offsets,
2289                      LLVMValueRef *colors_out)
2290 {
2291    struct lp_build_context *perquadi_bld = &bld->lodi_bld;
2292    struct lp_build_context *int_coord_bld = &bld->int_coord_bld;
2293    unsigned dims = bld->dims, chan;
2294    unsigned target = bld->static_texture_state->target;
2295    boolean out_of_bound_ret_zero = TRUE;
2296    LLVMValueRef size, ilevel;
2297    LLVMValueRef row_stride_vec = NULL, img_stride_vec = NULL;
2298    LLVMValueRef x = coords[0], y = coords[1], z = coords[2];
2299    LLVMValueRef width, height, depth, i, j;
2300    LLVMValueRef offset, out_of_bounds, out1;
2301
2302    out_of_bounds = int_coord_bld->zero;
2303
2304    if (explicit_lod && bld->static_texture_state->target != PIPE_BUFFER) {
2305       if (bld->num_mips != int_coord_bld->type.length) {
2306          ilevel = lp_build_pack_aos_scalars(bld->gallivm, int_coord_bld->type,
2307                                             perquadi_bld->type, explicit_lod, 0);
2308       }
2309       else {
2310          ilevel = explicit_lod;
2311       }
2312       lp_build_nearest_mip_level(bld, texture_unit, ilevel, &ilevel,
2313                                  out_of_bound_ret_zero ? &out_of_bounds : NULL);
2314    }
2315    else {
2316       assert(bld->num_mips == 1);
2317       if (bld->static_texture_state->target != PIPE_BUFFER) {
2318          ilevel = bld->dynamic_state->first_level(bld->dynamic_state, bld->gallivm,
2319                                                   bld->context_ptr, texture_unit);
2320       }
2321       else {
2322          ilevel = lp_build_const_int32(bld->gallivm, 0);
2323       }
2324    }
2325    lp_build_mipmap_level_sizes(bld, ilevel,
2326                                &size,
2327                                &row_stride_vec, &img_stride_vec);
2328    lp_build_extract_image_sizes(bld, &bld->int_size_bld, int_coord_bld->type,
2329                                 size, &width, &height, &depth);
2330
2331    if (target == PIPE_TEXTURE_1D_ARRAY ||
2332        target == PIPE_TEXTURE_2D_ARRAY) {
2333       if (out_of_bound_ret_zero) {
2334          z = lp_build_layer_coord(bld, texture_unit, FALSE, z, &out1);
2335          out_of_bounds = lp_build_or(int_coord_bld, out_of_bounds, out1);
2336       }
2337       else {
2338          z = lp_build_layer_coord(bld, texture_unit, FALSE, z, NULL);
2339       }
2340    }
2341
2342    /* This is a lot like border sampling */
2343    if (offsets[0]) {
2344       /*
2345        * coords are really unsigned, offsets are signed, but I don't think
2346        * exceeding 31 bits is possible
2347        */
2348       x = lp_build_add(int_coord_bld, x, offsets[0]);
2349    }
2350    out1 = lp_build_cmp(int_coord_bld, PIPE_FUNC_LESS, x, int_coord_bld->zero);
2351    out_of_bounds = lp_build_or(int_coord_bld, out_of_bounds, out1);
2352    out1 = lp_build_cmp(int_coord_bld, PIPE_FUNC_GEQUAL, x, width);
2353    out_of_bounds = lp_build_or(int_coord_bld, out_of_bounds, out1);
2354
2355    if (dims >= 2) {
2356       if (offsets[1]) {
2357          y = lp_build_add(int_coord_bld, y, offsets[1]);
2358       }
2359       out1 = lp_build_cmp(int_coord_bld, PIPE_FUNC_LESS, y, int_coord_bld->zero);
2360       out_of_bounds = lp_build_or(int_coord_bld, out_of_bounds, out1);
2361       out1 = lp_build_cmp(int_coord_bld, PIPE_FUNC_GEQUAL, y, height);
2362       out_of_bounds = lp_build_or(int_coord_bld, out_of_bounds, out1);
2363
2364       if (dims >= 3) {
2365          if (offsets[2]) {
2366             z = lp_build_add(int_coord_bld, z, offsets[2]);
2367          }
2368          out1 = lp_build_cmp(int_coord_bld, PIPE_FUNC_LESS, z, int_coord_bld->zero);
2369          out_of_bounds = lp_build_or(int_coord_bld, out_of_bounds, out1);
2370          out1 = lp_build_cmp(int_coord_bld, PIPE_FUNC_GEQUAL, z, depth);
2371          out_of_bounds = lp_build_or(int_coord_bld, out_of_bounds, out1);
2372       }
2373    }
2374
2375    lp_build_sample_offset(int_coord_bld,
2376                           bld->format_desc,
2377                           x, y, z, row_stride_vec, img_stride_vec,
2378                           &offset, &i, &j);
2379
2380    if (bld->static_texture_state->target != PIPE_BUFFER) {
2381       offset = lp_build_add(int_coord_bld, offset,
2382                             lp_build_get_mip_offsets(bld, ilevel));
2383    }
2384
2385    offset = lp_build_andnot(int_coord_bld, offset, out_of_bounds);
2386
2387    lp_build_fetch_rgba_soa(bld->gallivm,
2388                            bld->format_desc,
2389                            bld->texel_type,
2390                            bld->base_ptr, offset,
2391                            i, j,
2392                            colors_out);
2393
2394    if (out_of_bound_ret_zero) {
2395       /*
2396        * Only needed for ARB_robust_buffer_access_behavior and d3d10.
2397        * Could use min/max above instead of out-of-bounds comparisons
2398        * if we don't care about the result returned for out-of-bounds.
2399        */
2400       for (chan = 0; chan < 4; chan++) {
2401          colors_out[chan] = lp_build_select(&bld->texel_bld, out_of_bounds,
2402                                             bld->texel_bld.zero, colors_out[chan]);
2403       }
2404    }
2405 }
2406
2407
2408 /**
2409  * Just set texels to white instead of actually sampling the texture.
2410  * For debugging.
2411  */
2412 void
2413 lp_build_sample_nop(struct gallivm_state *gallivm,
2414                     struct lp_type type,
2415                     const LLVMValueRef *coords,
2416                     LLVMValueRef texel_out[4])
2417 {
2418    LLVMValueRef one = lp_build_one(gallivm, type);
2419    unsigned chan;
2420
2421    for (chan = 0; chan < 4; chan++) {
2422       texel_out[chan] = one;
2423    }
2424 }
2425
2426
2427 /**
2428  * Build the actual texture sampling code.
2429  * 'texel' will return a vector of four LLVMValueRefs corresponding to
2430  * R, G, B, A.
2431  * \param type  vector float type to use for coords, etc.
2432  * \param sample_key
2433  * \param derivs  partial derivatives of (s,t,r,q) with respect to x and y
2434  */
2435 static void
2436 lp_build_sample_soa_code(struct gallivm_state *gallivm,
2437                          const struct lp_static_texture_state *static_texture_state,
2438                          const struct lp_static_sampler_state *static_sampler_state,
2439                          struct lp_sampler_dynamic_state *dynamic_state,
2440                          struct lp_type type,
2441                          unsigned sample_key,
2442                          unsigned texture_index,
2443                          unsigned sampler_index,
2444                          LLVMValueRef context_ptr,
2445                          const LLVMValueRef *coords,
2446                          const LLVMValueRef *offsets,
2447                          const struct lp_derivatives *derivs, /* optional */
2448                          LLVMValueRef lod, /* optional */
2449                          LLVMValueRef texel_out[4])
2450 {
2451    unsigned target = static_texture_state->target;
2452    unsigned dims = texture_dims(target);
2453    unsigned num_quads = type.length / 4;
2454    unsigned mip_filter, min_img_filter, mag_img_filter, i;
2455    struct lp_build_sample_context bld;
2456    struct lp_static_sampler_state derived_sampler_state = *static_sampler_state;
2457    LLVMTypeRef i32t = LLVMInt32TypeInContext(gallivm->context);
2458    LLVMBuilderRef builder = gallivm->builder;
2459    LLVMValueRef tex_width, newcoords[5];
2460    enum lp_sampler_lod_property lod_property;
2461    enum lp_sampler_lod_control lod_control;
2462    enum lp_sampler_op_type op_type;
2463    LLVMValueRef lod_bias = NULL;
2464    LLVMValueRef explicit_lod = NULL;
2465    boolean op_is_tex;
2466
2467    if (0) {
2468       enum pipe_format fmt = static_texture_state->format;
2469       debug_printf("Sample from %s\n", util_format_name(fmt));
2470    }
2471
2472    lod_property = (sample_key & LP_SAMPLER_LOD_PROPERTY_MASK) >>
2473                      LP_SAMPLER_LOD_PROPERTY_SHIFT;
2474    lod_control = (sample_key & LP_SAMPLER_LOD_CONTROL_MASK) >>
2475                     LP_SAMPLER_LOD_CONTROL_SHIFT;
2476    op_type = (sample_key & LP_SAMPLER_OP_TYPE_MASK) >>
2477                  LP_SAMPLER_OP_TYPE_SHIFT;
2478
2479    op_is_tex = op_type == LP_SAMPLER_OP_TEXTURE;
2480
2481    if (lod_control == LP_SAMPLER_LOD_BIAS) {
2482       lod_bias = lod;
2483       assert(lod);
2484       assert(derivs == NULL);
2485    }
2486    else if (lod_control == LP_SAMPLER_LOD_EXPLICIT) {
2487       explicit_lod = lod;
2488       assert(lod);
2489       assert(derivs == NULL);
2490    }
2491    else if (lod_control == LP_SAMPLER_LOD_DERIVATIVES) {
2492       assert(derivs);
2493       assert(lod == NULL);
2494    }
2495    else {
2496       assert(derivs == NULL);
2497       assert(lod == NULL);
2498    }
2499
2500    if (static_texture_state->format == PIPE_FORMAT_NONE) {
2501       /*
2502        * If there's nothing bound, format is NONE, and we must return
2503        * all zero as mandated by d3d10 in this case.
2504        */
2505       unsigned chan;
2506       LLVMValueRef zero = lp_build_zero(gallivm, type);
2507       for (chan = 0; chan < 4; chan++) {
2508          texel_out[chan] = zero;
2509       }
2510       return;
2511    }
2512
2513    assert(type.floating);
2514
2515    /* Setup our build context */
2516    memset(&bld, 0, sizeof bld);
2517    bld.gallivm = gallivm;
2518    bld.context_ptr = context_ptr;
2519    bld.static_sampler_state = &derived_sampler_state;
2520    bld.static_texture_state = static_texture_state;
2521    bld.dynamic_state = dynamic_state;
2522    bld.format_desc = util_format_description(static_texture_state->format);
2523    bld.dims = dims;
2524
2525    bld.vector_width = lp_type_width(type);
2526
2527    bld.float_type = lp_type_float(32);
2528    bld.int_type = lp_type_int(32);
2529    bld.coord_type = type;
2530    bld.int_coord_type = lp_int_type(type);
2531    bld.float_size_in_type = lp_type_float(32);
2532    bld.float_size_in_type.length = dims > 1 ? 4 : 1;
2533    bld.int_size_in_type = lp_int_type(bld.float_size_in_type);
2534    bld.texel_type = type;
2535
2536    /* always using the first channel hopefully should be safe,
2537     * if not things WILL break in other places anyway.
2538     */
2539    if (bld.format_desc->colorspace == UTIL_FORMAT_COLORSPACE_RGB &&
2540        bld.format_desc->channel[0].pure_integer) {
2541       if (bld.format_desc->channel[0].type == UTIL_FORMAT_TYPE_SIGNED) {
2542          bld.texel_type = lp_type_int_vec(type.width, type.width * type.length);
2543       }
2544       else if (bld.format_desc->channel[0].type == UTIL_FORMAT_TYPE_UNSIGNED) {
2545          bld.texel_type = lp_type_uint_vec(type.width, type.width * type.length);
2546       }
2547    }
2548    else if (util_format_has_stencil(bld.format_desc) &&
2549        !util_format_has_depth(bld.format_desc)) {
2550       /* for stencil only formats, sample stencil (uint) */
2551       bld.texel_type = lp_type_int_vec(type.width, type.width * type.length);
2552    }
2553
2554    if (!static_texture_state->level_zero_only) {
2555       derived_sampler_state.min_mip_filter = static_sampler_state->min_mip_filter;
2556    } else {
2557       derived_sampler_state.min_mip_filter = PIPE_TEX_MIPFILTER_NONE;
2558    }
2559    if (op_type == LP_SAMPLER_OP_GATHER) {
2560       /*
2561        * gather4 is exactly like GL_LINEAR filtering but in the end skipping
2562        * the actual filtering. Using mostly the same paths, so cube face
2563        * selection, coord wrapping etc. all naturally uses the same code.
2564        */
2565       derived_sampler_state.min_mip_filter = PIPE_TEX_MIPFILTER_NONE;
2566       derived_sampler_state.min_img_filter = PIPE_TEX_FILTER_LINEAR;
2567       derived_sampler_state.mag_img_filter = PIPE_TEX_FILTER_LINEAR;
2568    }
2569    mip_filter = derived_sampler_state.min_mip_filter;
2570
2571    if (0) {
2572       debug_printf("  .min_mip_filter = %u\n", derived_sampler_state.min_mip_filter);
2573    }
2574
2575    if (static_texture_state->target == PIPE_TEXTURE_CUBE ||
2576        static_texture_state->target == PIPE_TEXTURE_CUBE_ARRAY)
2577    {
2578       /*
2579        * Seamless filtering ignores wrap modes.
2580        * Setting to CLAMP_TO_EDGE is correct for nearest filtering, for
2581        * bilinear it's not correct but way better than using for instance repeat.
2582        * Note we even set this for non-seamless. Technically GL allows any wrap
2583        * mode, which made sense when supporting true borders (can get seamless
2584        * effect with border and CLAMP_TO_BORDER), but gallium doesn't support
2585        * borders and d3d9 requires wrap modes to be ignored and it's a pain to fix
2586        * up the sampler state (as it makes it texture dependent).
2587        */
2588       derived_sampler_state.wrap_s = PIPE_TEX_WRAP_CLAMP_TO_EDGE;
2589       derived_sampler_state.wrap_t = PIPE_TEX_WRAP_CLAMP_TO_EDGE;
2590    }
2591    /*
2592     * We could force CLAMP to CLAMP_TO_EDGE here if min/mag filter is nearest,
2593     * so AoS path could be used. Not sure it's worth the trouble...
2594     */
2595
2596    min_img_filter = derived_sampler_state.min_img_filter;
2597    mag_img_filter = derived_sampler_state.mag_img_filter;
2598
2599
2600    /*
2601     * This is all a bit complicated different paths are chosen for performance
2602     * reasons.
2603     * Essentially, there can be 1 lod per element, 1 lod per quad or 1 lod for
2604     * everything (the last two options are equivalent for 4-wide case).
2605     * If there's per-quad lod but we split to 4-wide so we can use AoS, per-quad
2606     * lod is calculated then the lod value extracted afterwards so making this
2607     * case basically the same as far as lod handling is concerned for the
2608     * further sample/filter code as the 1 lod for everything case.
2609     * Different lod handling mostly shows up when building mipmap sizes
2610     * (lp_build_mipmap_level_sizes() and friends) and also in filtering
2611     * (getting the fractional part of the lod to the right texels).
2612     */
2613
2614    /*
2615     * There are other situations where at least the multiple int lods could be
2616     * avoided like min and max lod being equal.
2617     */
2618    bld.num_mips = bld.num_lods = 1;
2619
2620    if ((gallivm_debug & GALLIVM_DEBUG_NO_QUAD_LOD) &&
2621        (gallivm_debug & GALLIVM_DEBUG_NO_RHO_APPROX) &&
2622        (static_texture_state->target == PIPE_TEXTURE_CUBE ||
2623         static_texture_state->target == PIPE_TEXTURE_CUBE_ARRAY) &&
2624        (op_is_tex && mip_filter != PIPE_TEX_MIPFILTER_NONE)) {
2625       /*
2626        * special case for using per-pixel lod even for implicit lod,
2627        * which is generally never required (ok by APIs) except to please
2628        * some (somewhat broken imho) tests (because per-pixel face selection
2629        * can cause derivatives to be different for pixels outside the primitive
2630        * due to the major axis division even if pre-project derivatives are
2631        * looking normal).
2632        */
2633       bld.num_mips = type.length;
2634       bld.num_lods = type.length;
2635    }
2636    else if (lod_property == LP_SAMPLER_LOD_PER_ELEMENT ||
2637        (explicit_lod || lod_bias || derivs)) {
2638       if ((!op_is_tex && target != PIPE_BUFFER) ||
2639           (op_is_tex && mip_filter != PIPE_TEX_MIPFILTER_NONE)) {
2640          bld.num_mips = type.length;
2641          bld.num_lods = type.length;
2642       }
2643       else if (op_is_tex && min_img_filter != mag_img_filter) {
2644          bld.num_mips = 1;
2645          bld.num_lods = type.length;
2646       }
2647    }
2648    /* TODO: for true scalar_lod should only use 1 lod value */
2649    else if ((!op_is_tex && explicit_lod && target != PIPE_BUFFER) ||
2650             (op_is_tex && mip_filter != PIPE_TEX_MIPFILTER_NONE)) {
2651       bld.num_mips = num_quads;
2652       bld.num_lods = num_quads;
2653    }
2654    else if (op_is_tex && min_img_filter != mag_img_filter) {
2655       bld.num_mips = 1;
2656       bld.num_lods = num_quads;
2657    }
2658
2659
2660    bld.lodf_type = type;
2661    /* we want native vector size to be able to use our intrinsics */
2662    if (bld.num_lods != type.length) {
2663       /* TODO: this currently always has to be per-quad or per-element */
2664       bld.lodf_type.length = type.length > 4 ? ((type.length + 15) / 16) * 4 : 1;
2665    }
2666    bld.lodi_type = lp_int_type(bld.lodf_type);
2667    bld.levelf_type = bld.lodf_type;
2668    if (bld.num_mips == 1) {
2669       bld.levelf_type.length = 1;
2670    }
2671    bld.leveli_type = lp_int_type(bld.levelf_type);
2672    bld.float_size_type = bld.float_size_in_type;
2673    /* Note: size vectors may not be native. They contain minified w/h/d/_ values,
2674     * with per-element lod that is w0/h0/d0/_/w1/h1/d1_/... so up to 8x4f32 */
2675    if (bld.num_mips > 1) {
2676       bld.float_size_type.length = bld.num_mips == type.length ?
2677                                       bld.num_mips * bld.float_size_in_type.length :
2678                                       type.length;
2679    }
2680    bld.int_size_type = lp_int_type(bld.float_size_type);
2681
2682    lp_build_context_init(&bld.float_bld, gallivm, bld.float_type);
2683    lp_build_context_init(&bld.float_vec_bld, gallivm, type);
2684    lp_build_context_init(&bld.int_bld, gallivm, bld.int_type);
2685    lp_build_context_init(&bld.coord_bld, gallivm, bld.coord_type);
2686    lp_build_context_init(&bld.int_coord_bld, gallivm, bld.int_coord_type);
2687    lp_build_context_init(&bld.int_size_in_bld, gallivm, bld.int_size_in_type);
2688    lp_build_context_init(&bld.float_size_in_bld, gallivm, bld.float_size_in_type);
2689    lp_build_context_init(&bld.int_size_bld, gallivm, bld.int_size_type);
2690    lp_build_context_init(&bld.float_size_bld, gallivm, bld.float_size_type);
2691    lp_build_context_init(&bld.texel_bld, gallivm, bld.texel_type);
2692    lp_build_context_init(&bld.levelf_bld, gallivm, bld.levelf_type);
2693    lp_build_context_init(&bld.leveli_bld, gallivm, bld.leveli_type);
2694    lp_build_context_init(&bld.lodf_bld, gallivm, bld.lodf_type);
2695    lp_build_context_init(&bld.lodi_bld, gallivm, bld.lodi_type);
2696
2697    /* Get the dynamic state */
2698    tex_width = dynamic_state->width(dynamic_state, gallivm,
2699                                     context_ptr, texture_index);
2700    bld.row_stride_array = dynamic_state->row_stride(dynamic_state, gallivm,
2701                                                     context_ptr, texture_index);
2702    bld.img_stride_array = dynamic_state->img_stride(dynamic_state, gallivm,
2703                                                     context_ptr, texture_index);
2704    bld.base_ptr = dynamic_state->base_ptr(dynamic_state, gallivm,
2705                                           context_ptr, texture_index);
2706    bld.mip_offsets = dynamic_state->mip_offsets(dynamic_state, gallivm,
2707                                                 context_ptr, texture_index);
2708    /* Note that mip_offsets is an array[level] of offsets to texture images */
2709
2710    /* width, height, depth as single int vector */
2711    if (dims <= 1) {
2712       bld.int_size = tex_width;
2713    }
2714    else {
2715       bld.int_size = LLVMBuildInsertElement(builder, bld.int_size_in_bld.undef,
2716                                             tex_width,
2717                                             LLVMConstInt(i32t, 0, 0), "");
2718       if (dims >= 2) {
2719          LLVMValueRef tex_height =
2720             dynamic_state->height(dynamic_state, gallivm,
2721                                   context_ptr, texture_index);
2722          bld.int_size = LLVMBuildInsertElement(builder, bld.int_size,
2723                                                tex_height,
2724                                                LLVMConstInt(i32t, 1, 0), "");
2725          if (dims >= 3) {
2726             LLVMValueRef tex_depth =
2727                dynamic_state->depth(dynamic_state, gallivm, context_ptr,
2728                                     texture_index);
2729             bld.int_size = LLVMBuildInsertElement(builder, bld.int_size,
2730                                                   tex_depth,
2731                                                   LLVMConstInt(i32t, 2, 0), "");
2732          }
2733       }
2734    }
2735
2736    for (i = 0; i < 5; i++) {
2737       newcoords[i] = coords[i];
2738    }
2739
2740    if (0) {
2741       /* For debug: no-op texture sampling */
2742       lp_build_sample_nop(gallivm,
2743                           bld.texel_type,
2744                           newcoords,
2745                           texel_out);
2746    }
2747
2748    else if (op_type == LP_SAMPLER_OP_FETCH) {
2749       lp_build_fetch_texel(&bld, texture_index, newcoords,
2750                            lod, offsets,
2751                            texel_out);
2752    }
2753
2754    else {
2755       LLVMValueRef lod_fpart = NULL, lod_positive = NULL;
2756       LLVMValueRef ilevel0 = NULL, ilevel1 = NULL;
2757       boolean use_aos;
2758
2759       if (util_format_is_pure_integer(static_texture_state->format) &&
2760           !util_format_has_depth(bld.format_desc) &&
2761           (static_sampler_state->min_mip_filter == PIPE_TEX_MIPFILTER_LINEAR ||
2762            static_sampler_state->min_img_filter == PIPE_TEX_FILTER_LINEAR ||
2763            static_sampler_state->mag_img_filter == PIPE_TEX_FILTER_LINEAR)) {
2764          /*
2765           * Bail if impossible filtering is specified (the awkard additional
2766           * depth check is because it is legal in gallium to have things like S8Z24
2767           * here which would say it's pure int despite such formats should sample
2768           * the depth component).
2769           * In GL such filters make the texture incomplete, this makes it robust
2770           * against state trackers which set this up regardless (we'd crash in the
2771           * lerp later (except for gather)).
2772           * Must do this after fetch_texel code since with GL state tracker we'll
2773           * get some junk sampler for buffer textures.
2774           */
2775          unsigned chan;
2776          LLVMValueRef zero = lp_build_zero(gallivm, type);
2777          for (chan = 0; chan < 4; chan++) {
2778             texel_out[chan] = zero;
2779          }
2780          return;
2781       }
2782
2783       use_aos = util_format_fits_8unorm(bld.format_desc) &&
2784                 op_is_tex &&
2785                 /* not sure this is strictly needed or simply impossible */
2786                 derived_sampler_state.compare_mode == PIPE_TEX_COMPARE_NONE &&
2787                 lp_is_simple_wrap_mode(derived_sampler_state.wrap_s);
2788
2789       use_aos &= bld.num_lods <= num_quads ||
2790                  derived_sampler_state.min_img_filter ==
2791                     derived_sampler_state.mag_img_filter;
2792       if (dims > 1) {
2793          use_aos &= lp_is_simple_wrap_mode(derived_sampler_state.wrap_t);
2794          if (dims > 2) {
2795             use_aos &= lp_is_simple_wrap_mode(derived_sampler_state.wrap_r);
2796          }
2797       }
2798       if ((static_texture_state->target == PIPE_TEXTURE_CUBE ||
2799            static_texture_state->target == PIPE_TEXTURE_CUBE_ARRAY) &&
2800           derived_sampler_state.seamless_cube_map &&
2801           (derived_sampler_state.min_img_filter == PIPE_TEX_FILTER_LINEAR ||
2802            derived_sampler_state.mag_img_filter == PIPE_TEX_FILTER_LINEAR)) {
2803          /* theoretically possible with AoS filtering but not implemented (complex!) */
2804          use_aos = 0;
2805       }
2806
2807       if ((gallivm_debug & GALLIVM_DEBUG_PERF) &&
2808           !use_aos && util_format_fits_8unorm(bld.format_desc)) {
2809          debug_printf("%s: using floating point linear filtering for %s\n",
2810                       __FUNCTION__, bld.format_desc->short_name);
2811          debug_printf("  min_img %d  mag_img %d  mip %d  target %d  seamless %d"
2812                       "  wraps %d  wrapt %d  wrapr %d\n",
2813                       derived_sampler_state.min_img_filter,
2814                       derived_sampler_state.mag_img_filter,
2815                       derived_sampler_state.min_mip_filter,
2816                       static_texture_state->target,
2817                       derived_sampler_state.seamless_cube_map,
2818                       derived_sampler_state.wrap_s,
2819                       derived_sampler_state.wrap_t,
2820                       derived_sampler_state.wrap_r);
2821       }
2822
2823       lp_build_sample_common(&bld, texture_index, sampler_index,
2824                              newcoords,
2825                              derivs, lod_bias, explicit_lod,
2826                              &lod_positive, &lod_fpart,
2827                              &ilevel0, &ilevel1);
2828
2829       if (use_aos && static_texture_state->target == PIPE_TEXTURE_CUBE_ARRAY) {
2830          /* The aos path doesn't do seamless filtering so simply add cube layer
2831           * to face now.
2832           */
2833          newcoords[2] = lp_build_add(&bld.int_coord_bld, newcoords[2], newcoords[3]);
2834       }
2835
2836       /*
2837        * we only try 8-wide sampling with soa as it appears to
2838        * be a loss with aos with AVX (but it should work, except
2839        * for conformance if min_filter != mag_filter if num_lods > 1).
2840        * (It should be faster if we'd support avx2)
2841        */
2842       if (num_quads == 1 || !use_aos) {
2843          if (use_aos) {
2844             /* do sampling/filtering with fixed pt arithmetic */
2845             lp_build_sample_aos(&bld, sampler_index,
2846                                 newcoords[0], newcoords[1],
2847                                 newcoords[2],
2848                                 offsets, lod_positive, lod_fpart,
2849                                 ilevel0, ilevel1,
2850                                 texel_out);
2851          }
2852
2853          else {
2854             lp_build_sample_general(&bld, sampler_index,
2855                                     op_type == LP_SAMPLER_OP_GATHER,
2856                                     newcoords, offsets,
2857                                     lod_positive, lod_fpart,
2858                                     ilevel0, ilevel1,
2859                                     texel_out);
2860          }
2861       }
2862       else {
2863          unsigned j;
2864          struct lp_build_sample_context bld4;
2865          struct lp_type type4 = type;
2866          unsigned i;
2867          LLVMValueRef texelout4[4];
2868          LLVMValueRef texelouttmp[4][LP_MAX_VECTOR_LENGTH/16];
2869
2870          type4.length = 4;
2871
2872          /* Setup our build context */
2873          memset(&bld4, 0, sizeof bld4);
2874          bld4.gallivm = bld.gallivm;
2875          bld4.context_ptr = bld.context_ptr;
2876          bld4.static_texture_state = bld.static_texture_state;
2877          bld4.static_sampler_state = bld.static_sampler_state;
2878          bld4.dynamic_state = bld.dynamic_state;
2879          bld4.format_desc = bld.format_desc;
2880          bld4.dims = bld.dims;
2881          bld4.row_stride_array = bld.row_stride_array;
2882          bld4.img_stride_array = bld.img_stride_array;
2883          bld4.base_ptr = bld.base_ptr;
2884          bld4.mip_offsets = bld.mip_offsets;
2885          bld4.int_size = bld.int_size;
2886
2887          bld4.vector_width = lp_type_width(type4);
2888
2889          bld4.float_type = lp_type_float(32);
2890          bld4.int_type = lp_type_int(32);
2891          bld4.coord_type = type4;
2892          bld4.int_coord_type = lp_int_type(type4);
2893          bld4.float_size_in_type = lp_type_float(32);
2894          bld4.float_size_in_type.length = dims > 1 ? 4 : 1;
2895          bld4.int_size_in_type = lp_int_type(bld4.float_size_in_type);
2896          bld4.texel_type = bld.texel_type;
2897          bld4.texel_type.length = 4;
2898
2899          bld4.num_mips = bld4.num_lods = 1;
2900          if ((gallivm_debug & GALLIVM_DEBUG_NO_QUAD_LOD) &&
2901              (gallivm_debug & GALLIVM_DEBUG_NO_RHO_APPROX) &&
2902              (static_texture_state->target == PIPE_TEXTURE_CUBE ||
2903               static_texture_state->target == PIPE_TEXTURE_CUBE_ARRAY) &&
2904              (op_is_tex && mip_filter != PIPE_TEX_MIPFILTER_NONE)) {
2905             bld4.num_mips = type4.length;
2906             bld4.num_lods = type4.length;
2907          }
2908          if (lod_property == LP_SAMPLER_LOD_PER_ELEMENT &&
2909              (explicit_lod || lod_bias || derivs)) {
2910             if ((!op_is_tex && target != PIPE_BUFFER) ||
2911                 (op_is_tex && mip_filter != PIPE_TEX_MIPFILTER_NONE)) {
2912                bld4.num_mips = type4.length;
2913                bld4.num_lods = type4.length;
2914             }
2915             else if (op_is_tex && min_img_filter != mag_img_filter) {
2916                bld4.num_mips = 1;
2917                bld4.num_lods = type4.length;
2918             }
2919          }
2920
2921          /* we want native vector size to be able to use our intrinsics */
2922          bld4.lodf_type = type4;
2923          if (bld4.num_lods != type4.length) {
2924             bld4.lodf_type.length = 1;
2925          }
2926          bld4.lodi_type = lp_int_type(bld4.lodf_type);
2927          bld4.levelf_type = type4;
2928          if (bld4.num_mips != type4.length) {
2929             bld4.levelf_type.length = 1;
2930          }
2931          bld4.leveli_type = lp_int_type(bld4.levelf_type);
2932          bld4.float_size_type = bld4.float_size_in_type;
2933          if (bld4.num_mips > 1) {
2934             bld4.float_size_type.length = bld4.num_mips == type4.length ?
2935                                             bld4.num_mips * bld4.float_size_in_type.length :
2936                                             type4.length;
2937          }
2938          bld4.int_size_type = lp_int_type(bld4.float_size_type);
2939
2940          lp_build_context_init(&bld4.float_bld, gallivm, bld4.float_type);
2941          lp_build_context_init(&bld4.float_vec_bld, gallivm, type4);
2942          lp_build_context_init(&bld4.int_bld, gallivm, bld4.int_type);
2943          lp_build_context_init(&bld4.coord_bld, gallivm, bld4.coord_type);
2944          lp_build_context_init(&bld4.int_coord_bld, gallivm, bld4.int_coord_type);
2945          lp_build_context_init(&bld4.int_size_in_bld, gallivm, bld4.int_size_in_type);
2946          lp_build_context_init(&bld4.float_size_in_bld, gallivm, bld4.float_size_in_type);
2947          lp_build_context_init(&bld4.int_size_bld, gallivm, bld4.int_size_type);
2948          lp_build_context_init(&bld4.float_size_bld, gallivm, bld4.float_size_type);
2949          lp_build_context_init(&bld4.texel_bld, gallivm, bld4.texel_type);
2950          lp_build_context_init(&bld4.levelf_bld, gallivm, bld4.levelf_type);
2951          lp_build_context_init(&bld4.leveli_bld, gallivm, bld4.leveli_type);
2952          lp_build_context_init(&bld4.lodf_bld, gallivm, bld4.lodf_type);
2953          lp_build_context_init(&bld4.lodi_bld, gallivm, bld4.lodi_type);
2954
2955          for (i = 0; i < num_quads; i++) {
2956             LLVMValueRef s4, t4, r4;
2957             LLVMValueRef lod_positive4, lod_fpart4 = NULL;
2958             LLVMValueRef ilevel04, ilevel14 = NULL;
2959             LLVMValueRef offsets4[4] = { NULL };
2960             unsigned num_lods = bld4.num_lods;
2961
2962             s4 = lp_build_extract_range(gallivm, newcoords[0], 4*i, 4);
2963             t4 = lp_build_extract_range(gallivm, newcoords[1], 4*i, 4);
2964             r4 = lp_build_extract_range(gallivm, newcoords[2], 4*i, 4);
2965
2966             if (offsets[0]) {
2967                offsets4[0] = lp_build_extract_range(gallivm, offsets[0], 4*i, 4);
2968                if (dims > 1) {
2969                   offsets4[1] = lp_build_extract_range(gallivm, offsets[1], 4*i, 4);
2970                   if (dims > 2) {
2971                      offsets4[2] = lp_build_extract_range(gallivm, offsets[2], 4*i, 4);
2972                   }
2973                }
2974             }
2975             lod_positive4 = lp_build_extract_range(gallivm, lod_positive, num_lods * i, num_lods);
2976             ilevel04 = bld.num_mips == 1 ? ilevel0 :
2977                           lp_build_extract_range(gallivm, ilevel0, num_lods * i, num_lods);
2978             if (mip_filter == PIPE_TEX_MIPFILTER_LINEAR) {
2979                ilevel14 = lp_build_extract_range(gallivm, ilevel1, num_lods * i, num_lods);
2980                lod_fpart4 = lp_build_extract_range(gallivm, lod_fpart, num_lods * i, num_lods);
2981             }
2982
2983             if (use_aos) {
2984                /* do sampling/filtering with fixed pt arithmetic */
2985                lp_build_sample_aos(&bld4, sampler_index,
2986                                    s4, t4, r4, offsets4,
2987                                    lod_positive4, lod_fpart4,
2988                                    ilevel04, ilevel14,
2989                                    texelout4);
2990             }
2991
2992             else {
2993                /* this path is currently unreachable and hence might break easily... */
2994                LLVMValueRef newcoords4[5];
2995                newcoords4[0] = s4;
2996                newcoords4[1] = t4;
2997                newcoords4[2] = r4;
2998                newcoords4[3] = lp_build_extract_range(gallivm, newcoords[3], 4*i, 4);
2999                newcoords4[4] = lp_build_extract_range(gallivm, newcoords[4], 4*i, 4);
3000
3001                lp_build_sample_general(&bld4, sampler_index,
3002                                        op_type == LP_SAMPLER_OP_GATHER,
3003                                        newcoords4, offsets4,
3004                                        lod_positive4, lod_fpart4,
3005                                        ilevel04, ilevel14,
3006                                        texelout4);
3007             }
3008             for (j = 0; j < 4; j++) {
3009                texelouttmp[j][i] = texelout4[j];
3010             }
3011          }
3012
3013          for (j = 0; j < 4; j++) {
3014             texel_out[j] = lp_build_concat(gallivm, texelouttmp[j], type4, num_quads);
3015          }
3016       }
3017    }
3018
3019    if (target != PIPE_BUFFER && op_type != LP_SAMPLER_OP_GATHER) {
3020       apply_sampler_swizzle(&bld, texel_out);
3021    }
3022
3023    /*
3024     * texel type can be a (32bit) int/uint (for pure int formats only),
3025     * however we are expected to always return floats (storage is untyped).
3026     */
3027    if (!bld.texel_type.floating) {
3028       unsigned chan;
3029       for (chan = 0; chan < 4; chan++) {
3030          texel_out[chan] = LLVMBuildBitCast(builder, texel_out[chan],
3031                                             lp_build_vec_type(gallivm, type), "");
3032       }
3033    }
3034 }
3035
3036
3037 #define USE_TEX_FUNC_CALL 1
3038
3039 #define LP_MAX_TEX_FUNC_ARGS 32
3040
3041 static inline void
3042 get_target_info(enum pipe_texture_target target,
3043                 unsigned *num_coords, unsigned *num_derivs,
3044                 unsigned *num_offsets, unsigned *layer)
3045 {
3046    unsigned dims = texture_dims(target);
3047    *num_coords = dims;
3048    *num_offsets = dims;
3049    *num_derivs = (target == PIPE_TEXTURE_CUBE ||
3050                   target == PIPE_TEXTURE_CUBE_ARRAY) ? 3 : dims;
3051    *layer = has_layer_coord(target) ? 2: 0;
3052    if (target == PIPE_TEXTURE_CUBE_ARRAY) {
3053       /*
3054        * dims doesn't include r coord for cubes - this is handled
3055        * by layer instead, but need to fix up for cube arrays...
3056        */
3057       *layer = 3;
3058       *num_coords = 3;
3059    }
3060 }
3061
3062
3063 /**
3064  * Generate the function body for a texture sampling function.
3065  */
3066 static void
3067 lp_build_sample_gen_func(struct gallivm_state *gallivm,
3068                          const struct lp_static_texture_state *static_texture_state,
3069                          const struct lp_static_sampler_state *static_sampler_state,
3070                          struct lp_sampler_dynamic_state *dynamic_state,
3071                          struct lp_type type,
3072                          unsigned texture_index,
3073                          unsigned sampler_index,
3074                          LLVMValueRef function,
3075                          unsigned num_args,
3076                          unsigned sample_key)
3077 {
3078    LLVMBuilderRef old_builder;
3079    LLVMBasicBlockRef block;
3080    LLVMValueRef coords[5];
3081    LLVMValueRef offsets[3] = { NULL };
3082    LLVMValueRef lod = NULL;
3083    LLVMValueRef context_ptr;
3084    LLVMValueRef texel_out[4];
3085    struct lp_derivatives derivs;
3086    struct lp_derivatives *deriv_ptr = NULL;
3087    unsigned num_param = 0;
3088    unsigned i, num_coords, num_derivs, num_offsets, layer;
3089    enum lp_sampler_lod_control lod_control;
3090
3091    lod_control = (sample_key & LP_SAMPLER_LOD_CONTROL_MASK) >>
3092                     LP_SAMPLER_LOD_CONTROL_SHIFT;
3093
3094    get_target_info(static_texture_state->target,
3095                    &num_coords, &num_derivs, &num_offsets, &layer);
3096
3097    /* "unpack" arguments */
3098    context_ptr = LLVMGetParam(function, num_param++);
3099    for (i = 0; i < num_coords; i++) {
3100       coords[i] = LLVMGetParam(function, num_param++);
3101    }
3102    for (i = num_coords; i < 5; i++) {
3103       /* This is rather unfortunate... */
3104       coords[i] = lp_build_undef(gallivm, type);
3105    }
3106    if (layer) {
3107       coords[layer] = LLVMGetParam(function, num_param++);
3108    }
3109    if (sample_key & LP_SAMPLER_SHADOW) {
3110       coords[4] = LLVMGetParam(function, num_param++);
3111    }
3112    if (sample_key & LP_SAMPLER_OFFSETS) {
3113       for (i = 0; i < num_offsets; i++) {
3114          offsets[i] = LLVMGetParam(function, num_param++);
3115       }
3116    }
3117    if (lod_control == LP_SAMPLER_LOD_BIAS ||
3118        lod_control == LP_SAMPLER_LOD_EXPLICIT) {
3119       lod = LLVMGetParam(function, num_param++);
3120    }
3121    else if (lod_control == LP_SAMPLER_LOD_DERIVATIVES) {
3122       for (i = 0; i < num_derivs; i++) {
3123          derivs.ddx[i] = LLVMGetParam(function, num_param++);
3124          derivs.ddy[i] = LLVMGetParam(function, num_param++);
3125       }
3126       deriv_ptr = &derivs;
3127    }
3128
3129    assert(num_args == num_param);
3130
3131    /*
3132     * Function body
3133     */
3134
3135    old_builder = gallivm->builder;
3136    block = LLVMAppendBasicBlockInContext(gallivm->context, function, "entry");
3137    gallivm->builder = LLVMCreateBuilderInContext(gallivm->context);
3138    LLVMPositionBuilderAtEnd(gallivm->builder, block);
3139
3140    lp_build_sample_soa_code(gallivm,
3141                             static_texture_state,
3142                             static_sampler_state,
3143                             dynamic_state,
3144                             type,
3145                             sample_key,
3146                             texture_index,
3147                             sampler_index,
3148                             context_ptr,
3149                             coords,
3150                             offsets,
3151                             deriv_ptr,
3152                             lod,
3153                             texel_out);
3154
3155    LLVMBuildAggregateRet(gallivm->builder, texel_out, 4);
3156
3157    LLVMDisposeBuilder(gallivm->builder);
3158    gallivm->builder = old_builder;
3159
3160    gallivm_verify_function(gallivm, function);
3161 }
3162
3163
3164 /**
3165  * Call the matching function for texture sampling.
3166  * If there's no match, generate a new one.
3167  */
3168 static void
3169 lp_build_sample_soa_func(struct gallivm_state *gallivm,
3170                          const struct lp_static_texture_state *static_texture_state,
3171                          const struct lp_static_sampler_state *static_sampler_state,
3172                          struct lp_sampler_dynamic_state *dynamic_state,
3173                          const struct lp_sampler_params *params)
3174 {
3175    LLVMBuilderRef builder = gallivm->builder;
3176    LLVMModuleRef module = LLVMGetGlobalParent(LLVMGetBasicBlockParent(
3177                              LLVMGetInsertBlock(builder)));
3178    LLVMValueRef function, inst;
3179    LLVMValueRef args[LP_MAX_TEX_FUNC_ARGS];
3180    LLVMBasicBlockRef bb;
3181    LLVMValueRef tex_ret;
3182    unsigned num_args = 0;
3183    char func_name[64];
3184    unsigned i, num_coords, num_derivs, num_offsets, layer;
3185    unsigned texture_index = params->texture_index;
3186    unsigned sampler_index = params->sampler_index;
3187    unsigned sample_key = params->sample_key;
3188    const LLVMValueRef *coords = params->coords;
3189    const LLVMValueRef *offsets = params->offsets;
3190    const struct lp_derivatives *derivs = params->derivs;
3191    enum lp_sampler_lod_control lod_control;
3192
3193    lod_control = (sample_key & LP_SAMPLER_LOD_CONTROL_MASK) >>
3194                     LP_SAMPLER_LOD_CONTROL_SHIFT;
3195
3196    get_target_info(static_texture_state->target,
3197                    &num_coords, &num_derivs, &num_offsets, &layer);
3198
3199    /*
3200     * texture function matches are found by name.
3201     * Thus the name has to include both the texture and sampler unit
3202     * (which covers all static state) plus the actual texture function
3203     * (including things like offsets, shadow coord, lod control).
3204     * Additionally lod_property has to be included too.
3205     */
3206
3207    util_snprintf(func_name, sizeof(func_name), "texfunc_res_%d_sam_%d_%x",
3208                  texture_index, sampler_index, sample_key);
3209
3210    function = LLVMGetNamedFunction(module, func_name);
3211
3212    if(!function) {
3213       LLVMTypeRef arg_types[LP_MAX_TEX_FUNC_ARGS];
3214       LLVMTypeRef ret_type;
3215       LLVMTypeRef function_type;
3216       LLVMTypeRef val_type[4];
3217       unsigned num_param = 0;
3218
3219       /*
3220        * Generate the function prototype.
3221        */
3222
3223       arg_types[num_param++] = LLVMTypeOf(params->context_ptr);
3224       for (i = 0; i < num_coords; i++) {
3225          arg_types[num_param++] = LLVMTypeOf(coords[0]);
3226          assert(LLVMTypeOf(coords[0]) == LLVMTypeOf(coords[i]));
3227       }
3228       if (layer) {
3229          arg_types[num_param++] = LLVMTypeOf(coords[layer]);
3230          assert(LLVMTypeOf(coords[0]) == LLVMTypeOf(coords[layer]));
3231       }
3232       if (sample_key & LP_SAMPLER_SHADOW) {
3233          arg_types[num_param++] = LLVMTypeOf(coords[0]);
3234       }
3235       if (sample_key & LP_SAMPLER_OFFSETS) {
3236          for (i = 0; i < num_offsets; i++) {
3237             arg_types[num_param++] = LLVMTypeOf(offsets[0]);
3238             assert(LLVMTypeOf(offsets[0]) == LLVMTypeOf(offsets[i]));
3239          }
3240       }
3241       if (lod_control == LP_SAMPLER_LOD_BIAS ||
3242           lod_control == LP_SAMPLER_LOD_EXPLICIT) {
3243          arg_types[num_param++] = LLVMTypeOf(params->lod);
3244       }
3245       else if (lod_control == LP_SAMPLER_LOD_DERIVATIVES) {
3246          for (i = 0; i < num_derivs; i++) {
3247             arg_types[num_param++] = LLVMTypeOf(derivs->ddx[i]);
3248             arg_types[num_param++] = LLVMTypeOf(derivs->ddy[i]);
3249             assert(LLVMTypeOf(derivs->ddx[0]) == LLVMTypeOf(derivs->ddx[i]));
3250             assert(LLVMTypeOf(derivs->ddy[0]) == LLVMTypeOf(derivs->ddy[i]));
3251          }
3252       }
3253
3254       val_type[0] = val_type[1] = val_type[2] = val_type[3] =
3255          lp_build_vec_type(gallivm, params->type);
3256       ret_type = LLVMStructTypeInContext(gallivm->context, val_type, 4, 0);
3257       function_type = LLVMFunctionType(ret_type, arg_types, num_param, 0);
3258       function = LLVMAddFunction(module, func_name, function_type);
3259
3260       for (i = 0; i < num_param; ++i) {
3261          if(LLVMGetTypeKind(arg_types[i]) == LLVMPointerTypeKind) {
3262             LLVMAddAttribute(LLVMGetParam(function, i), LLVMNoAliasAttribute);
3263          }
3264       }
3265
3266       LLVMSetFunctionCallConv(function, LLVMFastCallConv);
3267       LLVMSetLinkage(function, LLVMPrivateLinkage);
3268
3269       lp_build_sample_gen_func(gallivm,
3270                                static_texture_state,
3271                                static_sampler_state,
3272                                dynamic_state,
3273                                params->type,
3274                                texture_index,
3275                                sampler_index,
3276                                function,
3277                                num_param,
3278                                sample_key);
3279    }
3280
3281    num_args = 0;
3282    args[num_args++] = params->context_ptr;
3283    for (i = 0; i < num_coords; i++) {
3284       args[num_args++] = coords[i];
3285    }
3286    if (layer) {
3287       args[num_args++] = coords[layer];
3288    }
3289    if (sample_key & LP_SAMPLER_SHADOW) {
3290       args[num_args++] = coords[4];
3291    }
3292    if (sample_key & LP_SAMPLER_OFFSETS) {
3293       for (i = 0; i < num_offsets; i++) {
3294          args[num_args++] = offsets[i];
3295       }
3296    }
3297    if (lod_control == LP_SAMPLER_LOD_BIAS ||
3298        lod_control == LP_SAMPLER_LOD_EXPLICIT) {
3299       args[num_args++] = params->lod;
3300    }
3301    else if (lod_control == LP_SAMPLER_LOD_DERIVATIVES) {
3302       for (i = 0; i < num_derivs; i++) {
3303          args[num_args++] = derivs->ddx[i];
3304          args[num_args++] = derivs->ddy[i];
3305       }
3306    }
3307
3308    assert(num_args <= LP_MAX_TEX_FUNC_ARGS);
3309
3310    tex_ret = LLVMBuildCall(builder, function, args, num_args, "");
3311    bb = LLVMGetInsertBlock(builder);
3312    inst = LLVMGetLastInstruction(bb);
3313    LLVMSetInstructionCallConv(inst, LLVMFastCallConv);
3314
3315    for (i = 0; i < 4; i++) {
3316       params->texel[i] = LLVMBuildExtractValue(gallivm->builder, tex_ret, i, "");
3317    }
3318 }
3319
3320
3321 /**
3322  * Build texture sampling code.
3323  * Either via a function call or inline it directly.
3324  */
3325 void
3326 lp_build_sample_soa(const struct lp_static_texture_state *static_texture_state,
3327                     const struct lp_static_sampler_state *static_sampler_state,
3328                     struct lp_sampler_dynamic_state *dynamic_state,
3329                     struct gallivm_state *gallivm,
3330                     const struct lp_sampler_params *params)
3331 {
3332    boolean use_tex_func = FALSE;
3333
3334    /*
3335     * Do not use a function call if the sampling is "simple enough".
3336     * We define this by
3337     * a) format
3338     * b) no mips (either one level only or no mip filter)
3339     * No mips will definitely make the code smaller, though
3340     * the format requirement is a bit iffy - there's some (SoA) formats
3341     * which definitely generate less code. This does happen to catch
3342     * some important cases though which are hurt quite a bit by using
3343     * a call (though not really because of the call overhead but because
3344     * they are reusing the same texture unit with some of the same
3345     * parameters).
3346     * Ideally we'd let llvm recognize this stuff by doing IPO passes.
3347     */
3348
3349    if (USE_TEX_FUNC_CALL) {
3350       const struct util_format_description *format_desc;
3351       boolean simple_format;
3352       boolean simple_tex;
3353       enum lp_sampler_op_type op_type;
3354       format_desc = util_format_description(static_texture_state->format);
3355       simple_format = !format_desc ||
3356                          (util_format_is_rgba8_variant(format_desc) &&
3357                           format_desc->colorspace == UTIL_FORMAT_COLORSPACE_RGB);
3358
3359       op_type = (params->sample_key & LP_SAMPLER_OP_TYPE_MASK) >>
3360                     LP_SAMPLER_OP_TYPE_SHIFT;
3361       simple_tex =
3362          op_type != LP_SAMPLER_OP_TEXTURE ||
3363            ((static_sampler_state->min_mip_filter == PIPE_TEX_MIPFILTER_NONE ||
3364              static_texture_state->level_zero_only == TRUE) &&
3365             static_sampler_state->min_img_filter == static_sampler_state->mag_img_filter);
3366
3367       use_tex_func = format_desc && !(simple_format && simple_tex);
3368    }
3369
3370    if (use_tex_func) {
3371       lp_build_sample_soa_func(gallivm,
3372                                static_texture_state,
3373                                static_sampler_state,
3374                                dynamic_state,
3375                                params);
3376    }
3377    else {
3378       lp_build_sample_soa_code(gallivm,
3379                                static_texture_state,
3380                                static_sampler_state,
3381                                dynamic_state,
3382                                params->type,
3383                                params->sample_key,
3384                                params->texture_index,
3385                                params->sampler_index,
3386                                params->context_ptr,
3387                                params->coords,
3388                                params->offsets,
3389                                params->derivs,
3390                                params->lod,
3391                                params->texel);
3392    }
3393 }
3394
3395
3396 void
3397 lp_build_size_query_soa(struct gallivm_state *gallivm,
3398                         const struct lp_static_texture_state *static_state,
3399                         struct lp_sampler_dynamic_state *dynamic_state,
3400                         struct lp_type int_type,
3401                         unsigned texture_unit,
3402                         unsigned target,
3403                         LLVMValueRef context_ptr,
3404                         boolean is_sviewinfo,
3405                         enum lp_sampler_lod_property lod_property,
3406                         LLVMValueRef explicit_lod,
3407                         LLVMValueRef *sizes_out)
3408 {
3409    LLVMValueRef lod, level, size;
3410    LLVMValueRef first_level = NULL;
3411    int dims, i;
3412    boolean has_array;
3413    unsigned num_lods = 1;
3414    struct lp_build_context bld_int_vec4;
3415
3416    if (static_state->format == PIPE_FORMAT_NONE) {
3417       /*
3418        * If there's nothing bound, format is NONE, and we must return
3419        * all zero as mandated by d3d10 in this case.
3420        */
3421       unsigned chan;
3422       LLVMValueRef zero = lp_build_const_vec(gallivm, int_type, 0.0F);
3423       for (chan = 0; chan < 4; chan++) {
3424          sizes_out[chan] = zero;
3425       }
3426       return;
3427    }
3428
3429    /*
3430     * Do some sanity verification about bound texture and shader dcl target.
3431     * Not entirely sure what's possible but assume array/non-array
3432     * always compatible (probably not ok for OpenGL but d3d10 has no
3433     * distinction of arrays at the resource level).
3434     * Everything else looks bogus (though not entirely sure about rect/2d).
3435     * Currently disabled because it causes assertion failures if there's
3436     * nothing bound (or rather a dummy texture, not that this case would
3437     * return the right values).
3438     */
3439    if (0 && static_state->target != target) {
3440       if (static_state->target == PIPE_TEXTURE_1D)
3441          assert(target == PIPE_TEXTURE_1D_ARRAY);
3442       else if (static_state->target == PIPE_TEXTURE_1D_ARRAY)
3443          assert(target == PIPE_TEXTURE_1D);
3444       else if (static_state->target == PIPE_TEXTURE_2D)
3445          assert(target == PIPE_TEXTURE_2D_ARRAY);
3446       else if (static_state->target == PIPE_TEXTURE_2D_ARRAY)
3447          assert(target == PIPE_TEXTURE_2D);
3448       else if (static_state->target == PIPE_TEXTURE_CUBE)
3449          assert(target == PIPE_TEXTURE_CUBE_ARRAY);
3450       else if (static_state->target == PIPE_TEXTURE_CUBE_ARRAY)
3451          assert(target == PIPE_TEXTURE_CUBE);
3452       else
3453          assert(0);
3454    }
3455
3456    dims = texture_dims(target);
3457
3458    switch (target) {
3459    case PIPE_TEXTURE_1D_ARRAY:
3460    case PIPE_TEXTURE_2D_ARRAY:
3461    case PIPE_TEXTURE_CUBE_ARRAY:
3462       has_array = TRUE;
3463       break;
3464    default:
3465       has_array = FALSE;
3466       break;
3467    }
3468
3469    assert(!int_type.floating);
3470
3471    lp_build_context_init(&bld_int_vec4, gallivm, lp_type_int_vec(32, 128));
3472
3473    if (explicit_lod) {
3474       /* FIXME: this needs to honor per-element lod */
3475       lod = LLVMBuildExtractElement(gallivm->builder, explicit_lod,
3476                                     lp_build_const_int32(gallivm, 0), "");
3477       first_level = dynamic_state->first_level(dynamic_state, gallivm,
3478                                                context_ptr, texture_unit);
3479       level = LLVMBuildAdd(gallivm->builder, lod, first_level, "level");
3480       lod = lp_build_broadcast_scalar(&bld_int_vec4, level);
3481    } else {
3482       lod = bld_int_vec4.zero;
3483    }
3484
3485    size = bld_int_vec4.undef;
3486
3487    size = LLVMBuildInsertElement(gallivm->builder, size,
3488                                  dynamic_state->width(dynamic_state, gallivm,
3489                                                       context_ptr, texture_unit),
3490                                  lp_build_const_int32(gallivm, 0), "");
3491
3492    if (dims >= 2) {
3493       size = LLVMBuildInsertElement(gallivm->builder, size,
3494                                     dynamic_state->height(dynamic_state, gallivm,
3495                                                           context_ptr, texture_unit),
3496                                     lp_build_const_int32(gallivm, 1), "");
3497    }
3498
3499    if (dims >= 3) {
3500       size = LLVMBuildInsertElement(gallivm->builder, size,
3501                                     dynamic_state->depth(dynamic_state, gallivm,
3502                                                          context_ptr, texture_unit),
3503                                     lp_build_const_int32(gallivm, 2), "");
3504    }
3505
3506    size = lp_build_minify(&bld_int_vec4, size, lod, TRUE);
3507
3508    if (has_array) {
3509       LLVMValueRef layers = dynamic_state->depth(dynamic_state, gallivm,
3510                                                  context_ptr, texture_unit);
3511       if (target == PIPE_TEXTURE_CUBE_ARRAY) {
3512          /*
3513           * It looks like GL wants number of cubes, d3d10.1 has it undefined?
3514           * Could avoid this by passing in number of cubes instead of total
3515           * number of layers (might make things easier elsewhere too).
3516           */
3517          LLVMValueRef six = lp_build_const_int32(gallivm, 6);
3518          layers = LLVMBuildSDiv(gallivm->builder, layers, six, "");
3519       }
3520       size = LLVMBuildInsertElement(gallivm->builder, size, layers,
3521                                     lp_build_const_int32(gallivm, dims), "");
3522    }
3523
3524    /*
3525     * d3d10 requires zero for x/y/z values (but not w, i.e. mip levels)
3526     * if level is out of bounds (note this can't cover unbound texture
3527     * here, which also requires returning zero).
3528     */
3529    if (explicit_lod && is_sviewinfo) {
3530       LLVMValueRef last_level, out, out1;
3531       struct lp_build_context leveli_bld;
3532
3533       /* everything is scalar for now */
3534       lp_build_context_init(&leveli_bld, gallivm, lp_type_int_vec(32, 32));
3535       last_level = dynamic_state->last_level(dynamic_state, gallivm,
3536                                              context_ptr, texture_unit);
3537
3538       out = lp_build_cmp(&leveli_bld, PIPE_FUNC_LESS, level, first_level);
3539       out1 = lp_build_cmp(&leveli_bld, PIPE_FUNC_GREATER, level, last_level);
3540       out = lp_build_or(&leveli_bld, out, out1);
3541       if (num_lods == 1) {
3542          out = lp_build_broadcast_scalar(&bld_int_vec4, out);
3543       }
3544       else {
3545          /* TODO */
3546          assert(0);
3547       }
3548       size = lp_build_andnot(&bld_int_vec4, size, out);
3549    }
3550    for (i = 0; i < dims + (has_array ? 1 : 0); i++) {
3551       sizes_out[i] = lp_build_extract_broadcast(gallivm, bld_int_vec4.type, int_type,
3552                                                 size,
3553                                                 lp_build_const_int32(gallivm, i));
3554    }
3555    if (is_sviewinfo) {
3556       for (; i < 4; i++) {
3557          sizes_out[i] = lp_build_const_vec(gallivm, int_type, 0.0);
3558       }
3559    }
3560
3561    /*
3562     * if there's no explicit_lod (buffers, rects) queries requiring nr of
3563     * mips would be illegal.
3564     */
3565    if (is_sviewinfo && explicit_lod) {
3566       struct lp_build_context bld_int_scalar;
3567       LLVMValueRef num_levels;
3568       lp_build_context_init(&bld_int_scalar, gallivm, lp_type_int(32));
3569
3570       if (static_state->level_zero_only) {
3571          num_levels = bld_int_scalar.one;
3572       }
3573       else {
3574          LLVMValueRef last_level;
3575
3576          last_level = dynamic_state->last_level(dynamic_state, gallivm,
3577                                                 context_ptr, texture_unit);
3578          num_levels = lp_build_sub(&bld_int_scalar, last_level, first_level);
3579          num_levels = lp_build_add(&bld_int_scalar, num_levels, bld_int_scalar.one);
3580       }
3581       sizes_out[3] = lp_build_broadcast(gallivm, lp_build_vec_type(gallivm, int_type),
3582                                         num_levels);
3583    }
3584 }