src/intel/vulkan/genX_pipeline.c

   1 /*
   2  * Copyright © 2015 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  */
  23
  24 #include "anv_private.h"
  25
  26 #include "genxml/gen_macros.h"
  27 #include "genxml/genX_pack.h"
  28
  29 #include "common/gen_l3_config.h"
  30 #include "common/gen_sample_positions.h"
  31 #include "vk_format_info.h"
  32
  33 static uint32_t
  34 vertex_element_comp_control(enum isl_format format, unsigned comp)
  35 {
  36    uint8_t bits;
  37    switch (comp) {
  38    case 0: bits = isl_format_layouts[format].channels.r.bits; break;
  39    case 1: bits = isl_format_layouts[format].channels.g.bits; break;
  40    case 2: bits = isl_format_layouts[format].channels.b.bits; break;
  41    case 3: bits = isl_format_layouts[format].channels.a.bits; break;
  42    default: unreachable("Invalid component");
  43    }
  44
  45    /*
  46     * Take in account hardware restrictions when dealing with 64-bit floats.
  47     *
  48     * From Broadwell spec, command reference structures, page 586:
  49     *  "When SourceElementFormat is set to one of the *64*_PASSTHRU formats,
  50     *   64-bit components are stored * in the URB without any conversion. In
  51     *   this case, vertex elements must be written as 128 or 256 bits, with
  52     *   VFCOMP_STORE_0 being used to pad the output as required. E.g., if
  53     *   R64_PASSTHRU is used to copy a 64-bit Red component into the URB,
  54     *   Component 1 must be specified as VFCOMP_STORE_0 (with Components 2,3
  55     *   set to VFCOMP_NOSTORE) in order to output a 128-bit vertex element, or
  56     *   Components 1-3 must be specified as VFCOMP_STORE_0 in order to output
  57     *   a 256-bit vertex element. Likewise, use of R64G64B64_PASSTHRU requires
  58     *   Component 3 to be specified as VFCOMP_STORE_0 in order to output a
  59     *   256-bit vertex element."
  60     */
  61    if (bits) {
  62       return VFCOMP_STORE_SRC;
  63    } else if (comp >= 2 &&
  64               !isl_format_layouts[format].channels.b.bits &&
  65               isl_format_layouts[format].channels.r.type == ISL_RAW) {
  66       /* When emitting 64-bit attributes, we need to write either 128 or 256
  67        * bit chunks, using VFCOMP_NOSTORE when not writing the chunk, and
  68        * VFCOMP_STORE_0 to pad the written chunk */
  69       return VFCOMP_NOSTORE;
  70    } else if (comp < 3 ||
  71               isl_format_layouts[format].channels.r.type == ISL_RAW) {
  72       /* Note we need to pad with value 0, not 1, due hardware restrictions
  73        * (see comment above) */
  74       return VFCOMP_STORE_0;
  75    } else if (isl_format_layouts[format].channels.r.type == ISL_UINT ||
  76             isl_format_layouts[format].channels.r.type == ISL_SINT) {
  77       assert(comp == 3);
  78       return VFCOMP_STORE_1_INT;
  79    } else {
  80       assert(comp == 3);
  81       return VFCOMP_STORE_1_FP;
  82    }
  83 }
  84
  85 static void
  86 emit_vertex_input(struct anv_pipeline *pipeline,
  87                   const VkPipelineVertexInputStateCreateInfo *info)
  88 {
  89    const struct brw_vs_prog_data *vs_prog_data = get_vs_prog_data(pipeline);
  90
  91    /* Pull inputs_read out of the VS prog data */
  92    const uint64_t inputs_read = vs_prog_data->inputs_read;
  93    const uint64_t double_inputs_read = vs_prog_data->double_inputs_read;
  94    assert((inputs_read & ((1 << VERT_ATTRIB_GENERIC0) - 1)) == 0);
  95    const uint32_t elements = inputs_read >> VERT_ATTRIB_GENERIC0;
  96    const uint32_t elements_double = double_inputs_read >> VERT_ATTRIB_GENERIC0;
  97    const bool needs_svgs_elem = vs_prog_data->uses_vertexid ||
  98                                 vs_prog_data->uses_instanceid ||
  99                                 vs_prog_data->uses_basevertex ||
 100                                 vs_prog_data->uses_baseinstance;
 101
 102    uint32_t elem_count = __builtin_popcount(elements) -
 103       __builtin_popcount(elements_double) / 2;
 104
 105    const uint32_t total_elems =
 106       elem_count + needs_svgs_elem + vs_prog_data->uses_drawid;
 107    if (total_elems == 0)
 108       return;
 109
 110    uint32_t *p;
 111
 112    const uint32_t num_dwords = 1 + total_elems * 2;
 113    p = anv_batch_emitn(&pipeline->batch, num_dwords,
 114                        GENX(3DSTATE_VERTEX_ELEMENTS));
 115    if (!p)
 116       return;
 117    memset(p + 1, 0, (num_dwords - 1) * 4);
 118
 119    for (uint32_t i = 0; i < info->vertexAttributeDescriptionCount; i++) {
 120       const VkVertexInputAttributeDescription *desc =
 121          &info->pVertexAttributeDescriptions[i];
 122       enum isl_format format = anv_get_isl_format(&pipeline->device->info,
 123                                                   desc->format,
 124                                                   VK_IMAGE_ASPECT_COLOR_BIT,
 125                                                   VK_IMAGE_TILING_LINEAR);
 126
 127       assert(desc->binding < MAX_VBS);
 128
 129       if ((elements & (1 << desc->location)) == 0)
 130          continue; /* Binding unused */
 131
 132       uint32_t slot =
 133          __builtin_popcount(elements & ((1 << desc->location) - 1)) -
 134          DIV_ROUND_UP(__builtin_popcount(elements_double &
 135                                         ((1 << desc->location) -1)), 2);
 136
 137       struct GENX(VERTEX_ELEMENT_STATE) element = {
 138          .VertexBufferIndex = desc->binding,
 139          .Valid = true,
 140          .SourceElementFormat = format,
 141          .EdgeFlagEnable = false,
 142          .SourceElementOffset = desc->offset,
 143          .Component0Control = vertex_element_comp_control(format, 0),
 144          .Component1Control = vertex_element_comp_control(format, 1),
 145          .Component2Control = vertex_element_comp_control(format, 2),
 146          .Component3Control = vertex_element_comp_control(format, 3),
 147       };
 148       GENX(VERTEX_ELEMENT_STATE_pack)(NULL, &p[1 + slot * 2], &element);
 149
 150 #if GEN_GEN >= 8
 151       /* On Broadwell and later, we have a separate VF_INSTANCING packet
 152        * that controls instancing.  On Haswell and prior, that's part of
 153        * VERTEX_BUFFER_STATE which we emit later.
 154        */
 155       anv_batch_emit(&pipeline->batch, GENX(3DSTATE_VF_INSTANCING), vfi) {
 156          vfi.InstancingEnable = pipeline->instancing_enable[desc->binding];
 157          vfi.VertexElementIndex = slot;
 158          /* Vulkan so far doesn't have an instance divisor, so
 159           * this is always 1 (ignored if not instancing). */
 160          vfi.InstanceDataStepRate = 1;
 161       }
 162 #endif
 163    }
 164
 165    const uint32_t id_slot = elem_count;
 166    if (needs_svgs_elem) {
 167       /* From the Broadwell PRM for the 3D_Vertex_Component_Control enum:
 168        *    "Within a VERTEX_ELEMENT_STATE structure, if a Component
 169        *    Control field is set to something other than VFCOMP_STORE_SRC,
 170        *    no higher-numbered Component Control fields may be set to
 171        *    VFCOMP_STORE_SRC"
 172        *
 173        * This means, that if we have BaseInstance, we need BaseVertex as
 174        * well.  Just do all or nothing.
 175        */
 176       uint32_t base_ctrl = (vs_prog_data->uses_basevertex ||
 177                             vs_prog_data->uses_baseinstance) ?
 178                            VFCOMP_STORE_SRC : VFCOMP_STORE_0;
 179
 180       struct GENX(VERTEX_ELEMENT_STATE) element = {
 181          .VertexBufferIndex = ANV_SVGS_VB_INDEX,
 182          .Valid = true,
 183          .SourceElementFormat = ISL_FORMAT_R32G32_UINT,
 184          .Component0Control = base_ctrl,
 185          .Component1Control = base_ctrl,
 186 #if GEN_GEN >= 8
 187          .Component2Control = VFCOMP_STORE_0,
 188          .Component3Control = VFCOMP_STORE_0,
 189 #else
 190          .Component2Control = VFCOMP_STORE_VID,
 191          .Component3Control = VFCOMP_STORE_IID,
 192 #endif
 193       };
 194       GENX(VERTEX_ELEMENT_STATE_pack)(NULL, &p[1 + id_slot * 2], &element);
 195    }
 196
 197 #if GEN_GEN >= 8
 198    anv_batch_emit(&pipeline->batch, GENX(3DSTATE_VF_SGVS), sgvs) {
 199       sgvs.VertexIDEnable              = vs_prog_data->uses_vertexid;
 200       sgvs.VertexIDComponentNumber     = 2;
 201       sgvs.VertexIDElementOffset       = id_slot;
 202       sgvs.InstanceIDEnable            = vs_prog_data->uses_instanceid;
 203       sgvs.InstanceIDComponentNumber   = 3;
 204       sgvs.InstanceIDElementOffset     = id_slot;
 205    }
 206 #endif
 207
 208    const uint32_t drawid_slot = elem_count + needs_svgs_elem;
 209    if (vs_prog_data->uses_drawid) {
 210       struct GENX(VERTEX_ELEMENT_STATE) element = {
 211          .VertexBufferIndex = ANV_DRAWID_VB_INDEX,
 212          .Valid = true,
 213          .SourceElementFormat = ISL_FORMAT_R32_UINT,
 214          .Component0Control = VFCOMP_STORE_SRC,
 215          .Component1Control = VFCOMP_STORE_0,
 216          .Component2Control = VFCOMP_STORE_0,
 217          .Component3Control = VFCOMP_STORE_0,
 218       };
 219       GENX(VERTEX_ELEMENT_STATE_pack)(NULL,
 220                                       &p[1 + drawid_slot * 2],
 221                                       &element);
 222
 223 #if GEN_GEN >= 8
 224       anv_batch_emit(&pipeline->batch, GENX(3DSTATE_VF_INSTANCING), vfi) {
 225          vfi.VertexElementIndex = drawid_slot;
 226       }
 227 #endif
 228    }
 229 }
 230
 231 void
 232 genX(emit_urb_setup)(struct anv_device *device, struct anv_batch *batch,
 233                      const struct gen_l3_config *l3_config,
 234                      VkShaderStageFlags active_stages,
 235                      const unsigned entry_size[4])
 236 {
 237    const struct gen_device_info *devinfo = &device->info;
 238 #if GEN_IS_HASWELL
 239    const unsigned push_constant_kb = devinfo->gt == 3 ? 32 : 16;
 240 #else
 241    const unsigned push_constant_kb = GEN_GEN >= 8 ? 32 : 16;
 242 #endif
 243
 244    const unsigned urb_size_kb = gen_get_l3_config_urb_size(devinfo, l3_config);
 245
 246    unsigned entries[4];
 247    unsigned start[4];
 248    gen_get_urb_config(devinfo,
 249                       1024 * push_constant_kb, 1024 * urb_size_kb,
 250                       active_stages &
 251                          VK_SHADER_STAGE_TESSELLATION_EVALUATION_BIT,
 252                       active_stages & VK_SHADER_STAGE_GEOMETRY_BIT,
 253                       entry_size, entries, start);
 254
 255 #if GEN_GEN == 7 && !GEN_IS_HASWELL
 256    /* From the IVB PRM Vol. 2, Part 1, Section 3.2.1:
 257     *
 258     *    "A PIPE_CONTROL with Post-Sync Operation set to 1h and a depth stall
 259     *    needs to be sent just prior to any 3DSTATE_VS, 3DSTATE_URB_VS,
 260     *    3DSTATE_CONSTANT_VS, 3DSTATE_BINDING_TABLE_POINTER_VS,
 261     *    3DSTATE_SAMPLER_STATE_POINTER_VS command.  Only one PIPE_CONTROL
 262     *    needs to be sent before any combination of VS associated 3DSTATE."
 263     */
 264    anv_batch_emit(batch, GEN7_PIPE_CONTROL, pc) {
 265       pc.DepthStallEnable  = true;
 266       pc.PostSyncOperation = WriteImmediateData;
 267       pc.Address           = (struct anv_address) { &device->workaround_bo, 0 };
 268    }
 269 #endif
 270
 271    for (int i = 0; i <= MESA_SHADER_GEOMETRY; i++) {
 272       anv_batch_emit(batch, GENX(3DSTATE_URB_VS), urb) {
 273          urb._3DCommandSubOpcode      += i;
 274          urb.VSURBStartingAddress      = start[i];
 275          urb.VSURBEntryAllocationSize  = entry_size[i] - 1;
 276          urb.VSNumberofURBEntries      = entries[i];
 277       }
 278    }
 279 }
 280
 281 static inline void
 282 emit_urb_setup(struct anv_pipeline *pipeline)
 283 {
 284    unsigned entry_size[4];
 285    for (int i = MESA_SHADER_VERTEX; i <= MESA_SHADER_GEOMETRY; i++) {
 286       const struct brw_vue_prog_data *prog_data =
 287          !anv_pipeline_has_stage(pipeline, i) ? NULL :
 288          (const struct brw_vue_prog_data *) pipeline->shaders[i]->prog_data;
 289
 290       entry_size[i] = prog_data ? prog_data->urb_entry_size : 1;
 291    }
 292
 293    genX(emit_urb_setup)(pipeline->device, &pipeline->batch,
 294                         pipeline->urb.l3_config,
 295                         pipeline->active_stages, entry_size);
 296 }
 297
 298 static void
 299 emit_3dstate_sbe(struct anv_pipeline *pipeline)
 300 {
 301    const struct brw_wm_prog_data *wm_prog_data = get_wm_prog_data(pipeline);
 302
 303    if (!anv_pipeline_has_stage(pipeline, MESA_SHADER_FRAGMENT)) {
 304       anv_batch_emit(&pipeline->batch, GENX(3DSTATE_SBE), sbe);
 305 #if GEN_GEN >= 8
 306       anv_batch_emit(&pipeline->batch, GENX(3DSTATE_SBE_SWIZ), sbe);
 307 #endif
 308       return;
 309    }
 310
 311    const struct brw_vue_map *fs_input_map =
 312       &anv_pipeline_get_last_vue_prog_data(pipeline)->vue_map;
 313
 314    struct GENX(3DSTATE_SBE) sbe = {
 315       GENX(3DSTATE_SBE_header),
 316       .AttributeSwizzleEnable = true,
 317       .PointSpriteTextureCoordinateOrigin = UPPERLEFT,
 318       .NumberofSFOutputAttributes = wm_prog_data->num_varying_inputs,
 319       .ConstantInterpolationEnable = wm_prog_data->flat_inputs,
 320    };
 321
 322 #if GEN_GEN >= 9
 323    for (unsigned i = 0; i < 32; i++)
 324       sbe.AttributeActiveComponentFormat[i] = ACF_XYZW;
 325 #endif
 326
 327 #if GEN_GEN >= 8
 328    /* On Broadwell, they broke 3DSTATE_SBE into two packets */
 329    struct GENX(3DSTATE_SBE_SWIZ) swiz = {
 330       GENX(3DSTATE_SBE_SWIZ_header),
 331    };
 332 #else
 333 #  define swiz sbe
 334 #endif
 335
 336    /* Skip the VUE header and position slots by default */
 337    unsigned urb_entry_read_offset = 1;
 338    int max_source_attr = 0;
 339    for (int attr = 0; attr < VARYING_SLOT_MAX; attr++) {
 340       int input_index = wm_prog_data->urb_setup[attr];
 341
 342       if (input_index < 0)
 343          continue;
 344
 345       /* gl_Layer is stored in the VUE header */
 346       if (attr == VARYING_SLOT_LAYER) {
 347          urb_entry_read_offset = 0;
 348          continue;
 349       }
 350
 351       if (attr == VARYING_SLOT_PNTC) {
 352          sbe.PointSpriteTextureCoordinateEnable = 1 << input_index;
 353          continue;
 354       }
 355
 356       const int slot = fs_input_map->varying_to_slot[attr];
 357
 358       if (input_index >= 16)
 359          continue;
 360
 361       if (slot == -1) {
 362          /* This attribute does not exist in the VUE--that means that the
 363           * vertex shader did not write to it.  It could be that it's a
 364           * regular varying read by the fragment shader but not written by
 365           * the vertex shader or it's gl_PrimitiveID. In the first case the
 366           * value is undefined, in the second it needs to be
 367           * gl_PrimitiveID.
 368           */
 369          swiz.Attribute[input_index].ConstantSource = PRIM_ID;
 370          swiz.Attribute[input_index].ComponentOverrideX = true;
 371          swiz.Attribute[input_index].ComponentOverrideY = true;
 372          swiz.Attribute[input_index].ComponentOverrideZ = true;
 373          swiz.Attribute[input_index].ComponentOverrideW = true;
 374       } else {
 375          /* We have to subtract two slots to accout for the URB entry output
 376           * read offset in the VS and GS stages.
 377           */
 378          assert(slot >= 2);
 379          const int source_attr = slot - 2 * urb_entry_read_offset;
 380          max_source_attr = MAX2(max_source_attr, source_attr);
 381          swiz.Attribute[input_index].SourceAttribute = source_attr;
 382       }
 383    }
 384
 385    sbe.VertexURBEntryReadOffset = urb_entry_read_offset;
 386    sbe.VertexURBEntryReadLength = DIV_ROUND_UP(max_source_attr + 1, 2);
 387 #if GEN_GEN >= 8
 388    sbe.ForceVertexURBEntryReadOffset = true;
 389    sbe.ForceVertexURBEntryReadLength = true;
 390 #endif
 391
 392    uint32_t *dw = anv_batch_emit_dwords(&pipeline->batch,
 393                                         GENX(3DSTATE_SBE_length));
 394    if (!dw)
 395       return;
 396    GENX(3DSTATE_SBE_pack)(&pipeline->batch, dw, &sbe);
 397
 398 #if GEN_GEN >= 8
 399    dw = anv_batch_emit_dwords(&pipeline->batch, GENX(3DSTATE_SBE_SWIZ_length));
 400    if (!dw)
 401       return;
 402    GENX(3DSTATE_SBE_SWIZ_pack)(&pipeline->batch, dw, &swiz);
 403 #endif
 404 }
 405
 406 static const uint32_t vk_to_gen_cullmode[] = {
 407    [VK_CULL_MODE_NONE]                       = CULLMODE_NONE,
 408    [VK_CULL_MODE_FRONT_BIT]                  = CULLMODE_FRONT,
 409    [VK_CULL_MODE_BACK_BIT]                   = CULLMODE_BACK,
 410    [VK_CULL_MODE_FRONT_AND_BACK]             = CULLMODE_BOTH
 411 };
 412
 413 static const uint32_t vk_to_gen_fillmode[] = {
 414    [VK_POLYGON_MODE_FILL]                    = FILL_MODE_SOLID,
 415    [VK_POLYGON_MODE_LINE]                    = FILL_MODE_WIREFRAME,
 416    [VK_POLYGON_MODE_POINT]                   = FILL_MODE_POINT,
 417 };
 418
 419 static const uint32_t vk_to_gen_front_face[] = {
 420    [VK_FRONT_FACE_COUNTER_CLOCKWISE]         = 1,
 421    [VK_FRONT_FACE_CLOCKWISE]                 = 0
 422 };
 423
 424 static void
 425 emit_rs_state(struct anv_pipeline *pipeline,
 426               const VkPipelineRasterizationStateCreateInfo *rs_info,
 427               const VkPipelineMultisampleStateCreateInfo *ms_info,
 428               const struct anv_render_pass *pass,
 429               const struct anv_subpass *subpass)
 430 {
 431    struct GENX(3DSTATE_SF) sf = {
 432       GENX(3DSTATE_SF_header),
 433    };
 434
 435    sf.ViewportTransformEnable = true;
 436    sf.StatisticsEnable = true;
 437    sf.TriangleStripListProvokingVertexSelect = 0;
 438    sf.LineStripListProvokingVertexSelect = 0;
 439    sf.TriangleFanProvokingVertexSelect = 1;
 440
 441    const struct brw_vue_prog_data *last_vue_prog_data =
 442       anv_pipeline_get_last_vue_prog_data(pipeline);
 443
 444    if (last_vue_prog_data->vue_map.slots_valid & VARYING_BIT_PSIZ) {
 445       sf.PointWidthSource = Vertex;
 446    } else {
 447       sf.PointWidthSource = State;
 448       sf.PointWidth = 1.0;
 449    }
 450
 451 #if GEN_GEN >= 8
 452    struct GENX(3DSTATE_RASTER) raster = {
 453       GENX(3DSTATE_RASTER_header),
 454    };
 455 #else
 456 #  define raster sf
 457 #endif
 458
 459    /* For details on 3DSTATE_RASTER multisample state, see the BSpec table
 460     * "Multisample Modes State".
 461     */
 462 #if GEN_GEN >= 8
 463    raster.DXMultisampleRasterizationEnable = true;
 464    /* NOTE: 3DSTATE_RASTER::ForcedSampleCount affects the BDW and SKL PMA fix
 465     * computations.  If we ever set this bit to a different value, they will
 466     * need to be updated accordingly.
 467     */
 468    raster.ForcedSampleCount = FSC_NUMRASTSAMPLES_0;
 469    raster.ForceMultisampling = false;
 470 #else
 471    raster.MultisampleRasterizationMode =
 472       (ms_info && ms_info->rasterizationSamples > 1) ?
 473       MSRASTMODE_ON_PATTERN : MSRASTMODE_OFF_PIXEL;
 474 #endif
 475
 476    raster.FrontWinding = vk_to_gen_front_face[rs_info->frontFace];
 477    raster.CullMode = vk_to_gen_cullmode[rs_info->cullMode];
 478    raster.FrontFaceFillMode = vk_to_gen_fillmode[rs_info->polygonMode];
 479    raster.BackFaceFillMode = vk_to_gen_fillmode[rs_info->polygonMode];
 480    raster.ScissorRectangleEnable = true;
 481
 482 #if GEN_GEN >= 9
 483    /* GEN9+ splits ViewportZClipTestEnable into near and far enable bits */
 484    raster.ViewportZFarClipTestEnable = !pipeline->depth_clamp_enable;
 485    raster.ViewportZNearClipTestEnable = !pipeline->depth_clamp_enable;
 486 #elif GEN_GEN >= 8
 487    raster.ViewportZClipTestEnable = !pipeline->depth_clamp_enable;
 488 #endif
 489
 490    raster.GlobalDepthOffsetEnableSolid = rs_info->depthBiasEnable;
 491    raster.GlobalDepthOffsetEnableWireframe = rs_info->depthBiasEnable;
 492    raster.GlobalDepthOffsetEnablePoint = rs_info->depthBiasEnable;
 493
 494 #if GEN_GEN == 7
 495    /* Gen7 requires that we provide the depth format in 3DSTATE_SF so that it
 496     * can get the depth offsets correct.
 497     */
 498    if (subpass->depth_stencil_attachment.attachment < pass->attachment_count) {
 499       VkFormat vk_format =
 500          pass->attachments[subpass->depth_stencil_attachment.attachment].format;
 501       assert(vk_format_is_depth_or_stencil(vk_format));
 502       if (vk_format_aspects(vk_format) & VK_IMAGE_ASPECT_DEPTH_BIT) {
 503          enum isl_format isl_format =
 504             anv_get_isl_format(&pipeline->device->info, vk_format,
 505                                VK_IMAGE_ASPECT_DEPTH_BIT,
 506                                VK_IMAGE_TILING_OPTIMAL);
 507          sf.DepthBufferSurfaceFormat =
 508             isl_format_get_depth_format(isl_format, false);
 509       }
 510    }
 511 #endif
 512
 513 #if GEN_GEN >= 8
 514    GENX(3DSTATE_SF_pack)(NULL, pipeline->gen8.sf, &sf);
 515    GENX(3DSTATE_RASTER_pack)(NULL, pipeline->gen8.raster, &raster);
 516 #else
 517 #  undef raster
 518    GENX(3DSTATE_SF_pack)(NULL, &pipeline->gen7.sf, &sf);
 519 #endif
 520 }
 521
 522 static void
 523 emit_ms_state(struct anv_pipeline *pipeline,
 524               const VkPipelineMultisampleStateCreateInfo *info)
 525 {
 526    uint32_t samples = 1;
 527    uint32_t log2_samples = 0;
 528
 529    /* From the Vulkan 1.0 spec:
 530     *    If pSampleMask is NULL, it is treated as if the mask has all bits
 531     *    enabled, i.e. no coverage is removed from fragments.
 532     *
 533     * 3DSTATE_SAMPLE_MASK.SampleMask is 16 bits.
 534     */
 535 #if GEN_GEN >= 8
 536    uint32_t sample_mask = 0xffff;
 537 #else
 538    uint32_t sample_mask = 0xff;
 539 #endif
 540
 541    if (info) {
 542       samples = info->rasterizationSamples;
 543       log2_samples = __builtin_ffs(samples) - 1;
 544    }
 545
 546    if (info && info->pSampleMask)
 547       sample_mask &= info->pSampleMask[0];
 548
 549    anv_batch_emit(&pipeline->batch, GENX(3DSTATE_MULTISAMPLE), ms) {
 550       ms.NumberofMultisamples       = log2_samples;
 551
 552 #if GEN_GEN >= 8
 553       /* The PRM says that this bit is valid only for DX9:
 554        *
 555        *    SW can choose to set this bit only for DX9 API. DX10/OGL API's
 556        *    should not have any effect by setting or not setting this bit.
 557        */
 558       ms.PixelPositionOffsetEnable  = false;
 559       ms.PixelLocation              = CENTER;
 560 #else
 561       ms.PixelLocation              = PIXLOC_CENTER;
 562
 563       switch (samples) {
 564       case 1:
 565          GEN_SAMPLE_POS_1X(ms.Sample);
 566          break;
 567       case 2:
 568          GEN_SAMPLE_POS_2X(ms.Sample);
 569          break;
 570       case 4:
 571          GEN_SAMPLE_POS_4X(ms.Sample);
 572          break;
 573       case 8:
 574          GEN_SAMPLE_POS_8X(ms.Sample);
 575          break;
 576       default:
 577          break;
 578       }
 579 #endif
 580    }
 581
 582    anv_batch_emit(&pipeline->batch, GENX(3DSTATE_SAMPLE_MASK), sm) {
 583       sm.SampleMask = sample_mask;
 584    }
 585 }
 586
 587 static const uint32_t vk_to_gen_logic_op[] = {
 588    [VK_LOGIC_OP_COPY]                        = LOGICOP_COPY,
 589    [VK_LOGIC_OP_CLEAR]                       = LOGICOP_CLEAR,
 590    [VK_LOGIC_OP_AND]                         = LOGICOP_AND,
 591    [VK_LOGIC_OP_AND_REVERSE]                 = LOGICOP_AND_REVERSE,
 592    [VK_LOGIC_OP_AND_INVERTED]                = LOGICOP_AND_INVERTED,
 593    [VK_LOGIC_OP_NO_OP]                       = LOGICOP_NOOP,
 594    [VK_LOGIC_OP_XOR]                         = LOGICOP_XOR,
 595    [VK_LOGIC_OP_OR]                          = LOGICOP_OR,
 596    [VK_LOGIC_OP_NOR]                         = LOGICOP_NOR,
 597    [VK_LOGIC_OP_EQUIVALENT]                  = LOGICOP_EQUIV,
 598    [VK_LOGIC_OP_INVERT]                      = LOGICOP_INVERT,
 599    [VK_LOGIC_OP_OR_REVERSE]                  = LOGICOP_OR_REVERSE,
 600    [VK_LOGIC_OP_COPY_INVERTED]               = LOGICOP_COPY_INVERTED,
 601    [VK_LOGIC_OP_OR_INVERTED]                 = LOGICOP_OR_INVERTED,
 602    [VK_LOGIC_OP_NAND]                        = LOGICOP_NAND,
 603    [VK_LOGIC_OP_SET]                         = LOGICOP_SET,
 604 };
 605
 606 static const uint32_t vk_to_gen_blend[] = {
 607    [VK_BLEND_FACTOR_ZERO]                    = BLENDFACTOR_ZERO,
 608    [VK_BLEND_FACTOR_ONE]                     = BLENDFACTOR_ONE,
 609    [VK_BLEND_FACTOR_SRC_COLOR]               = BLENDFACTOR_SRC_COLOR,
 610    [VK_BLEND_FACTOR_ONE_MINUS_SRC_COLOR]     = BLENDFACTOR_INV_SRC_COLOR,
 611    [VK_BLEND_FACTOR_DST_COLOR]               = BLENDFACTOR_DST_COLOR,
 612    [VK_BLEND_FACTOR_ONE_MINUS_DST_COLOR]     = BLENDFACTOR_INV_DST_COLOR,
 613    [VK_BLEND_FACTOR_SRC_ALPHA]               = BLENDFACTOR_SRC_ALPHA,
 614    [VK_BLEND_FACTOR_ONE_MINUS_SRC_ALPHA]     = BLENDFACTOR_INV_SRC_ALPHA,
 615    [VK_BLEND_FACTOR_DST_ALPHA]               = BLENDFACTOR_DST_ALPHA,
 616    [VK_BLEND_FACTOR_ONE_MINUS_DST_ALPHA]     = BLENDFACTOR_INV_DST_ALPHA,
 617    [VK_BLEND_FACTOR_CONSTANT_COLOR]          = BLENDFACTOR_CONST_COLOR,
 618    [VK_BLEND_FACTOR_ONE_MINUS_CONSTANT_COLOR]= BLENDFACTOR_INV_CONST_COLOR,
 619    [VK_BLEND_FACTOR_CONSTANT_ALPHA]          = BLENDFACTOR_CONST_ALPHA,
 620    [VK_BLEND_FACTOR_ONE_MINUS_CONSTANT_ALPHA]= BLENDFACTOR_INV_CONST_ALPHA,
 621    [VK_BLEND_FACTOR_SRC_ALPHA_SATURATE]      = BLENDFACTOR_SRC_ALPHA_SATURATE,
 622    [VK_BLEND_FACTOR_SRC1_COLOR]              = BLENDFACTOR_SRC1_COLOR,
 623    [VK_BLEND_FACTOR_ONE_MINUS_SRC1_COLOR]    = BLENDFACTOR_INV_SRC1_COLOR,
 624    [VK_BLEND_FACTOR_SRC1_ALPHA]              = BLENDFACTOR_SRC1_ALPHA,
 625    [VK_BLEND_FACTOR_ONE_MINUS_SRC1_ALPHA]    = BLENDFACTOR_INV_SRC1_ALPHA,
 626 };
 627
 628 static const uint32_t vk_to_gen_blend_op[] = {
 629    [VK_BLEND_OP_ADD]                         = BLENDFUNCTION_ADD,
 630    [VK_BLEND_OP_SUBTRACT]                    = BLENDFUNCTION_SUBTRACT,
 631    [VK_BLEND_OP_REVERSE_SUBTRACT]            = BLENDFUNCTION_REVERSE_SUBTRACT,
 632    [VK_BLEND_OP_MIN]                         = BLENDFUNCTION_MIN,
 633    [VK_BLEND_OP_MAX]                         = BLENDFUNCTION_MAX,
 634 };
 635
 636 static const uint32_t vk_to_gen_compare_op[] = {
 637    [VK_COMPARE_OP_NEVER]                        = PREFILTEROPNEVER,
 638    [VK_COMPARE_OP_LESS]                         = PREFILTEROPLESS,
 639    [VK_COMPARE_OP_EQUAL]                        = PREFILTEROPEQUAL,
 640    [VK_COMPARE_OP_LESS_OR_EQUAL]                = PREFILTEROPLEQUAL,
 641    [VK_COMPARE_OP_GREATER]                      = PREFILTEROPGREATER,
 642    [VK_COMPARE_OP_NOT_EQUAL]                    = PREFILTEROPNOTEQUAL,
 643    [VK_COMPARE_OP_GREATER_OR_EQUAL]             = PREFILTEROPGEQUAL,
 644    [VK_COMPARE_OP_ALWAYS]                       = PREFILTEROPALWAYS,
 645 };
 646
 647 static const uint32_t vk_to_gen_stencil_op[] = {
 648    [VK_STENCIL_OP_KEEP]                         = STENCILOP_KEEP,
 649    [VK_STENCIL_OP_ZERO]                         = STENCILOP_ZERO,
 650    [VK_STENCIL_OP_REPLACE]                      = STENCILOP_REPLACE,
 651    [VK_STENCIL_OP_INCREMENT_AND_CLAMP]          = STENCILOP_INCRSAT,
 652    [VK_STENCIL_OP_DECREMENT_AND_CLAMP]          = STENCILOP_DECRSAT,
 653    [VK_STENCIL_OP_INVERT]                       = STENCILOP_INVERT,
 654    [VK_STENCIL_OP_INCREMENT_AND_WRAP]           = STENCILOP_INCR,
 655    [VK_STENCIL_OP_DECREMENT_AND_WRAP]           = STENCILOP_DECR,
 656 };
 657
 658 /* This function sanitizes the VkStencilOpState by looking at the compare ops
 659  * and trying to determine whether or not a given stencil op can ever actually
 660  * occur.  Stencil ops which can never occur are set to VK_STENCIL_OP_KEEP.
 661  * This function returns true if, after sanitation, any of the stencil ops are
 662  * set to something other than VK_STENCIL_OP_KEEP.
 663  */
 664 static bool
 665 sanitize_stencil_face(VkStencilOpState *face,
 666                       VkCompareOp depthCompareOp)
 667 {
 668    /* If compareOp is ALWAYS then the stencil test will never fail and failOp
 669     * will never happen.  Set failOp to KEEP in this case.
 670     */
 671    if (face->compareOp == VK_COMPARE_OP_ALWAYS)
 672       face->failOp = VK_STENCIL_OP_KEEP;
 673
 674    /* If compareOp is NEVER or depthCompareOp is NEVER then one of the depth
 675     * or stencil tests will fail and passOp will never happen.
 676     */
 677    if (face->compareOp == VK_COMPARE_OP_NEVER ||
 678        depthCompareOp == VK_COMPARE_OP_NEVER)
 679       face->passOp = VK_STENCIL_OP_KEEP;
 680
 681    /* If compareOp is NEVER or depthCompareOp is ALWAYS then either the
 682     * stencil test will fail or the depth test will pass.  In either case,
 683     * depthFailOp will never happen.
 684     */
 685    if (face->compareOp == VK_COMPARE_OP_NEVER ||
 686        depthCompareOp == VK_COMPARE_OP_ALWAYS)
 687       face->depthFailOp = VK_STENCIL_OP_KEEP;
 688
 689    return face->failOp != VK_STENCIL_OP_KEEP ||
 690           face->depthFailOp != VK_STENCIL_OP_KEEP ||
 691           face->passOp != VK_STENCIL_OP_KEEP;
 692 }
 693
 694 /* Intel hardware is fairly sensitive to whether or not depth/stencil writes
 695  * are enabled.  In the presence of discards, it's fairly easy to get into the
 696  * non-promoted case which means a fairly big performance hit.  From the Iron
 697  * Lake PRM, Vol 2, pt. 1, section 8.4.3.2, "Early Depth Test Cases":
 698  *
 699  *    "Non-promoted depth (N) is active whenever the depth test can be done
 700  *    early but it cannot determine whether or not to write source depth to
 701  *    the depth buffer, therefore the depth write must be performed post pixel
 702  *    shader. This includes cases where the pixel shader can kill pixels,
 703  *    including via sampler chroma key, as well as cases where the alpha test
 704  *    function is enabled, which kills pixels based on a programmable alpha
 705  *    test. In this case, even if the depth test fails, the pixel cannot be
 706  *    killed if a stencil write is indicated. Whether or not the stencil write
 707  *    happens depends on whether or not the pixel is killed later. In these
 708  *    cases if stencil test fails and stencil writes are off, the pixels can
 709  *    also be killed early. If stencil writes are enabled, the pixels must be
 710  *    treated as Computed depth (described above)."
 711  *
 712  * The same thing as mentioned in the stencil case can happen in the depth
 713  * case as well if it thinks it writes depth but, thanks to the depth test
 714  * being GL_EQUAL, the write doesn't actually matter.  A little extra work
 715  * up-front to try and disable depth and stencil writes can make a big
 716  * difference.
 717  *
 718  * Unfortunately, the way depth and stencil testing is specified, there are
 719  * many case where, regardless of depth/stencil writes being enabled, nothing
 720  * actually gets written due to some other bit of state being set.  This
 721  * function attempts to "sanitize" the depth stencil state and disable writes
 722  * and sometimes even testing whenever possible.
 723  */
 724 static void
 725 sanitize_ds_state(VkPipelineDepthStencilStateCreateInfo *state,
 726                   bool *stencilWriteEnable,
 727                   VkImageAspectFlags ds_aspects)
 728 {
 729    *stencilWriteEnable = state->stencilTestEnable;
 730
 731    /* If the depth test is disabled, we won't be writing anything. */
 732    if (!state->depthTestEnable)
 733       state->depthWriteEnable = false;
 734
 735    /* The Vulkan spec requires that if either depth or stencil is not present,
 736     * the pipeline is to act as if the test silently passes.
 737     */
 738    if (!(ds_aspects & VK_IMAGE_ASPECT_DEPTH_BIT)) {
 739       state->depthWriteEnable = false;
 740       state->depthCompareOp = VK_COMPARE_OP_ALWAYS;
 741    }
 742
 743    if (!(ds_aspects & VK_IMAGE_ASPECT_STENCIL_BIT)) {
 744       *stencilWriteEnable = false;
 745       state->front.compareOp = VK_COMPARE_OP_ALWAYS;
 746       state->back.compareOp = VK_COMPARE_OP_ALWAYS;
 747    }
 748
 749    /* If the stencil test is enabled and always fails, then we will never get
 750     * to the depth test so we can just disable the depth test entirely.
 751     */
 752    if (state->stencilTestEnable &&
 753        state->front.compareOp == VK_COMPARE_OP_NEVER &&
 754        state->back.compareOp == VK_COMPARE_OP_NEVER) {
 755       state->depthTestEnable = false;
 756       state->depthWriteEnable = false;
 757    }
 758
 759    /* If depthCompareOp is EQUAL then the value we would be writing to the
 760     * depth buffer is the same as the value that's already there so there's no
 761     * point in writing it.
 762     */
 763    if (state->depthCompareOp == VK_COMPARE_OP_EQUAL)
 764       state->depthWriteEnable = false;
 765
 766    /* If the stencil ops are such that we don't actually ever modify the
 767     * stencil buffer, we should disable writes.
 768     */
 769    if (!sanitize_stencil_face(&state->front, state->depthCompareOp) &&
 770        !sanitize_stencil_face(&state->back, state->depthCompareOp))
 771       *stencilWriteEnable = false;
 772
 773    /* If the depth test always passes and we never write out depth, that's the
 774     * same as if the depth test is disabled entirely.
 775     */
 776    if (state->depthCompareOp == VK_COMPARE_OP_ALWAYS &&
 777        !state->depthWriteEnable)
 778       state->depthTestEnable = false;
 779
 780    /* If the stencil test always passes and we never write out stencil, that's
 781     * the same as if the stencil test is disabled entirely.
 782     */
 783    if (state->front.compareOp == VK_COMPARE_OP_ALWAYS &&
 784        state->back.compareOp == VK_COMPARE_OP_ALWAYS &&
 785        !*stencilWriteEnable)
 786       state->stencilTestEnable = false;
 787 }
 788
 789 static void
 790 emit_ds_state(struct anv_pipeline *pipeline,
 791               const VkPipelineDepthStencilStateCreateInfo *pCreateInfo,
 792               const struct anv_render_pass *pass,
 793               const struct anv_subpass *subpass)
 794 {
 795 #if GEN_GEN == 7
 796 #  define depth_stencil_dw pipeline->gen7.depth_stencil_state
 797 #elif GEN_GEN == 8
 798 #  define depth_stencil_dw pipeline->gen8.wm_depth_stencil
 799 #else
 800 #  define depth_stencil_dw pipeline->gen9.wm_depth_stencil
 801 #endif
 802
 803    if (pCreateInfo == NULL) {
 804       /* We're going to OR this together with the dynamic state.  We need
 805        * to make sure it's initialized to something useful.
 806        */
 807       pipeline->writes_stencil = false;
 808       pipeline->stencil_test_enable = false;
 809       pipeline->writes_depth = false;
 810       pipeline->depth_test_enable = false;
 811       memset(depth_stencil_dw, 0, sizeof(depth_stencil_dw));
 812       return;
 813    }
 814
 815    VkImageAspectFlags ds_aspects = 0;
 816    if (subpass->depth_stencil_attachment.attachment != VK_ATTACHMENT_UNUSED) {
 817       VkFormat depth_stencil_format =
 818          pass->attachments[subpass->depth_stencil_attachment.attachment].format;
 819       ds_aspects = vk_format_aspects(depth_stencil_format);
 820    }
 821
 822    VkPipelineDepthStencilStateCreateInfo info = *pCreateInfo;
 823    sanitize_ds_state(&info, &pipeline->writes_stencil, ds_aspects);
 824    pipeline->stencil_test_enable = info.stencilTestEnable;
 825    pipeline->writes_depth = info.depthWriteEnable;
 826    pipeline->depth_test_enable = info.depthTestEnable;
 827
 828    /* VkBool32 depthBoundsTestEnable; // optional (depth_bounds_test) */
 829
 830 #if GEN_GEN <= 7
 831    struct GENX(DEPTH_STENCIL_STATE) depth_stencil = {
 832 #else
 833    struct GENX(3DSTATE_WM_DEPTH_STENCIL) depth_stencil = {
 834 #endif
 835       .DepthTestEnable = info.depthTestEnable,
 836       .DepthBufferWriteEnable = info.depthWriteEnable,
 837       .DepthTestFunction = vk_to_gen_compare_op[info.depthCompareOp],
 838       .DoubleSidedStencilEnable = true,
 839
 840       .StencilTestEnable = info.stencilTestEnable,
 841       .StencilFailOp = vk_to_gen_stencil_op[info.front.failOp],
 842       .StencilPassDepthPassOp = vk_to_gen_stencil_op[info.front.passOp],
 843       .StencilPassDepthFailOp = vk_to_gen_stencil_op[info.front.depthFailOp],
 844       .StencilTestFunction = vk_to_gen_compare_op[info.front.compareOp],
 845       .BackfaceStencilFailOp = vk_to_gen_stencil_op[info.back.failOp],
 846       .BackfaceStencilPassDepthPassOp = vk_to_gen_stencil_op[info.back.passOp],
 847       .BackfaceStencilPassDepthFailOp =vk_to_gen_stencil_op[info.back.depthFailOp],
 848       .BackfaceStencilTestFunction = vk_to_gen_compare_op[info.back.compareOp],
 849    };
 850
 851 #if GEN_GEN <= 7
 852    GENX(DEPTH_STENCIL_STATE_pack)(NULL, depth_stencil_dw, &depth_stencil);
 853 #else
 854    GENX(3DSTATE_WM_DEPTH_STENCIL_pack)(NULL, depth_stencil_dw, &depth_stencil);
 855 #endif
 856 }
 857
 858 static void
 859 emit_cb_state(struct anv_pipeline *pipeline,
 860               const VkPipelineColorBlendStateCreateInfo *info,
 861               const VkPipelineMultisampleStateCreateInfo *ms_info)
 862 {
 863    struct anv_device *device = pipeline->device;
 864
 865    const uint32_t num_dwords = GENX(BLEND_STATE_length);
 866    pipeline->blend_state =
 867       anv_state_pool_alloc(&device->dynamic_state_pool, num_dwords * 4, 64);
 868
 869    struct GENX(BLEND_STATE) blend_state = {
 870 #if GEN_GEN >= 8
 871       .AlphaToCoverageEnable = ms_info && ms_info->alphaToCoverageEnable,
 872       .AlphaToOneEnable = ms_info && ms_info->alphaToOneEnable,
 873 #else
 874       /* Make sure it gets zeroed */
 875       .Entry = { { 0, }, },
 876 #endif
 877    };
 878
 879    /* Default everything to disabled */
 880    for (uint32_t i = 0; i < 8; i++) {
 881       blend_state.Entry[i].WriteDisableAlpha = true;
 882       blend_state.Entry[i].WriteDisableRed = true;
 883       blend_state.Entry[i].WriteDisableGreen = true;
 884       blend_state.Entry[i].WriteDisableBlue = true;
 885    }
 886
 887    uint32_t surface_count = 0;
 888    struct anv_pipeline_bind_map *map;
 889    if (anv_pipeline_has_stage(pipeline, MESA_SHADER_FRAGMENT)) {
 890       map = &pipeline->shaders[MESA_SHADER_FRAGMENT]->bind_map;
 891       surface_count = map->surface_count;
 892    }
 893
 894    bool has_writeable_rt = false;
 895    for (unsigned i = 0; i < surface_count; i++) {
 896       struct anv_pipeline_binding *binding = &map->surface_to_descriptor[i];
 897
 898       /* All color attachments are at the beginning of the binding table */
 899       if (binding->set != ANV_DESCRIPTOR_SET_COLOR_ATTACHMENTS)
 900          break;
 901
 902       /* We can have at most 8 attachments */
 903       assert(i < 8);
 904
 905       if (info == NULL || binding->index >= info->attachmentCount)
 906          continue;
 907
 908       assert(binding->binding == 0);
 909       const VkPipelineColorBlendAttachmentState *a =
 910          &info->pAttachments[binding->index];
 911
 912       blend_state.Entry[i] = (struct GENX(BLEND_STATE_ENTRY)) {
 913 #if GEN_GEN < 8
 914          .AlphaToCoverageEnable = ms_info && ms_info->alphaToCoverageEnable,
 915          .AlphaToOneEnable = ms_info && ms_info->alphaToOneEnable,
 916 #endif
 917          .LogicOpEnable = info->logicOpEnable,
 918          .LogicOpFunction = vk_to_gen_logic_op[info->logicOp],
 919          .ColorBufferBlendEnable = a->blendEnable,
 920          .ColorClampRange = COLORCLAMP_RTFORMAT,
 921          .PreBlendColorClampEnable = true,
 922          .PostBlendColorClampEnable = true,
 923          .SourceBlendFactor = vk_to_gen_blend[a->srcColorBlendFactor],
 924          .DestinationBlendFactor = vk_to_gen_blend[a->dstColorBlendFactor],
 925          .ColorBlendFunction = vk_to_gen_blend_op[a->colorBlendOp],
 926          .SourceAlphaBlendFactor = vk_to_gen_blend[a->srcAlphaBlendFactor],
 927          .DestinationAlphaBlendFactor = vk_to_gen_blend[a->dstAlphaBlendFactor],
 928          .AlphaBlendFunction = vk_to_gen_blend_op[a->alphaBlendOp],
 929          .WriteDisableAlpha = !(a->colorWriteMask & VK_COLOR_COMPONENT_A_BIT),
 930          .WriteDisableRed = !(a->colorWriteMask & VK_COLOR_COMPONENT_R_BIT),
 931          .WriteDisableGreen = !(a->colorWriteMask & VK_COLOR_COMPONENT_G_BIT),
 932          .WriteDisableBlue = !(a->colorWriteMask & VK_COLOR_COMPONENT_B_BIT),
 933       };
 934
 935       if (a->srcColorBlendFactor != a->srcAlphaBlendFactor ||
 936           a->dstColorBlendFactor != a->dstAlphaBlendFactor ||
 937           a->colorBlendOp != a->alphaBlendOp) {
 938 #if GEN_GEN >= 8
 939          blend_state.IndependentAlphaBlendEnable = true;
 940 #else
 941          blend_state.Entry[i].IndependentAlphaBlendEnable = true;
 942 #endif
 943       }
 944
 945       if (a->colorWriteMask != 0)
 946          has_writeable_rt = true;
 947
 948       /* Our hardware applies the blend factor prior to the blend function
 949        * regardless of what function is used.  Technically, this means the
 950        * hardware can do MORE than GL or Vulkan specify.  However, it also
 951        * means that, for MIN and MAX, we have to stomp the blend factor to
 952        * ONE to make it a no-op.
 953        */
 954       if (a->colorBlendOp == VK_BLEND_OP_MIN ||
 955           a->colorBlendOp == VK_BLEND_OP_MAX) {
 956          blend_state.Entry[i].SourceBlendFactor = BLENDFACTOR_ONE;
 957          blend_state.Entry[i].DestinationBlendFactor = BLENDFACTOR_ONE;
 958       }
 959       if (a->alphaBlendOp == VK_BLEND_OP_MIN ||
 960           a->alphaBlendOp == VK_BLEND_OP_MAX) {
 961          blend_state.Entry[i].SourceAlphaBlendFactor = BLENDFACTOR_ONE;
 962          blend_state.Entry[i].DestinationAlphaBlendFactor = BLENDFACTOR_ONE;
 963       }
 964    }
 965
 966 #if GEN_GEN >= 8
 967    struct GENX(BLEND_STATE_ENTRY) *bs0 = &blend_state.Entry[0];
 968    anv_batch_emit(&pipeline->batch, GENX(3DSTATE_PS_BLEND), blend) {
 969       blend.AlphaToCoverageEnable         = blend_state.AlphaToCoverageEnable;
 970       blend.HasWriteableRT                = has_writeable_rt;
 971       blend.ColorBufferBlendEnable        = bs0->ColorBufferBlendEnable;
 972       blend.SourceAlphaBlendFactor        = bs0->SourceAlphaBlendFactor;
 973       blend.DestinationAlphaBlendFactor   = bs0->DestinationAlphaBlendFactor;
 974       blend.SourceBlendFactor             = bs0->SourceBlendFactor;
 975       blend.DestinationBlendFactor        = bs0->DestinationBlendFactor;
 976       blend.AlphaTestEnable               = false;
 977       blend.IndependentAlphaBlendEnable   =
 978          blend_state.IndependentAlphaBlendEnable;
 979    }
 980 #else
 981    (void)has_writeable_rt;
 982 #endif
 983
 984    GENX(BLEND_STATE_pack)(NULL, pipeline->blend_state.map, &blend_state);
 985    anv_state_flush(device, pipeline->blend_state);
 986
 987    anv_batch_emit(&pipeline->batch, GENX(3DSTATE_BLEND_STATE_POINTERS), bsp) {
 988       bsp.BlendStatePointer      = pipeline->blend_state.offset;
 989 #if GEN_GEN >= 8
 990       bsp.BlendStatePointerValid = true;
 991 #endif
 992    }
 993 }
 994
 995 static void
 996 emit_3dstate_clip(struct anv_pipeline *pipeline,
 997                   const VkPipelineViewportStateCreateInfo *vp_info,
 998                   const VkPipelineRasterizationStateCreateInfo *rs_info)
 999 {
1000    const struct brw_wm_prog_data *wm_prog_data = get_wm_prog_data(pipeline);
1001    (void) wm_prog_data;
1002    anv_batch_emit(&pipeline->batch, GENX(3DSTATE_CLIP), clip) {
1003       clip.ClipEnable               = true;
1004       clip.StatisticsEnable         = true;
1005       clip.EarlyCullEnable          = true;
1006       clip.APIMode                  = APIMODE_D3D,
1007       clip.ViewportXYClipTestEnable = true;
1008
1009       clip.ClipMode = CLIPMODE_NORMAL;
1010
1011       clip.TriangleStripListProvokingVertexSelect = 0;
1012       clip.LineStripListProvokingVertexSelect     = 0;
1013       clip.TriangleFanProvokingVertexSelect       = 1;
1014
1015       clip.MinimumPointWidth = 0.125;
1016       clip.MaximumPointWidth = 255.875;
1017       clip.MaximumVPIndex    = (vp_info ? vp_info->viewportCount : 1) - 1;
1018
1019 #if GEN_GEN == 7
1020       clip.FrontWinding            = vk_to_gen_front_face[rs_info->frontFace];
1021       clip.CullMode                = vk_to_gen_cullmode[rs_info->cullMode];
1022       clip.ViewportZClipTestEnable = !pipeline->depth_clamp_enable;
1023       const struct brw_vue_prog_data *last =
1024          anv_pipeline_get_last_vue_prog_data(pipeline);
1025       if (last) {
1026          clip.UserClipDistanceClipTestEnableBitmask = last->clip_distance_mask;
1027          clip.UserClipDistanceCullTestEnableBitmask = last->cull_distance_mask;
1028       }
1029 #else
1030       clip.NonPerspectiveBarycentricEnable = wm_prog_data ?
1031          (wm_prog_data->barycentric_interp_modes & 0x38) != 0 : 0;
1032 #endif
1033    }
1034 }
1035
1036 static void
1037 emit_3dstate_streamout(struct anv_pipeline *pipeline,
1038                        const VkPipelineRasterizationStateCreateInfo *rs_info)
1039 {
1040    anv_batch_emit(&pipeline->batch, GENX(3DSTATE_STREAMOUT), so) {
1041       so.RenderingDisable = rs_info->rasterizerDiscardEnable;
1042    }
1043 }
1044
1045 static inline uint32_t
1046 get_sampler_count(const struct anv_shader_bin *bin)
1047 {
1048    return DIV_ROUND_UP(bin->bind_map.sampler_count, 4);
1049 }
1050
1051 static inline uint32_t
1052 get_binding_table_entry_count(const struct anv_shader_bin *bin)
1053 {
1054    return DIV_ROUND_UP(bin->bind_map.surface_count, 32);
1055 }
1056
1057 static inline struct anv_address
1058 get_scratch_address(struct anv_pipeline *pipeline,
1059                     gl_shader_stage stage,
1060                     const struct anv_shader_bin *bin)
1061 {
1062    return (struct anv_address) {
1063       .bo = anv_scratch_pool_alloc(pipeline->device,
1064                                    &pipeline->device->scratch_pool,
1065                                    stage, bin->prog_data->total_scratch),
1066       .offset = 0,
1067    };
1068 }
1069
1070 static inline uint32_t
1071 get_scratch_space(const struct anv_shader_bin *bin)
1072 {
1073    return ffs(bin->prog_data->total_scratch / 2048);
1074 }
1075
1076 static inline uint32_t
1077 get_urb_output_offset()
1078 {
1079    /* Skip the VUE header and position slots */
1080    return 1;
1081 }
1082
1083 static inline uint32_t
1084 get_urb_output_length(const struct anv_shader_bin *bin)
1085 {
1086    const struct brw_vue_prog_data *prog_data =
1087       (const struct brw_vue_prog_data *)bin->prog_data;
1088
1089    return (prog_data->vue_map.num_slots + 1) / 2 - get_urb_output_offset();
1090 }
1091
1092 static void
1093 emit_3dstate_vs(struct anv_pipeline *pipeline)
1094 {
1095    const struct gen_device_info *devinfo = &pipeline->device->info;
1096    const struct brw_vs_prog_data *vs_prog_data = get_vs_prog_data(pipeline);
1097    const struct anv_shader_bin *vs_bin =
1098       pipeline->shaders[MESA_SHADER_VERTEX];
1099
1100    assert(anv_pipeline_has_stage(pipeline, MESA_SHADER_VERTEX));
1101
1102    anv_batch_emit(&pipeline->batch, GENX(3DSTATE_VS), vs) {
1103       vs.FunctionEnable       = true;
1104       vs.StatisticsEnable     = true;
1105       vs.KernelStartPointer   = vs_bin->kernel.offset;
1106 #if GEN_GEN >= 8
1107       vs.SIMD8DispatchEnable  =
1108          vs_prog_data->base.dispatch_mode == DISPATCH_MODE_SIMD8;
1109 #endif
1110
1111       assert(!vs_prog_data->base.base.use_alt_mode);
1112       vs.SingleVertexDispatch       = false;
1113       vs.VectorMaskEnable           = false;
1114       vs.SamplerCount               = get_sampler_count(vs_bin);
1115       vs.BindingTableEntryCount     = get_binding_table_entry_count(vs_bin);
1116       vs.FloatingPointMode          = IEEE754;
1117       vs.IllegalOpcodeExceptionEnable = false;
1118       vs.SoftwareExceptionEnable    = false;
1119       vs.MaximumNumberofThreads     = devinfo->max_vs_threads - 1;
1120       vs.VertexCacheDisable         = false;
1121
1122       vs.VertexURBEntryReadLength      = vs_prog_data->base.urb_read_length;
1123       vs.VertexURBEntryReadOffset      = 0;
1124       vs.DispatchGRFStartRegisterForURBData =
1125          vs_prog_data->base.base.dispatch_grf_start_reg;
1126
1127 #if GEN_GEN >= 8
1128       vs.VertexURBEntryOutputReadOffset = get_urb_output_offset();
1129       vs.VertexURBEntryOutputLength     = get_urb_output_length(vs_bin);
1130
1131       vs.UserClipDistanceClipTestEnableBitmask =
1132          vs_prog_data->base.clip_distance_mask;
1133       vs.UserClipDistanceCullTestEnableBitmask =
1134          vs_prog_data->base.cull_distance_mask;
1135 #endif
1136
1137       vs.PerThreadScratchSpace   = get_scratch_space(vs_bin);
1138       vs.ScratchSpaceBasePointer =
1139          get_scratch_address(pipeline, MESA_SHADER_VERTEX, vs_bin);
1140    }
1141 }
1142
1143 static void
1144 emit_3dstate_hs_te_ds(struct anv_pipeline *pipeline)
1145 {
1146    if (!anv_pipeline_has_stage(pipeline, MESA_SHADER_TESS_EVAL)) {
1147       anv_batch_emit(&pipeline->batch, GENX(3DSTATE_HS), hs);
1148       anv_batch_emit(&pipeline->batch, GENX(3DSTATE_TE), te);
1149       anv_batch_emit(&pipeline->batch, GENX(3DSTATE_DS), ds);
1150       return;
1151    }
1152
1153    const struct gen_device_info *devinfo = &pipeline->device->info;
1154    const struct anv_shader_bin *tcs_bin =
1155       pipeline->shaders[MESA_SHADER_TESS_CTRL];
1156    const struct anv_shader_bin *tes_bin =
1157       pipeline->shaders[MESA_SHADER_TESS_EVAL];
1158
1159    const struct brw_tcs_prog_data *tcs_prog_data = get_tcs_prog_data(pipeline);
1160    const struct brw_tes_prog_data *tes_prog_data = get_tes_prog_data(pipeline);
1161
1162    anv_batch_emit(&pipeline->batch, GENX(3DSTATE_HS), hs) {
1163       hs.FunctionEnable = true;
1164       hs.StatisticsEnable = true;
1165       hs.KernelStartPointer = tcs_bin->kernel.offset;
1166
1167       hs.SamplerCount = get_sampler_count(tcs_bin);
1168       hs.BindingTableEntryCount = get_binding_table_entry_count(tcs_bin);
1169       hs.MaximumNumberofThreads = devinfo->max_tcs_threads - 1;
1170       hs.IncludeVertexHandles = true;
1171       hs.InstanceCount = tcs_prog_data->instances - 1;
1172
1173       hs.VertexURBEntryReadLength = 0;
1174       hs.VertexURBEntryReadOffset = 0;
1175       hs.DispatchGRFStartRegisterForURBData =
1176          tcs_prog_data->base.base.dispatch_grf_start_reg;
1177
1178       hs.PerThreadScratchSpace = get_scratch_space(tcs_bin);
1179       hs.ScratchSpaceBasePointer =
1180          get_scratch_address(pipeline, MESA_SHADER_TESS_CTRL, tcs_bin);
1181    }
1182
1183    anv_batch_emit(&pipeline->batch, GENX(3DSTATE_TE), te) {
1184       te.Partitioning = tes_prog_data->partitioning;
1185       te.OutputTopology = tes_prog_data->output_topology;
1186       te.TEDomain = tes_prog_data->domain;
1187       te.TEEnable = true;
1188       te.MaximumTessellationFactorOdd = 63.0;
1189       te.MaximumTessellationFactorNotOdd = 64.0;
1190    }
1191
1192    anv_batch_emit(&pipeline->batch, GENX(3DSTATE_DS), ds) {
1193       ds.FunctionEnable = true;
1194       ds.StatisticsEnable = true;
1195       ds.KernelStartPointer = tes_bin->kernel.offset;
1196
1197       ds.SamplerCount = get_sampler_count(tes_bin);
1198       ds.BindingTableEntryCount = get_binding_table_entry_count(tes_bin);
1199       ds.MaximumNumberofThreads = devinfo->max_tes_threads - 1;
1200
1201       ds.ComputeWCoordinateEnable =
1202          tes_prog_data->domain == BRW_TESS_DOMAIN_TRI;
1203
1204       ds.PatchURBEntryReadLength = tes_prog_data->base.urb_read_length;
1205       ds.PatchURBEntryReadOffset = 0;
1206       ds.DispatchGRFStartRegisterForURBData =
1207          tes_prog_data->base.base.dispatch_grf_start_reg;
1208
1209 #if GEN_GEN >= 8
1210       ds.VertexURBEntryOutputReadOffset = 1;
1211       ds.VertexURBEntryOutputLength =
1212          (tes_prog_data->base.vue_map.num_slots + 1) / 2 - 1;
1213
1214       ds.DispatchMode =
1215          tes_prog_data->base.dispatch_mode == DISPATCH_MODE_SIMD8 ?
1216             DISPATCH_MODE_SIMD8_SINGLE_PATCH :
1217             DISPATCH_MODE_SIMD4X2;
1218
1219       ds.UserClipDistanceClipTestEnableBitmask =
1220          tes_prog_data->base.clip_distance_mask;
1221       ds.UserClipDistanceCullTestEnableBitmask =
1222          tes_prog_data->base.cull_distance_mask;
1223 #endif
1224
1225       ds.PerThreadScratchSpace = get_scratch_space(tes_bin);
1226       ds.ScratchSpaceBasePointer =
1227          get_scratch_address(pipeline, MESA_SHADER_TESS_EVAL, tes_bin);
1228    }
1229 }
1230
1231 static void
1232 emit_3dstate_gs(struct anv_pipeline *pipeline)
1233 {
1234    const struct gen_device_info *devinfo = &pipeline->device->info;
1235    const struct anv_shader_bin *gs_bin =
1236       pipeline->shaders[MESA_SHADER_GEOMETRY];
1237
1238    if (!anv_pipeline_has_stage(pipeline, MESA_SHADER_GEOMETRY)) {
1239       anv_batch_emit(&pipeline->batch, GENX(3DSTATE_GS), gs);
1240       return;
1241    }
1242
1243    const struct brw_gs_prog_data *gs_prog_data = get_gs_prog_data(pipeline);
1244
1245    anv_batch_emit(&pipeline->batch, GENX(3DSTATE_GS), gs) {
1246       gs.FunctionEnable          = true;
1247       gs.StatisticsEnable        = true;
1248       gs.KernelStartPointer      = gs_bin->kernel.offset;
1249       gs.DispatchMode            = gs_prog_data->base.dispatch_mode;
1250
1251       gs.SingleProgramFlow       = false;
1252       gs.VectorMaskEnable        = false;
1253       gs.SamplerCount            = get_sampler_count(gs_bin);
1254       gs.BindingTableEntryCount  = get_binding_table_entry_count(gs_bin);
1255       gs.IncludeVertexHandles    = gs_prog_data->base.include_vue_handles;
1256       gs.IncludePrimitiveID      = gs_prog_data->include_primitive_id;
1257
1258       if (GEN_GEN == 8) {
1259          /* Broadwell is weird.  It needs us to divide by 2. */
1260          gs.MaximumNumberofThreads = devinfo->max_gs_threads / 2 - 1;
1261       } else {
1262          gs.MaximumNumberofThreads = devinfo->max_gs_threads - 1;
1263       }
1264
1265       gs.OutputVertexSize        = gs_prog_data->output_vertex_size_hwords * 2 - 1;
1266       gs.OutputTopology          = gs_prog_data->output_topology;
1267       gs.VertexURBEntryReadLength = gs_prog_data->base.urb_read_length;
1268       gs.ControlDataFormat       = gs_prog_data->control_data_format;
1269       gs.ControlDataHeaderSize   = gs_prog_data->control_data_header_size_hwords;
1270       gs.InstanceControl         = MAX2(gs_prog_data->invocations, 1) - 1;
1271 #if GEN_GEN >= 8 || GEN_IS_HASWELL
1272       gs.ReorderMode             = TRAILING;
1273 #else
1274       gs.ReorderEnable           = true;
1275 #endif
1276
1277 #if GEN_GEN >= 8
1278       gs.ExpectedVertexCount     = gs_prog_data->vertices_in;
1279       gs.StaticOutput            = gs_prog_data->static_vertex_count >= 0;
1280       gs.StaticOutputVertexCount = gs_prog_data->static_vertex_count >= 0 ?
1281                                    gs_prog_data->static_vertex_count : 0;
1282 #endif
1283
1284       gs.VertexURBEntryReadOffset = 0;
1285       gs.VertexURBEntryReadLength = gs_prog_data->base.urb_read_length;
1286       gs.DispatchGRFStartRegisterForURBData =
1287          gs_prog_data->base.base.dispatch_grf_start_reg;
1288
1289 #if GEN_GEN >= 8
1290       gs.VertexURBEntryOutputReadOffset = get_urb_output_offset();
1291       gs.VertexURBEntryOutputLength     = get_urb_output_length(gs_bin);
1292
1293       gs.UserClipDistanceClipTestEnableBitmask =
1294          gs_prog_data->base.clip_distance_mask;
1295       gs.UserClipDistanceCullTestEnableBitmask =
1296          gs_prog_data->base.cull_distance_mask;
1297 #endif
1298
1299       gs.PerThreadScratchSpace   = get_scratch_space(gs_bin);
1300       gs.ScratchSpaceBasePointer =
1301          get_scratch_address(pipeline, MESA_SHADER_GEOMETRY, gs_bin);
1302    }
1303 }
1304
1305 static inline bool
1306 has_color_buffer_write_enabled(const struct anv_pipeline *pipeline)
1307 {
1308    const struct anv_shader_bin *shader_bin =
1309       pipeline->shaders[MESA_SHADER_FRAGMENT];
1310    if (!shader_bin)
1311       return false;
1312
1313    const struct anv_pipeline_bind_map *bind_map = &shader_bin->bind_map;
1314    for (int i = 0; i < bind_map->surface_count; i++) {
1315       if (bind_map->surface_to_descriptor[i].set !=
1316           ANV_DESCRIPTOR_SET_COLOR_ATTACHMENTS)
1317          continue;
1318       if (bind_map->surface_to_descriptor[i].index != UINT8_MAX)
1319          return true;
1320    }
1321
1322    return false;
1323 }
1324
1325 static void
1326 emit_3dstate_wm(struct anv_pipeline *pipeline, struct anv_subpass *subpass,
1327                 const VkPipelineMultisampleStateCreateInfo *multisample)
1328 {
1329    const struct brw_wm_prog_data *wm_prog_data = get_wm_prog_data(pipeline);
1330
1331    MAYBE_UNUSED uint32_t samples =
1332       multisample ? multisample->rasterizationSamples : 1;
1333
1334    anv_batch_emit(&pipeline->batch, GENX(3DSTATE_WM), wm) {
1335       wm.StatisticsEnable                    = true;
1336       wm.LineEndCapAntialiasingRegionWidth   = _05pixels;
1337       wm.LineAntialiasingRegionWidth         = _10pixels;
1338       wm.PointRasterizationRule              = RASTRULE_UPPER_RIGHT;
1339
1340       if (anv_pipeline_has_stage(pipeline, MESA_SHADER_FRAGMENT)) {
1341          if (wm_prog_data->early_fragment_tests) {
1342             wm.EarlyDepthStencilControl         = EDSC_PREPS;
1343          } else if (wm_prog_data->has_side_effects) {
1344             wm.EarlyDepthStencilControl         = EDSC_PSEXEC;
1345          } else {
1346             wm.EarlyDepthStencilControl         = EDSC_NORMAL;
1347          }
1348
1349          wm.BarycentricInterpolationMode =
1350             wm_prog_data->barycentric_interp_modes;
1351
1352 #if GEN_GEN < 8
1353          wm.PixelShaderComputedDepthMode  = wm_prog_data->computed_depth_mode;
1354          wm.PixelShaderUsesSourceDepth    = wm_prog_data->uses_src_depth;
1355          wm.PixelShaderUsesSourceW        = wm_prog_data->uses_src_w;
1356          wm.PixelShaderUsesInputCoverageMask = wm_prog_data->uses_sample_mask;
1357
1358          /* If the subpass has a depth or stencil self-dependency, then we
1359           * need to force the hardware to do the depth/stencil write *after*
1360           * fragment shader execution.  Otherwise, the writes may hit memory
1361           * before we get around to fetching from the input attachment and we
1362           * may get the depth or stencil value from the current draw rather
1363           * than the previous one.
1364           */
1365          wm.PixelShaderKillsPixel         = subpass->has_ds_self_dep ||
1366                                             wm_prog_data->uses_kill;
1367
1368          if (wm.PixelShaderComputedDepthMode != PSCDEPTH_OFF ||
1369              wm_prog_data->has_side_effects ||
1370              wm.PixelShaderKillsPixel ||
1371              has_color_buffer_write_enabled(pipeline))
1372             wm.ThreadDispatchEnable = true;
1373
1374          if (samples > 1) {
1375             wm.MultisampleRasterizationMode = MSRASTMODE_ON_PATTERN;
1376             if (wm_prog_data->persample_dispatch) {
1377                wm.MultisampleDispatchMode = MSDISPMODE_PERSAMPLE;
1378             } else {
1379                wm.MultisampleDispatchMode = MSDISPMODE_PERPIXEL;
1380             }
1381          } else {
1382             wm.MultisampleRasterizationMode = MSRASTMODE_OFF_PIXEL;
1383             wm.MultisampleDispatchMode = MSDISPMODE_PERSAMPLE;
1384          }
1385 #endif
1386       }
1387    }
1388 }
1389
1390 static inline bool
1391 is_dual_src_blend_factor(VkBlendFactor factor)
1392 {
1393    return factor == VK_BLEND_FACTOR_SRC1_COLOR ||
1394           factor == VK_BLEND_FACTOR_ONE_MINUS_SRC1_COLOR ||
1395           factor == VK_BLEND_FACTOR_SRC1_ALPHA ||
1396           factor == VK_BLEND_FACTOR_ONE_MINUS_SRC1_ALPHA;
1397 }
1398
1399 static void
1400 emit_3dstate_ps(struct anv_pipeline *pipeline,
1401                 const VkPipelineColorBlendStateCreateInfo *blend)
1402 {
1403    MAYBE_UNUSED const struct gen_device_info *devinfo = &pipeline->device->info;
1404    const struct anv_shader_bin *fs_bin =
1405       pipeline->shaders[MESA_SHADER_FRAGMENT];
1406
1407    if (!anv_pipeline_has_stage(pipeline, MESA_SHADER_FRAGMENT)) {
1408       anv_batch_emit(&pipeline->batch, GENX(3DSTATE_PS), ps) {
1409 #if GEN_GEN == 7
1410          /* Even if no fragments are ever dispatched, gen7 hardware hangs if
1411           * we don't at least set the maximum number of threads.
1412           */
1413          ps.MaximumNumberofThreads = devinfo->max_wm_threads - 1;
1414 #endif
1415       }
1416       return;
1417    }
1418
1419    const struct brw_wm_prog_data *wm_prog_data = get_wm_prog_data(pipeline);
1420
1421 #if GEN_GEN < 8
1422    /* The hardware wedges if you have this bit set but don't turn on any dual
1423     * source blend factors.
1424     */
1425    bool dual_src_blend = false;
1426    if (wm_prog_data->dual_src_blend && blend) {
1427       for (uint32_t i = 0; i < blend->attachmentCount; i++) {
1428          const VkPipelineColorBlendAttachmentState *bstate =
1429             &blend->pAttachments[i];
1430
1431          if (bstate->blendEnable &&
1432              (is_dual_src_blend_factor(bstate->srcColorBlendFactor) ||
1433               is_dual_src_blend_factor(bstate->dstColorBlendFactor) ||
1434               is_dual_src_blend_factor(bstate->srcAlphaBlendFactor) ||
1435               is_dual_src_blend_factor(bstate->dstAlphaBlendFactor))) {
1436             dual_src_blend = true;
1437             break;
1438          }
1439       }
1440    }
1441 #endif
1442
1443    anv_batch_emit(&pipeline->batch, GENX(3DSTATE_PS), ps) {
1444       ps.KernelStartPointer0        = fs_bin->kernel.offset;
1445       ps.KernelStartPointer1        = 0;
1446       ps.KernelStartPointer2        = fs_bin->kernel.offset +
1447                                       wm_prog_data->prog_offset_2;
1448       ps._8PixelDispatchEnable      = wm_prog_data->dispatch_8;
1449       ps._16PixelDispatchEnable     = wm_prog_data->dispatch_16;
1450       ps._32PixelDispatchEnable     = false;
1451
1452       ps.SingleProgramFlow          = false;
1453       ps.VectorMaskEnable           = true;
1454       ps.SamplerCount               = get_sampler_count(fs_bin);
1455       ps.BindingTableEntryCount     = get_binding_table_entry_count(fs_bin);
1456       ps.PushConstantEnable         = wm_prog_data->base.nr_params > 0;
1457       ps.PositionXYOffsetSelect     = wm_prog_data->uses_pos_offset ?
1458                                       POSOFFSET_SAMPLE: POSOFFSET_NONE;
1459 #if GEN_GEN < 8
1460       ps.AttributeEnable            = wm_prog_data->num_varying_inputs > 0;
1461       ps.oMaskPresenttoRenderTarget = wm_prog_data->uses_omask;
1462       ps.DualSourceBlendEnable      = dual_src_blend;
1463 #endif
1464
1465 #if GEN_IS_HASWELL
1466       /* Haswell requires the sample mask to be set in this packet as well
1467        * as in 3DSTATE_SAMPLE_MASK; the values should match.
1468        */
1469       ps.SampleMask                 = 0xff;
1470 #endif
1471
1472 #if GEN_GEN >= 9
1473       ps.MaximumNumberofThreadsPerPSD  = 64 - 1;
1474 #elif GEN_GEN >= 8
1475       ps.MaximumNumberofThreadsPerPSD  = 64 - 2;
1476 #else
1477       ps.MaximumNumberofThreads        = devinfo->max_wm_threads - 1;
1478 #endif
1479
1480       ps.DispatchGRFStartRegisterForConstantSetupData0 =
1481          wm_prog_data->base.dispatch_grf_start_reg;
1482       ps.DispatchGRFStartRegisterForConstantSetupData1 = 0;
1483       ps.DispatchGRFStartRegisterForConstantSetupData2 =
1484          wm_prog_data->dispatch_grf_start_reg_2;
1485
1486       ps.PerThreadScratchSpace   = get_scratch_space(fs_bin);
1487       ps.ScratchSpaceBasePointer =
1488          get_scratch_address(pipeline, MESA_SHADER_FRAGMENT, fs_bin);
1489    }
1490 }
1491
1492 #if GEN_GEN >= 8
1493 static void
1494 emit_3dstate_ps_extra(struct anv_pipeline *pipeline,
1495                       struct anv_subpass *subpass)
1496 {
1497    const struct brw_wm_prog_data *wm_prog_data = get_wm_prog_data(pipeline);
1498
1499    if (!anv_pipeline_has_stage(pipeline, MESA_SHADER_FRAGMENT)) {
1500       anv_batch_emit(&pipeline->batch, GENX(3DSTATE_PS_EXTRA), ps);
1501       return;
1502    }
1503
1504    anv_batch_emit(&pipeline->batch, GENX(3DSTATE_PS_EXTRA), ps) {
1505       ps.PixelShaderValid              = true;
1506       ps.AttributeEnable               = wm_prog_data->num_varying_inputs > 0;
1507       ps.oMaskPresenttoRenderTarget    = wm_prog_data->uses_omask;
1508       ps.PixelShaderIsPerSample        = wm_prog_data->persample_dispatch;
1509       ps.PixelShaderComputedDepthMode  = wm_prog_data->computed_depth_mode;
1510       ps.PixelShaderUsesSourceDepth    = wm_prog_data->uses_src_depth;
1511       ps.PixelShaderUsesSourceW        = wm_prog_data->uses_src_w;
1512
1513       /* If the subpass has a depth or stencil self-dependency, then we need
1514        * to force the hardware to do the depth/stencil write *after* fragment
1515        * shader execution.  Otherwise, the writes may hit memory before we get
1516        * around to fetching from the input attachment and we may get the depth
1517        * or stencil value from the current draw rather than the previous one.
1518        */
1519       ps.PixelShaderKillsPixel         = subpass->has_ds_self_dep ||
1520                                          wm_prog_data->uses_kill;
1521
1522       /* The stricter cross-primitive coherency guarantees that the hardware
1523        * gives us with the "Accesses UAV" bit set for at least one shader stage
1524        * and the "UAV coherency required" bit set on the 3DPRIMITIVE command are
1525        * redundant within the current image, atomic counter and SSBO GL APIs,
1526        * which all have very loose ordering and coherency requirements and
1527        * generally rely on the application to insert explicit barriers when a
1528        * shader invocation is expected to see the memory writes performed by the
1529        * invocations of some previous primitive.  Regardless of the value of
1530        * "UAV coherency required", the "Accesses UAV" bits will implicitly cause
1531        * an in most cases useless DC flush when the lowermost stage with the bit
1532        * set finishes execution.
1533        *
1534        * It would be nice to disable it, but in some cases we can't because on
1535        * Gen8+ it also has an influence on rasterization via the PS UAV-only
1536        * signal (which could be set independently from the coherency mechanism
1537        * in the 3DSTATE_WM command on Gen7), and because in some cases it will
1538        * determine whether the hardware skips execution of the fragment shader
1539        * or not via the ThreadDispatchEnable signal.  However if we know that
1540        * GEN8_PS_BLEND_HAS_WRITEABLE_RT is going to be set and
1541        * GEN8_PSX_PIXEL_SHADER_NO_RT_WRITE is not set it shouldn't make any
1542        * difference so we may just disable it here.
1543        *
1544        * Gen8 hardware tries to compute ThreadDispatchEnable for us but doesn't
1545        * take into account KillPixels when no depth or stencil writes are
1546        * enabled. In order for occlusion queries to work correctly with no
1547        * attachments, we need to force-enable here.
1548        */
1549       if ((wm_prog_data->has_side_effects || wm_prog_data->uses_kill) &&
1550           !has_color_buffer_write_enabled(pipeline))
1551          ps.PixelShaderHasUAV = true;
1552
1553 #if GEN_GEN >= 9
1554       ps.PixelShaderPullsBary    = wm_prog_data->pulls_bary;
1555       ps.InputCoverageMaskState  = wm_prog_data->uses_sample_mask ?
1556                                    ICMS_INNER_CONSERVATIVE : ICMS_NONE;
1557 #else
1558       ps.PixelShaderUsesInputCoverageMask = wm_prog_data->uses_sample_mask;
1559 #endif
1560    }
1561 }
1562
1563 static void
1564 emit_3dstate_vf_topology(struct anv_pipeline *pipeline)
1565 {
1566    anv_batch_emit(&pipeline->batch, GENX(3DSTATE_VF_TOPOLOGY), vft) {
1567       vft.PrimitiveTopologyType = pipeline->topology;
1568    }
1569 }
1570 #endif
1571
1572 static void
1573 emit_3dstate_vf_statistics(struct anv_pipeline *pipeline)
1574 {
1575    anv_batch_emit(&pipeline->batch, GENX(3DSTATE_VF_STATISTICS), vfs) {
1576       vfs.StatisticsEnable = true;
1577    }
1578 }
1579
1580 static void
1581 compute_kill_pixel(struct anv_pipeline *pipeline,
1582                    const VkPipelineMultisampleStateCreateInfo *ms_info,
1583                    const struct anv_subpass *subpass)
1584 {
1585    if (!anv_pipeline_has_stage(pipeline, MESA_SHADER_FRAGMENT)) {
1586       pipeline->kill_pixel = false;
1587       return;
1588    }
1589
1590    const struct brw_wm_prog_data *wm_prog_data = get_wm_prog_data(pipeline);
1591
1592    /* This computes the KillPixel portion of the computation for whether or
1593     * not we want to enable the PMA fix on gen8 or gen9.  It's given by this
1594     * chunk of the giant formula:
1595     *
1596     *    (3DSTATE_PS_EXTRA::PixelShaderKillsPixels ||
1597     *     3DSTATE_PS_EXTRA::oMask Present to RenderTarget ||
1598     *     3DSTATE_PS_BLEND::AlphaToCoverageEnable ||
1599     *     3DSTATE_PS_BLEND::AlphaTestEnable ||
1600     *     3DSTATE_WM_CHROMAKEY::ChromaKeyKillEnable)
1601     *
1602     * 3DSTATE_WM_CHROMAKEY::ChromaKeyKillEnable is always false and so is
1603     * 3DSTATE_PS_BLEND::AlphaTestEnable since Vulkan doesn't have a concept
1604     * of an alpha test.
1605     */
1606    pipeline->kill_pixel =
1607       subpass->has_ds_self_dep || wm_prog_data->uses_kill ||
1608       wm_prog_data->uses_omask ||
1609       (ms_info && ms_info->alphaToCoverageEnable);
1610 }
1611
1612 static VkResult
1613 genX(graphics_pipeline_create)(
1614     VkDevice                                    _device,
1615     struct anv_pipeline_cache *                 cache,
1616     const VkGraphicsPipelineCreateInfo*         pCreateInfo,
1617     const VkAllocationCallbacks*                pAllocator,
1618     VkPipeline*                                 pPipeline)
1619 {
1620    ANV_FROM_HANDLE(anv_device, device, _device);
1621    ANV_FROM_HANDLE(anv_render_pass, pass, pCreateInfo->renderPass);
1622    struct anv_subpass *subpass = &pass->subpasses[pCreateInfo->subpass];
1623    struct anv_pipeline *pipeline;
1624    VkResult result;
1625
1626    assert(pCreateInfo->sType == VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO);
1627
1628    pipeline = vk_alloc2(&device->alloc, pAllocator, sizeof(*pipeline), 8,
1629                          VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
1630    if (pipeline == NULL)
1631       return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY);
1632
1633    result = anv_pipeline_init(pipeline, device, cache,
1634                               pCreateInfo, pAllocator);
1635    if (result != VK_SUCCESS) {
1636       vk_free2(&device->alloc, pAllocator, pipeline);
1637       return result;
1638    }
1639
1640    assert(pCreateInfo->pVertexInputState);
1641    emit_vertex_input(pipeline, pCreateInfo->pVertexInputState);
1642    assert(pCreateInfo->pRasterizationState);
1643    emit_rs_state(pipeline, pCreateInfo->pRasterizationState,
1644                  pCreateInfo->pMultisampleState, pass, subpass);
1645    emit_ms_state(pipeline, pCreateInfo->pMultisampleState);
1646    emit_ds_state(pipeline, pCreateInfo->pDepthStencilState, pass, subpass);
1647    emit_cb_state(pipeline, pCreateInfo->pColorBlendState,
1648                            pCreateInfo->pMultisampleState);
1649    compute_kill_pixel(pipeline, pCreateInfo->pMultisampleState, subpass);
1650
1651    emit_urb_setup(pipeline);
1652
1653    emit_3dstate_clip(pipeline, pCreateInfo->pViewportState,
1654                      pCreateInfo->pRasterizationState);
1655    emit_3dstate_streamout(pipeline, pCreateInfo->pRasterizationState);
1656
1657 #if 0
1658    /* From gen7_vs_state.c */
1659
1660    /**
1661     * From Graphics BSpec: 3D-Media-GPGPU Engine > 3D Pipeline Stages >
1662     * Geometry > Geometry Shader > State:
1663     *
1664     *     "Note: Because of corruption in IVB:GT2, software needs to flush the
1665     *     whole fixed function pipeline when the GS enable changes value in
1666     *     the 3DSTATE_GS."
1667     *
1668     * The hardware architects have clarified that in this context "flush the
1669     * whole fixed function pipeline" means to emit a PIPE_CONTROL with the "CS
1670     * Stall" bit set.
1671     */
1672    if (!brw->is_haswell && !brw->is_baytrail)
1673       gen7_emit_vs_workaround_flush(brw);
1674 #endif
1675
1676    emit_3dstate_vs(pipeline);
1677    emit_3dstate_hs_te_ds(pipeline);
1678    emit_3dstate_gs(pipeline);
1679    emit_3dstate_sbe(pipeline);
1680    emit_3dstate_wm(pipeline, subpass, pCreateInfo->pMultisampleState);
1681    emit_3dstate_ps(pipeline, pCreateInfo->pColorBlendState);
1682 #if GEN_GEN >= 8
1683    emit_3dstate_ps_extra(pipeline, subpass);
1684    emit_3dstate_vf_topology(pipeline);
1685 #endif
1686    emit_3dstate_vf_statistics(pipeline);
1687
1688    *pPipeline = anv_pipeline_to_handle(pipeline);
1689
1690    return pipeline->batch.status;
1691 }
1692
1693 static VkResult
1694 compute_pipeline_create(
1695     VkDevice                                    _device,
1696     struct anv_pipeline_cache *                 cache,
1697     const VkComputePipelineCreateInfo*          pCreateInfo,
1698     const VkAllocationCallbacks*                pAllocator,
1699     VkPipeline*                                 pPipeline)
1700 {
1701    ANV_FROM_HANDLE(anv_device, device, _device);
1702    const struct anv_physical_device *physical_device =
1703       &device->instance->physicalDevice;
1704    const struct gen_device_info *devinfo = &physical_device->info;
1705    struct anv_pipeline *pipeline;
1706    VkResult result;
1707
1708    assert(pCreateInfo->sType == VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO);
1709
1710    pipeline = vk_alloc2(&device->alloc, pAllocator, sizeof(*pipeline), 8,
1711                          VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
1712    if (pipeline == NULL)
1713       return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY);
1714
1715    pipeline->device = device;
1716    pipeline->layout = anv_pipeline_layout_from_handle(pCreateInfo->layout);
1717
1718    pipeline->blend_state.map = NULL;
1719
1720    result = anv_reloc_list_init(&pipeline->batch_relocs,
1721                                 pAllocator ? pAllocator : &device->alloc);
1722    if (result != VK_SUCCESS) {
1723       vk_free2(&device->alloc, pAllocator, pipeline);
1724       return result;
1725    }
1726    pipeline->batch.next = pipeline->batch.start = pipeline->batch_data;
1727    pipeline->batch.end = pipeline->batch.start + sizeof(pipeline->batch_data);
1728    pipeline->batch.relocs = &pipeline->batch_relocs;
1729    pipeline->batch.status = VK_SUCCESS;
1730
1731    /* When we free the pipeline, we detect stages based on the NULL status
1732     * of various prog_data pointers.  Make them NULL by default.
1733     */
1734    memset(pipeline->shaders, 0, sizeof(pipeline->shaders));
1735
1736    pipeline->active_stages = 0;
1737
1738    pipeline->needs_data_cache = false;
1739
1740    assert(pCreateInfo->stage.stage == VK_SHADER_STAGE_COMPUTE_BIT);
1741    ANV_FROM_HANDLE(anv_shader_module, module,  pCreateInfo->stage.module);
1742    result = anv_pipeline_compile_cs(pipeline, cache, pCreateInfo, module,
1743                                     pCreateInfo->stage.pName,
1744                                     pCreateInfo->stage.pSpecializationInfo);
1745    if (result != VK_SUCCESS) {
1746       vk_free2(&device->alloc, pAllocator, pipeline);
1747       return result;
1748    }
1749
1750    const struct brw_cs_prog_data *cs_prog_data = get_cs_prog_data(pipeline);
1751
1752    anv_pipeline_setup_l3_config(pipeline, cs_prog_data->base.total_shared > 0);
1753
1754    uint32_t group_size = cs_prog_data->local_size[0] *
1755       cs_prog_data->local_size[1] * cs_prog_data->local_size[2];
1756    uint32_t remainder = group_size & (cs_prog_data->simd_size - 1);
1757
1758    if (remainder > 0)
1759       pipeline->cs_right_mask = ~0u >> (32 - remainder);
1760    else
1761       pipeline->cs_right_mask = ~0u >> (32 - cs_prog_data->simd_size);
1762
1763    const uint32_t vfe_curbe_allocation =
1764       ALIGN(cs_prog_data->push.per_thread.regs * cs_prog_data->threads +
1765             cs_prog_data->push.cross_thread.regs, 2);
1766
1767    const uint32_t subslices = MAX2(physical_device->subslice_total, 1);
1768
1769    const struct anv_shader_bin *cs_bin =
1770       pipeline->shaders[MESA_SHADER_COMPUTE];
1771
1772    anv_batch_emit(&pipeline->batch, GENX(MEDIA_VFE_STATE), vfe) {
1773 #if GEN_GEN > 7
1774       vfe.StackSize              = 0;
1775 #else
1776       vfe.GPGPUMode              = true;
1777 #endif
1778       vfe.MaximumNumberofThreads =
1779          devinfo->max_cs_threads * subslices - 1;
1780       vfe.NumberofURBEntries     = GEN_GEN <= 7 ? 0 : 2;
1781       vfe.ResetGatewayTimer      = true;
1782 #if GEN_GEN <= 8
1783       vfe.BypassGatewayControl   = true;
1784 #endif
1785       vfe.URBEntryAllocationSize = GEN_GEN <= 7 ? 0 : 2;
1786       vfe.CURBEAllocationSize    = vfe_curbe_allocation;
1787
1788       vfe.PerThreadScratchSpace = get_scratch_space(cs_bin);
1789       vfe.ScratchSpaceBasePointer =
1790          get_scratch_address(pipeline, MESA_SHADER_COMPUTE, cs_bin);
1791    }
1792
1793    struct GENX(INTERFACE_DESCRIPTOR_DATA) desc = {
1794       .KernelStartPointer     = cs_bin->kernel.offset,
1795
1796       .SamplerCount           = get_sampler_count(cs_bin),
1797       .BindingTableEntryCount = get_binding_table_entry_count(cs_bin),
1798       .BarrierEnable          = cs_prog_data->uses_barrier,
1799       .SharedLocalMemorySize  =
1800          encode_slm_size(GEN_GEN, cs_prog_data->base.total_shared),
1801
1802 #if !GEN_IS_HASWELL
1803       .ConstantURBEntryReadOffset = 0,
1804 #endif
1805       .ConstantURBEntryReadLength = cs_prog_data->push.per_thread.regs,
1806 #if GEN_GEN >= 8 || GEN_IS_HASWELL
1807       .CrossThreadConstantDataReadLength =
1808          cs_prog_data->push.cross_thread.regs,
1809 #endif
1810
1811       .NumberofThreadsinGPGPUThreadGroup = cs_prog_data->threads,
1812    };
1813    GENX(INTERFACE_DESCRIPTOR_DATA_pack)(NULL,
1814                                         pipeline->interface_descriptor_data,
1815                                         &desc);
1816
1817    *pPipeline = anv_pipeline_to_handle(pipeline);
1818
1819    return pipeline->batch.status;
1820 }
1821
1822 VkResult genX(CreateGraphicsPipelines)(
1823     VkDevice                                    _device,
1824     VkPipelineCache                             pipelineCache,
1825     uint32_t                                    count,
1826     const VkGraphicsPipelineCreateInfo*         pCreateInfos,
1827     const VkAllocationCallbacks*                pAllocator,
1828     VkPipeline*                                 pPipelines)
1829 {
1830    ANV_FROM_HANDLE(anv_pipeline_cache, pipeline_cache, pipelineCache);
1831
1832    VkResult result = VK_SUCCESS;
1833
1834    unsigned i;
1835    for (i = 0; i < count; i++) {
1836       result = genX(graphics_pipeline_create)(_device,
1837                                               pipeline_cache,
1838                                               &pCreateInfos[i],
1839                                               pAllocator, &pPipelines[i]);
1840
1841       /* Bail out on the first error as it is not obvious what error should be
1842        * report upon 2 different failures. */
1843       if (result != VK_SUCCESS)
1844          break;
1845    }
1846
1847    for (; i < count; i++)
1848       pPipelines[i] = VK_NULL_HANDLE;
1849
1850    return result;
1851 }
1852
1853 VkResult genX(CreateComputePipelines)(
1854     VkDevice                                    _device,
1855     VkPipelineCache                             pipelineCache,
1856     uint32_t                                    count,
1857     const VkComputePipelineCreateInfo*          pCreateInfos,
1858     const VkAllocationCallbacks*                pAllocator,
1859     VkPipeline*                                 pPipelines)
1860 {
1861    ANV_FROM_HANDLE(anv_pipeline_cache, pipeline_cache, pipelineCache);
1862
1863    VkResult result = VK_SUCCESS;
1864
1865    unsigned i;
1866    for (i = 0; i < count; i++) {
1867       result = compute_pipeline_create(_device, pipeline_cache,
1868                                        &pCreateInfos[i],
1869                                        pAllocator, &pPipelines[i]);
1870
1871       /* Bail out on the first error as it is not obvious what error should be
1872        * report upon 2 different failures. */
1873       if (result != VK_SUCCESS)
1874          break;
1875    }
1876
1877    for (; i < count; i++)
1878       pPipelines[i] = VK_NULL_HANDLE;
1879
1880    return result;
1881 }