src/amd/vulkan/radv_image.c

   1 /*
   2  * Copyright © 2016 Red Hat.
   3  * Copyright © 2016 Bas Nieuwenhuizen
   4  *
   5  * based in part on anv driver which is:
   6  * Copyright © 2015 Intel Corporation
   7  *
   8  * Permission is hereby granted, free of charge, to any person obtaining a
   9  * copy of this software and associated documentation files (the "Software"),
  10  * to deal in the Software without restriction, including without limitation
  11  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  12  * and/or sell copies of the Software, and to permit persons to whom the
  13  * Software is furnished to do so, subject to the following conditions:
  14  *
  15  * The above copyright notice and this permission notice (including the next
  16  * paragraph) shall be included in all copies or substantial portions of the
  17  * Software.
  18  *
  19  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  20  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  21  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  22  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  23  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  24  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  25  * IN THE SOFTWARE.
  26  */
  27
  28 #include "radv_private.h"
  29 #include "vk_format.h"
  30 #include "radv_radeon_winsys.h"
  31 #include "sid.h"
  32 #include "util/debug.h"
  33 static unsigned
  34 radv_choose_tiling(struct radv_device *Device,
  35                    const struct radv_image_create_info *create_info)
  36 {
  37         const VkImageCreateInfo *pCreateInfo = create_info->vk_info;
  38
  39         if (pCreateInfo->tiling == VK_IMAGE_TILING_LINEAR) {
  40                 assert(pCreateInfo->samples <= 1);
  41                 return RADEON_SURF_MODE_LINEAR_ALIGNED;
  42         }
  43
  44         /* MSAA resources must be 2D tiled. */
  45         if (pCreateInfo->samples > 1)
  46                 return RADEON_SURF_MODE_2D;
  47
  48         return RADEON_SURF_MODE_2D;
  49 }
  50 static int
  51 radv_init_surface(struct radv_device *device,
  52                   struct radeon_surf *surface,
  53                   const struct radv_image_create_info *create_info)
  54 {
  55         const VkImageCreateInfo *pCreateInfo = create_info->vk_info;
  56         unsigned array_mode = radv_choose_tiling(device, create_info);
  57         const struct vk_format_description *desc =
  58                 vk_format_description(pCreateInfo->format);
  59         bool is_depth, is_stencil, blendable;
  60
  61         is_depth = vk_format_has_depth(desc);
  62         is_stencil = vk_format_has_stencil(desc);
  63         surface->npix_x = pCreateInfo->extent.width;
  64         surface->npix_y = pCreateInfo->extent.height;
  65         surface->npix_z = pCreateInfo->extent.depth;
  66
  67         surface->blk_w = vk_format_get_blockwidth(pCreateInfo->format);
  68         surface->blk_h = vk_format_get_blockheight(pCreateInfo->format);
  69         surface->blk_d = 1;
  70         surface->array_size = pCreateInfo->arrayLayers;
  71         surface->last_level = pCreateInfo->mipLevels - 1;
  72
  73         surface->bpe = vk_format_get_blocksize(pCreateInfo->format);
  74         /* align byte per element on dword */
  75         if (surface->bpe == 3) {
  76                 surface->bpe = 4;
  77         }
  78         surface->nsamples = pCreateInfo->samples ? pCreateInfo->samples : 1;
  79         surface->flags = RADEON_SURF_SET(array_mode, MODE);
  80
  81         switch (pCreateInfo->imageType){
  82         case VK_IMAGE_TYPE_1D:
  83                 if (pCreateInfo->arrayLayers > 1)
  84                         surface->flags |= RADEON_SURF_SET(RADEON_SURF_TYPE_1D_ARRAY, TYPE);
  85                 else
  86                         surface->flags |= RADEON_SURF_SET(RADEON_SURF_TYPE_1D, TYPE);
  87                 break;
  88         case VK_IMAGE_TYPE_2D:
  89                 if (pCreateInfo->arrayLayers > 1)
  90                         surface->flags |= RADEON_SURF_SET(RADEON_SURF_TYPE_2D_ARRAY, TYPE);
  91                 else
  92                         surface->flags |= RADEON_SURF_SET(RADEON_SURF_TYPE_2D, TYPE);
  93                 break;
  94         case VK_IMAGE_TYPE_3D:
  95                 surface->flags |= RADEON_SURF_SET(RADEON_SURF_TYPE_3D, TYPE);
  96                 break;
  97         default:
  98                 unreachable("unhandled image type");
  99         }
 100
 101         if (is_depth) {
 102                 surface->flags |= RADEON_SURF_ZBUFFER;
 103         }
 104
 105         if (is_stencil)
 106                 surface->flags |= RADEON_SURF_SBUFFER |
 107                         RADEON_SURF_HAS_SBUFFER_MIPTREE;
 108
 109         surface->flags |= RADEON_SURF_HAS_TILE_MODE_INDEX;
 110
 111         if ((pCreateInfo->usage & (VK_IMAGE_USAGE_TRANSFER_SRC_BIT |
 112                                    VK_IMAGE_USAGE_STORAGE_BIT)) ||
 113             (pCreateInfo->flags & VK_IMAGE_CREATE_MUTABLE_FORMAT_BIT) ||
 114             (pCreateInfo->tiling == VK_IMAGE_TILING_LINEAR) ||
 115             device->physical_device->rad_info.chip_class < VI ||
 116             create_info->scanout || (device->debug_flags & RADV_DEBUG_NO_DCC) ||
 117             !radv_is_colorbuffer_format_supported(pCreateInfo->format, &blendable))
 118                 surface->flags |= RADEON_SURF_DISABLE_DCC;
 119         if (create_info->scanout)
 120                 surface->flags |= RADEON_SURF_SCANOUT;
 121         return 0;
 122 }
 123 #define ATI_VENDOR_ID 0x1002
 124 static uint32_t si_get_bo_metadata_word1(struct radv_device *device)
 125 {
 126         return (ATI_VENDOR_ID << 16) | device->physical_device->rad_info.pci_id;
 127 }
 128
 129 static inline unsigned
 130 si_tile_mode_index(const struct radv_image *image, unsigned level, bool stencil)
 131 {
 132         if (stencil)
 133                 return image->surface.stencil_tiling_index[level];
 134         else
 135                 return image->surface.tiling_index[level];
 136 }
 137
 138 static unsigned radv_map_swizzle(unsigned swizzle)
 139 {
 140         switch (swizzle) {
 141         case VK_SWIZZLE_Y:
 142                 return V_008F0C_SQ_SEL_Y;
 143         case VK_SWIZZLE_Z:
 144                 return V_008F0C_SQ_SEL_Z;
 145         case VK_SWIZZLE_W:
 146                 return V_008F0C_SQ_SEL_W;
 147         case VK_SWIZZLE_0:
 148                 return V_008F0C_SQ_SEL_0;
 149         case VK_SWIZZLE_1:
 150                 return V_008F0C_SQ_SEL_1;
 151         default: /* VK_SWIZZLE_X */
 152                 return V_008F0C_SQ_SEL_X;
 153         }
 154 }
 155
 156 static void
 157 radv_make_buffer_descriptor(struct radv_device *device,
 158                             struct radv_buffer *buffer,
 159                             VkFormat vk_format,
 160                             unsigned offset,
 161                             unsigned range,
 162                             uint32_t *state)
 163 {
 164         const struct vk_format_description *desc;
 165         unsigned stride;
 166         uint64_t gpu_address = device->ws->buffer_get_va(buffer->bo);
 167         uint64_t va = gpu_address + buffer->offset;
 168         unsigned num_format, data_format;
 169         int first_non_void;
 170         desc = vk_format_description(vk_format);
 171         first_non_void = vk_format_get_first_non_void_channel(vk_format);
 172         stride = desc->block.bits / 8;
 173
 174         num_format = radv_translate_buffer_numformat(desc, first_non_void);
 175         data_format = radv_translate_buffer_dataformat(desc, first_non_void);
 176
 177         va += offset;
 178         state[0] = va;
 179         state[1] = S_008F04_BASE_ADDRESS_HI(va >> 32) |
 180                 S_008F04_STRIDE(stride);
 181         state[2] = range;
 182         state[3] = S_008F0C_DST_SEL_X(radv_map_swizzle(desc->swizzle[0])) |
 183                    S_008F0C_DST_SEL_Y(radv_map_swizzle(desc->swizzle[1])) |
 184                    S_008F0C_DST_SEL_Z(radv_map_swizzle(desc->swizzle[2])) |
 185                    S_008F0C_DST_SEL_W(radv_map_swizzle(desc->swizzle[3])) |
 186                    S_008F0C_NUM_FORMAT(num_format) |
 187                    S_008F0C_DATA_FORMAT(data_format);
 188 }
 189
 190 static void
 191 si_set_mutable_tex_desc_fields(struct radv_device *device,
 192                                struct radv_image *image,
 193                                const struct radeon_surf_level *base_level_info,
 194                                unsigned base_level, unsigned first_level,
 195                                unsigned block_width, bool is_stencil,
 196                                uint32_t *state)
 197 {
 198         uint64_t gpu_address = device->ws->buffer_get_va(image->bo) + image->offset;
 199         uint64_t va = gpu_address + base_level_info->offset;
 200         unsigned pitch = base_level_info->nblk_x * block_width;
 201
 202         state[1] &= C_008F14_BASE_ADDRESS_HI;
 203         state[3] &= C_008F1C_TILING_INDEX;
 204         state[4] &= C_008F20_PITCH;
 205         state[6] &= C_008F28_COMPRESSION_EN;
 206
 207         assert(!(va & 255));
 208
 209         state[0] = va >> 8;
 210         state[1] |= S_008F14_BASE_ADDRESS_HI(va >> 40);
 211         state[3] |= S_008F1C_TILING_INDEX(si_tile_mode_index(image, base_level,
 212                                                              is_stencil));
 213         state[4] |= S_008F20_PITCH(pitch - 1);
 214
 215         if (image->surface.dcc_size && image->surface.level[first_level].dcc_enabled) {
 216                 state[6] |= S_008F28_COMPRESSION_EN(1);
 217                 state[7] = (gpu_address +
 218                             image->dcc_offset +
 219                             base_level_info->dcc_offset) >> 8;
 220         }
 221 }
 222
 223 static unsigned radv_tex_dim(VkImageType image_type, VkImageViewType view_type,
 224                              unsigned nr_layers, unsigned nr_samples, bool is_storage_image)
 225 {
 226         if (view_type == VK_IMAGE_VIEW_TYPE_CUBE || view_type == VK_IMAGE_VIEW_TYPE_CUBE_ARRAY)
 227                 return is_storage_image ? V_008F1C_SQ_RSRC_IMG_2D_ARRAY : V_008F1C_SQ_RSRC_IMG_CUBE;
 228         switch (image_type) {
 229         case VK_IMAGE_TYPE_1D:
 230                 return nr_layers > 1 ? V_008F1C_SQ_RSRC_IMG_1D_ARRAY : V_008F1C_SQ_RSRC_IMG_1D;
 231         case VK_IMAGE_TYPE_2D:
 232                 if (nr_samples > 1)
 233                         return nr_layers > 1 ? V_008F1C_SQ_RSRC_IMG_2D_MSAA_ARRAY : V_008F1C_SQ_RSRC_IMG_2D_MSAA;
 234                 else
 235                         return nr_layers > 1 ? V_008F1C_SQ_RSRC_IMG_2D_ARRAY : V_008F1C_SQ_RSRC_IMG_2D;
 236         case VK_IMAGE_TYPE_3D:
 237                 if (view_type == VK_IMAGE_VIEW_TYPE_3D)
 238                         return V_008F1C_SQ_RSRC_IMG_3D;
 239                 else
 240                         return V_008F1C_SQ_RSRC_IMG_2D_ARRAY;
 241         default:
 242                 unreachable("illegale image type");
 243         }
 244 }
 245 /**
 246  * Build the sampler view descriptor for a texture.
 247  */
 248 static void
 249 si_make_texture_descriptor(struct radv_device *device,
 250                            struct radv_image *image,
 251                            bool sampler,
 252                            VkImageViewType view_type,
 253                            VkFormat vk_format,
 254                            const VkComponentMapping *mapping,
 255                            unsigned first_level, unsigned last_level,
 256                            unsigned first_layer, unsigned last_layer,
 257                            unsigned width, unsigned height, unsigned depth,
 258                            uint32_t *state,
 259                            uint32_t *fmask_state)
 260 {
 261         const struct vk_format_description *desc;
 262         enum vk_swizzle swizzle[4];
 263         int first_non_void;
 264         unsigned num_format, data_format, type;
 265
 266         desc = vk_format_description(vk_format);
 267
 268         if (desc->colorspace == VK_FORMAT_COLORSPACE_ZS) {
 269                 const unsigned char swizzle_xxxx[4] = {0, 0, 0, 0};
 270                 vk_format_compose_swizzles(mapping, swizzle_xxxx, swizzle);
 271         } else {
 272                 vk_format_compose_swizzles(mapping, desc->swizzle, swizzle);
 273         }
 274
 275         first_non_void = vk_format_get_first_non_void_channel(vk_format);
 276
 277         num_format = radv_translate_tex_numformat(vk_format, desc, first_non_void);
 278         if (num_format == ~0) {
 279                 num_format = 0;
 280         }
 281
 282         data_format = radv_translate_tex_dataformat(vk_format, desc, first_non_void);
 283         if (data_format == ~0) {
 284                 data_format = 0;
 285         }
 286
 287         type = radv_tex_dim(image->type, view_type, image->array_size, image->samples,
 288                             (image->usage & VK_IMAGE_USAGE_STORAGE_BIT));
 289         if (type == V_008F1C_SQ_RSRC_IMG_1D_ARRAY) {
 290                 height = 1;
 291                 depth = image->array_size;
 292         } else if (type == V_008F1C_SQ_RSRC_IMG_2D_ARRAY ||
 293                    type == V_008F1C_SQ_RSRC_IMG_2D_MSAA_ARRAY) {
 294                 if (view_type != VK_IMAGE_VIEW_TYPE_3D)
 295                         depth = image->array_size;
 296         } else if (type == V_008F1C_SQ_RSRC_IMG_CUBE)
 297                 depth = image->array_size / 6;
 298
 299         state[0] = 0;
 300         state[1] = (S_008F14_DATA_FORMAT(data_format) |
 301                     S_008F14_NUM_FORMAT(num_format));
 302         state[2] = (S_008F18_WIDTH(width - 1) |
 303                     S_008F18_HEIGHT(height - 1));
 304         state[3] = (S_008F1C_DST_SEL_X(radv_map_swizzle(swizzle[0])) |
 305                     S_008F1C_DST_SEL_Y(radv_map_swizzle(swizzle[1])) |
 306                     S_008F1C_DST_SEL_Z(radv_map_swizzle(swizzle[2])) |
 307                     S_008F1C_DST_SEL_W(radv_map_swizzle(swizzle[3])) |
 308                     S_008F1C_BASE_LEVEL(image->samples > 1 ?
 309                                         0 : first_level) |
 310                     S_008F1C_LAST_LEVEL(image->samples > 1 ?
 311                                         util_logbase2(image->samples) :
 312                                         last_level) |
 313                     S_008F1C_POW2_PAD(image->levels > 1) |
 314                     S_008F1C_TYPE(type));
 315         state[4] = S_008F20_DEPTH(depth - 1);
 316         state[5] = (S_008F24_BASE_ARRAY(first_layer) |
 317                     S_008F24_LAST_ARRAY(last_layer));
 318         state[6] = 0;
 319         state[7] = 0;
 320
 321         if (image->dcc_offset) {
 322                 unsigned swap = radv_translate_colorswap(vk_format, FALSE);
 323
 324                 state[6] = S_008F28_ALPHA_IS_ON_MSB(swap <= 1);
 325         } else {
 326                 /* The last dword is unused by hw. The shader uses it to clear
 327                  * bits in the first dword of sampler state.
 328                  */
 329                 if (device->physical_device->rad_info.chip_class <= CIK && image->samples <= 1) {
 330                         if (first_level == last_level)
 331                                 state[7] = C_008F30_MAX_ANISO_RATIO;
 332                         else
 333                                 state[7] = 0xffffffff;
 334                 }
 335         }
 336
 337         /* Initialize the sampler view for FMASK. */
 338         if (image->fmask.size) {
 339                 uint32_t fmask_format;
 340                 uint64_t gpu_address = device->ws->buffer_get_va(image->bo);
 341                 uint64_t va;
 342
 343                 va = gpu_address + image->offset + image->fmask.offset;
 344
 345                 switch (image->samples) {
 346                 case 2:
 347                         fmask_format = V_008F14_IMG_DATA_FORMAT_FMASK8_S2_F2;
 348                         break;
 349                 case 4:
 350                         fmask_format = V_008F14_IMG_DATA_FORMAT_FMASK8_S4_F4;
 351                         break;
 352                 case 8:
 353                         fmask_format = V_008F14_IMG_DATA_FORMAT_FMASK32_S8_F8;
 354                         break;
 355                 default:
 356                         assert(0);
 357                         fmask_format = V_008F14_IMG_DATA_FORMAT_INVALID;
 358                 }
 359
 360                 fmask_state[0] = va >> 8;
 361                 fmask_state[1] = S_008F14_BASE_ADDRESS_HI(va >> 40) |
 362                         S_008F14_DATA_FORMAT(fmask_format) |
 363                         S_008F14_NUM_FORMAT(V_008F14_IMG_NUM_FORMAT_UINT);
 364                 fmask_state[2] = S_008F18_WIDTH(width - 1) |
 365                         S_008F18_HEIGHT(height - 1);
 366                 fmask_state[3] = S_008F1C_DST_SEL_X(V_008F1C_SQ_SEL_X) |
 367                         S_008F1C_DST_SEL_Y(V_008F1C_SQ_SEL_X) |
 368                         S_008F1C_DST_SEL_Z(V_008F1C_SQ_SEL_X) |
 369                         S_008F1C_DST_SEL_W(V_008F1C_SQ_SEL_X) |
 370                         S_008F1C_TILING_INDEX(image->fmask.tile_mode_index) |
 371                         S_008F1C_TYPE(radv_tex_dim(image->type, view_type, 1, 0, false));
 372                 fmask_state[4] = S_008F20_DEPTH(depth - 1) |
 373                         S_008F20_PITCH(image->fmask.pitch_in_pixels - 1);
 374                 fmask_state[5] = S_008F24_BASE_ARRAY(first_layer) |
 375                         S_008F24_LAST_ARRAY(last_layer);
 376                 fmask_state[6] = 0;
 377                 fmask_state[7] = 0;
 378         }
 379 }
 380
 381 static void
 382 radv_query_opaque_metadata(struct radv_device *device,
 383                            struct radv_image *image,
 384                            struct radeon_bo_metadata *md)
 385 {
 386         static const VkComponentMapping fixedmapping;
 387         uint32_t desc[8], i;
 388
 389         /* Metadata image format format version 1:
 390          * [0] = 1 (metadata format identifier)
 391          * [1] = (VENDOR_ID << 16) | PCI_ID
 392          * [2:9] = image descriptor for the whole resource
 393          *         [2] is always 0, because the base address is cleared
 394          *         [9] is the DCC offset bits [39:8] from the beginning of
 395          *             the buffer
 396          * [10:10+LAST_LEVEL] = mipmap level offset bits [39:8] for each level
 397          */
 398         md->metadata[0] = 1; /* metadata image format version 1 */
 399
 400         /* TILE_MODE_INDEX is ambiguous without a PCI ID. */
 401         md->metadata[1] = si_get_bo_metadata_word1(device);
 402
 403
 404         si_make_texture_descriptor(device, image, true,
 405                                    (VkImageViewType)image->type, image->vk_format,
 406                                    &fixedmapping, 0, image->levels - 1, 0,
 407                                    image->array_size,
 408                                    image->extent.width, image->extent.height,
 409                                    image->extent.depth,
 410                                    desc, NULL);
 411
 412         si_set_mutable_tex_desc_fields(device, image, &image->surface.level[0], 0, 0,
 413                                        image->surface.blk_w, false, desc);
 414
 415         /* Clear the base address and set the relative DCC offset. */
 416         desc[0] = 0;
 417         desc[1] &= C_008F14_BASE_ADDRESS_HI;
 418         desc[7] = image->dcc_offset >> 8;
 419
 420         /* Dwords [2:9] contain the image descriptor. */
 421         memcpy(&md->metadata[2], desc, sizeof(desc));
 422
 423         /* Dwords [10:..] contain the mipmap level offsets. */
 424         for (i = 0; i <= image->levels - 1; i++)
 425                 md->metadata[10+i] = image->surface.level[i].offset >> 8;
 426
 427         md->size_metadata = (11 + image->levels - 1) * 4;
 428 }
 429
 430 void
 431 radv_init_metadata(struct radv_device *device,
 432                    struct radv_image *image,
 433                    struct radeon_bo_metadata *metadata)
 434 {
 435         struct radeon_surf *surface = &image->surface;
 436
 437         memset(metadata, 0, sizeof(*metadata));
 438         metadata->microtile = surface->level[0].mode >= RADEON_SURF_MODE_1D ?
 439                 RADEON_LAYOUT_TILED : RADEON_LAYOUT_LINEAR;
 440         metadata->macrotile = surface->level[0].mode >= RADEON_SURF_MODE_2D ?
 441                 RADEON_LAYOUT_TILED : RADEON_LAYOUT_LINEAR;
 442         metadata->pipe_config = surface->pipe_config;
 443         metadata->bankw = surface->bankw;
 444         metadata->bankh = surface->bankh;
 445         metadata->tile_split = surface->tile_split;
 446         metadata->mtilea = surface->mtilea;
 447         metadata->num_banks = surface->num_banks;
 448         metadata->stride = surface->level[0].pitch_bytes;
 449         metadata->scanout = (surface->flags & RADEON_SURF_SCANOUT) != 0;
 450
 451         radv_query_opaque_metadata(device, image, metadata);
 452 }
 453
 454 /* The number of samples can be specified independently of the texture. */
 455 static void
 456 radv_image_get_fmask_info(struct radv_device *device,
 457                           struct radv_image *image,
 458                           unsigned nr_samples,
 459                           struct radv_fmask_info *out)
 460 {
 461         /* FMASK is allocated like an ordinary texture. */
 462         struct radeon_surf fmask = image->surface;
 463
 464         memset(out, 0, sizeof(*out));
 465
 466         fmask.bo_alignment = 0;
 467         fmask.bo_size = 0;
 468         fmask.nsamples = 1;
 469         fmask.flags |= RADEON_SURF_FMASK;
 470
 471         /* Force 2D tiling if it wasn't set. This may occur when creating
 472          * FMASK for MSAA resolve on R6xx. On R6xx, the single-sample
 473          * destination buffer must have an FMASK too. */
 474         fmask.flags = RADEON_SURF_CLR(fmask.flags, MODE);
 475         fmask.flags |= RADEON_SURF_SET(RADEON_SURF_MODE_2D, MODE);
 476
 477         fmask.flags |= RADEON_SURF_HAS_TILE_MODE_INDEX;
 478
 479         switch (nr_samples) {
 480         case 2:
 481         case 4:
 482                 fmask.bpe = 1;
 483                 break;
 484         case 8:
 485                 fmask.bpe = 4;
 486                 break;
 487         default:
 488                 return;
 489         }
 490
 491         device->ws->surface_init(device->ws, &fmask);
 492         assert(fmask.level[0].mode == RADEON_SURF_MODE_2D);
 493
 494         out->slice_tile_max = (fmask.level[0].nblk_x * fmask.level[0].nblk_y) / 64;
 495         if (out->slice_tile_max)
 496                 out->slice_tile_max -= 1;
 497
 498         out->tile_mode_index = fmask.tiling_index[0];
 499         out->pitch_in_pixels = fmask.level[0].nblk_x;
 500         out->bank_height = fmask.bankh;
 501         out->alignment = MAX2(256, fmask.bo_alignment);
 502         out->size = fmask.bo_size;
 503 }
 504
 505 static void
 506 radv_image_alloc_fmask(struct radv_device *device,
 507                        struct radv_image *image)
 508 {
 509         radv_image_get_fmask_info(device, image, image->samples, &image->fmask);
 510
 511         image->fmask.offset = align64(image->size, image->fmask.alignment);
 512         image->size = image->fmask.offset + image->fmask.size;
 513         image->alignment = MAX2(image->alignment, image->fmask.alignment);
 514 }
 515
 516 static void
 517 radv_image_get_cmask_info(struct radv_device *device,
 518                           struct radv_image *image,
 519                           struct radv_cmask_info *out)
 520 {
 521         unsigned pipe_interleave_bytes = device->physical_device->rad_info.pipe_interleave_bytes;
 522         unsigned num_pipes = device->physical_device->rad_info.num_tile_pipes;
 523         unsigned cl_width, cl_height;
 524
 525         switch (num_pipes) {
 526         case 2:
 527                 cl_width = 32;
 528                 cl_height = 16;
 529                 break;
 530         case 4:
 531                 cl_width = 32;
 532                 cl_height = 32;
 533                 break;
 534         case 8:
 535                 cl_width = 64;
 536                 cl_height = 32;
 537                 break;
 538         case 16: /* Hawaii */
 539                 cl_width = 64;
 540                 cl_height = 64;
 541                 break;
 542         default:
 543                 assert(0);
 544                 return;
 545         }
 546
 547         unsigned base_align = num_pipes * pipe_interleave_bytes;
 548
 549         unsigned width = align(image->surface.npix_x, cl_width*8);
 550         unsigned height = align(image->surface.npix_y, cl_height*8);
 551         unsigned slice_elements = (width * height) / (8*8);
 552
 553         /* Each element of CMASK is a nibble. */
 554         unsigned slice_bytes = slice_elements / 2;
 555
 556         out->slice_tile_max = (width * height) / (128*128);
 557         if (out->slice_tile_max)
 558                 out->slice_tile_max -= 1;
 559
 560         out->alignment = MAX2(256, base_align);
 561         out->size = (image->type == VK_IMAGE_TYPE_3D ? image->extent.depth : image->array_size) *
 562                     align(slice_bytes, base_align);
 563 }
 564
 565 static void
 566 radv_image_alloc_cmask(struct radv_device *device,
 567                        struct radv_image *image)
 568 {
 569         radv_image_get_cmask_info(device, image, &image->cmask);
 570
 571         image->cmask.offset = align64(image->size, image->cmask.alignment);
 572         /* + 8 for storing the clear values */
 573         image->clear_value_offset = image->cmask.offset + image->cmask.size;
 574         image->size = image->cmask.offset + image->cmask.size + 8;
 575         image->alignment = MAX2(image->alignment, image->cmask.alignment);
 576 }
 577
 578 static void
 579 radv_image_alloc_dcc(struct radv_device *device,
 580                        struct radv_image *image)
 581 {
 582         image->dcc_offset = align64(image->size, image->surface.dcc_alignment);
 583         /* + 8 for storing the clear values */
 584         image->clear_value_offset = image->dcc_offset + image->surface.dcc_size;
 585         image->size = image->dcc_offset + image->surface.dcc_size + 8;
 586         image->alignment = MAX2(image->alignment, image->surface.dcc_alignment);
 587 }
 588
 589 static unsigned
 590 radv_image_get_htile_size(struct radv_device *device,
 591                           struct radv_image *image)
 592 {
 593         unsigned cl_width, cl_height, width, height;
 594         unsigned slice_elements, slice_bytes, base_align;
 595         unsigned num_pipes = device->physical_device->rad_info.num_tile_pipes;
 596         unsigned pipe_interleave_bytes = device->physical_device->rad_info.pipe_interleave_bytes;
 597
 598         /* Overalign HTILE on P2 configs to work around GPU hangs in
 599          * piglit/depthstencil-render-miplevels 585.
 600          *
 601          * This has been confirmed to help Kabini & Stoney, where the hangs
 602          * are always reproducible. I think I have seen the test hang
 603          * on Carrizo too, though it was very rare there.
 604          */
 605         if (device->physical_device->rad_info.chip_class >= CIK && num_pipes < 4)
 606                 num_pipes = 4;
 607
 608         switch (num_pipes) {
 609         case 1:
 610                 cl_width = 32;
 611                 cl_height = 16;
 612                 break;
 613         case 2:
 614                 cl_width = 32;
 615                 cl_height = 32;
 616                 break;
 617         case 4:
 618                 cl_width = 64;
 619                 cl_height = 32;
 620                 break;
 621         case 8:
 622                 cl_width = 64;
 623                 cl_height = 64;
 624                 break;
 625         case 16:
 626                 cl_width = 128;
 627                 cl_height = 64;
 628                 break;
 629         default:
 630                 assert(0);
 631                 return 0;
 632         }
 633
 634         width = align(image->surface.npix_x, cl_width * 8);
 635         height = align(image->surface.npix_y, cl_height * 8);
 636
 637         slice_elements = (width * height) / (8 * 8);
 638         slice_bytes = slice_elements * 4;
 639
 640         base_align = num_pipes * pipe_interleave_bytes;
 641
 642         image->htile.pitch = width;
 643         image->htile.height = height;
 644         image->htile.xalign = cl_width * 8;
 645         image->htile.yalign = cl_height * 8;
 646
 647         return image->array_size *
 648                 align(slice_bytes, base_align);
 649 }
 650
 651 static void
 652 radv_image_alloc_htile(struct radv_device *device,
 653                        struct radv_image *image)
 654 {
 655         if (device->debug_flags & RADV_DEBUG_NO_HIZ)
 656                 return;
 657
 658         if (image->array_size > 1 || image->levels > 1)
 659                 return;
 660
 661         image->htile.size = radv_image_get_htile_size(device, image);
 662
 663         if (!image->htile.size)
 664                 return;
 665
 666         image->htile.offset = align64(image->size, 32768);
 667
 668         /* + 8 for storing the clear values */
 669         image->clear_value_offset = image->htile.offset + image->htile.size;
 670         image->size = image->htile.offset + image->htile.size + 8;
 671         image->alignment = align64(image->alignment, 32768);
 672 }
 673
 674 VkResult
 675 radv_image_create(VkDevice _device,
 676                   const struct radv_image_create_info *create_info,
 677                   const VkAllocationCallbacks* alloc,
 678                   VkImage *pImage)
 679 {
 680         RADV_FROM_HANDLE(radv_device, device, _device);
 681         const VkImageCreateInfo *pCreateInfo = create_info->vk_info;
 682         struct radv_image *image = NULL;
 683         bool can_cmask_dcc = false;
 684         assert(pCreateInfo->sType == VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO);
 685
 686         radv_assert(pCreateInfo->mipLevels > 0);
 687         radv_assert(pCreateInfo->arrayLayers > 0);
 688         radv_assert(pCreateInfo->samples > 0);
 689         radv_assert(pCreateInfo->extent.width > 0);
 690         radv_assert(pCreateInfo->extent.height > 0);
 691         radv_assert(pCreateInfo->extent.depth > 0);
 692
 693         image = vk_alloc2(&device->alloc, alloc, sizeof(*image), 8,
 694                             VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
 695         if (!image)
 696                 return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY);
 697
 698         memset(image, 0, sizeof(*image));
 699         image->type = pCreateInfo->imageType;
 700         image->extent = pCreateInfo->extent;
 701         image->vk_format = pCreateInfo->format;
 702         image->levels = pCreateInfo->mipLevels;
 703         image->array_size = pCreateInfo->arrayLayers;
 704         image->samples = pCreateInfo->samples;
 705         image->tiling = pCreateInfo->tiling;
 706         image->usage = pCreateInfo->usage;
 707
 708         image->exclusive = pCreateInfo->sharingMode == VK_SHARING_MODE_EXCLUSIVE;
 709         if (pCreateInfo->sharingMode == VK_SHARING_MODE_CONCURRENT) {
 710                 for (uint32_t i = 0; i < pCreateInfo->queueFamilyIndexCount; ++i)
 711                         image->queue_family_mask |= 1u << pCreateInfo->pQueueFamilyIndices[i];
 712         }
 713
 714         radv_init_surface(device, &image->surface, create_info);
 715
 716         device->ws->surface_init(device->ws, &image->surface);
 717
 718         image->size = image->surface.bo_size;
 719         image->alignment = image->surface.bo_alignment;
 720
 721         if (image->exclusive || image->queue_family_mask == 1)
 722                 can_cmask_dcc = true;
 723
 724         if ((pCreateInfo->usage & VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT) &&
 725             image->surface.dcc_size && can_cmask_dcc)
 726                 radv_image_alloc_dcc(device, image);
 727         else
 728                 image->surface.dcc_size = 0;
 729
 730         if ((pCreateInfo->usage & VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT) &&
 731             pCreateInfo->mipLevels == 1 &&
 732             !image->surface.dcc_size && image->extent.depth == 1 && can_cmask_dcc)
 733                 radv_image_alloc_cmask(device, image);
 734         if (image->samples > 1 && vk_format_is_color(pCreateInfo->format)) {
 735                 radv_image_alloc_fmask(device, image);
 736         } else if (vk_format_is_depth(pCreateInfo->format)) {
 737
 738                 radv_image_alloc_htile(device, image);
 739         }
 740
 741
 742         if (create_info->stride && create_info->stride != image->surface.level[0].pitch_bytes) {
 743                 image->surface.level[0].nblk_x = create_info->stride / image->surface.bpe;
 744                 image->surface.level[0].pitch_bytes = create_info->stride;
 745                 image->surface.level[0].slice_size = create_info->stride * image->surface.level[0].nblk_y;
 746         }
 747         *pImage = radv_image_to_handle(image);
 748
 749         return VK_SUCCESS;
 750 }
 751
 752 void
 753 radv_image_view_init(struct radv_image_view *iview,
 754                      struct radv_device *device,
 755                      const VkImageViewCreateInfo* pCreateInfo,
 756                      struct radv_cmd_buffer *cmd_buffer,
 757                      VkImageUsageFlags usage_mask)
 758 {
 759         RADV_FROM_HANDLE(radv_image, image, pCreateInfo->image);
 760         const VkImageSubresourceRange *range = &pCreateInfo->subresourceRange;
 761         uint32_t blk_w;
 762         bool is_stencil = false;
 763         switch (image->type) {
 764         case VK_IMAGE_TYPE_1D:
 765         case VK_IMAGE_TYPE_2D:
 766                 assert(range->baseArrayLayer + radv_get_layerCount(image, range) - 1 <= image->array_size);
 767                 break;
 768         case VK_IMAGE_TYPE_3D:
 769                 assert(range->baseArrayLayer + radv_get_layerCount(image, range) - 1
 770                        <= radv_minify(image->extent.depth, range->baseMipLevel));
 771                 break;
 772         default:
 773                 unreachable("bad VkImageType");
 774         }
 775         iview->image = image;
 776         iview->bo = image->bo;
 777         iview->type = pCreateInfo->viewType;
 778         iview->vk_format = pCreateInfo->format;
 779         iview->aspect_mask = pCreateInfo->subresourceRange.aspectMask;
 780
 781         if (iview->aspect_mask == VK_IMAGE_ASPECT_STENCIL_BIT) {
 782                 is_stencil = true;
 783                 iview->vk_format = vk_format_stencil_only(iview->vk_format);
 784         } else if (iview->aspect_mask == VK_IMAGE_ASPECT_DEPTH_BIT) {
 785                 iview->vk_format = vk_format_depth_only(iview->vk_format);
 786         }
 787
 788         iview->extent = (VkExtent3D) {
 789                 .width  = radv_minify(image->extent.width , range->baseMipLevel),
 790                 .height = radv_minify(image->extent.height, range->baseMipLevel),
 791                 .depth  = radv_minify(image->extent.depth , range->baseMipLevel),
 792         };
 793
 794         iview->extent.width = round_up_u32(iview->extent.width * vk_format_get_blockwidth(iview->vk_format),
 795                                            vk_format_get_blockwidth(image->vk_format));
 796         iview->extent.height = round_up_u32(iview->extent.height * vk_format_get_blockheight(iview->vk_format),
 797                                             vk_format_get_blockheight(image->vk_format));
 798
 799         assert(image->surface.blk_w % vk_format_get_blockwidth(image->vk_format) == 0);
 800         blk_w = image->surface.blk_w / vk_format_get_blockwidth(image->vk_format) * vk_format_get_blockwidth(iview->vk_format);
 801         iview->base_layer = range->baseArrayLayer;
 802         iview->layer_count = radv_get_layerCount(image, range);
 803         iview->base_mip = range->baseMipLevel;
 804
 805         si_make_texture_descriptor(device, image, false,
 806                                    iview->type,
 807                                    iview->vk_format,
 808                                    &pCreateInfo->components,
 809                                    0, radv_get_levelCount(image, range) - 1,
 810                                    range->baseArrayLayer,
 811                                    range->baseArrayLayer + radv_get_layerCount(image, range) - 1,
 812                                    iview->extent.width,
 813                                    iview->extent.height,
 814                                    iview->extent.depth,
 815                                    iview->descriptor,
 816                                    iview->fmask_descriptor);
 817         si_set_mutable_tex_desc_fields(device, image,
 818                                        is_stencil ? &image->surface.stencil_level[range->baseMipLevel] : &image->surface.level[range->baseMipLevel], range->baseMipLevel,
 819                                        range->baseMipLevel,
 820                                        blk_w, is_stencil, iview->descriptor);
 821 }
 822
 823 void radv_image_set_optimal_micro_tile_mode(struct radv_device *device,
 824                                             struct radv_image *image, uint32_t micro_tile_mode)
 825 {
 826         /* These magic numbers were copied from addrlib. It doesn't use any
 827          * definitions for them either. They are all 2D_TILED_THIN1 modes with
 828          * different bpp and micro tile mode.
 829          */
 830         if (device->physical_device->rad_info.chip_class >= CIK) {
 831                 switch (micro_tile_mode) {
 832                 case 0: /* displayable */
 833                         image->surface.tiling_index[0] = 10;
 834                         break;
 835                 case 1: /* thin */
 836                         image->surface.tiling_index[0] = 14;
 837                         break;
 838                 case 3: /* rotated */
 839                         image->surface.tiling_index[0] = 28;
 840                         break;
 841                 default: /* depth, thick */
 842                         assert(!"unexpected micro mode");
 843                         return;
 844                 }
 845         } else { /* SI */
 846                 switch (micro_tile_mode) {
 847                 case 0: /* displayable */
 848                         switch (image->surface.bpe) {
 849                         case 1:
 850                             image->surface.tiling_index[0] = 10;
 851                             break;
 852                         case 2:
 853                             image->surface.tiling_index[0] = 11;
 854                             break;
 855                         default: /* 4, 8 */
 856                             image->surface.tiling_index[0] = 12;
 857                             break;
 858                         }
 859                         break;
 860                 case 1: /* thin */
 861                         switch (image->surface.bpe) {
 862                         case 1:
 863                                 image->surface.tiling_index[0] = 14;
 864                                 break;
 865                         case 2:
 866                                 image->surface.tiling_index[0] = 15;
 867                                 break;
 868                         case 4:
 869                                 image->surface.tiling_index[0] = 16;
 870                                 break;
 871                         default: /* 8, 16 */
 872                                 image->surface.tiling_index[0] = 17;
 873                                 break;
 874                         }
 875                         break;
 876                 default: /* depth, thick */
 877                         assert(!"unexpected micro mode");
 878                         return;
 879                 }
 880         }
 881
 882         image->surface.micro_tile_mode = micro_tile_mode;
 883 }
 884
 885 bool radv_layout_has_htile(const struct radv_image *image,
 886                            VkImageLayout layout)
 887 {
 888         return (layout == VK_IMAGE_LAYOUT_DEPTH_STENCIL_ATTACHMENT_OPTIMAL ||
 889                 layout == VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL);
 890 }
 891
 892 bool radv_layout_is_htile_compressed(const struct radv_image *image,
 893                                      VkImageLayout layout)
 894 {
 895         return layout == VK_IMAGE_LAYOUT_DEPTH_STENCIL_ATTACHMENT_OPTIMAL;
 896 }
 897
 898 bool radv_layout_can_expclear(const struct radv_image *image,
 899                               VkImageLayout layout)
 900 {
 901         return (layout == VK_IMAGE_LAYOUT_DEPTH_STENCIL_ATTACHMENT_OPTIMAL ||
 902                 layout == VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL);
 903 }
 904
 905 bool radv_layout_can_fast_clear(const struct radv_image *image,
 906                                 VkImageLayout layout,
 907                                 unsigned queue_mask)
 908 {
 909         return layout == VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL &&
 910                 queue_mask == (1u << RADV_QUEUE_GENERAL);
 911 }
 912
 913
 914 unsigned radv_image_queue_family_mask(const struct radv_image *image, uint32_t family, uint32_t queue_family)
 915 {
 916         if (!image->exclusive)
 917                 return image->queue_family_mask;
 918         if (family == VK_QUEUE_FAMILY_IGNORED)
 919                 return 1u << queue_family;
 920         return 1u << family;
 921 }
 922
 923 VkResult
 924 radv_CreateImage(VkDevice device,
 925                  const VkImageCreateInfo *pCreateInfo,
 926                  const VkAllocationCallbacks *pAllocator,
 927                  VkImage *pImage)
 928 {
 929         return radv_image_create(device,
 930                                  &(struct radv_image_create_info) {
 931                                          .vk_info = pCreateInfo,
 932                                                  .scanout = false,
 933                                                  },
 934                                  pAllocator,
 935                                  pImage);
 936 }
 937
 938 void
 939 radv_DestroyImage(VkDevice _device, VkImage _image,
 940                   const VkAllocationCallbacks *pAllocator)
 941 {
 942         RADV_FROM_HANDLE(radv_device, device, _device);
 943
 944         if (!_image)
 945                 return;
 946
 947         vk_free2(&device->alloc, pAllocator, radv_image_from_handle(_image));
 948 }
 949
 950 void radv_GetImageSubresourceLayout(
 951         VkDevice                                    device,
 952         VkImage                                     _image,
 953         const VkImageSubresource*                   pSubresource,
 954         VkSubresourceLayout*                        pLayout)
 955 {
 956         RADV_FROM_HANDLE(radv_image, image, _image);
 957         int level = pSubresource->mipLevel;
 958         int layer = pSubresource->arrayLayer;
 959
 960         pLayout->offset = image->surface.level[level].offset + image->surface.level[level].slice_size * layer;
 961         pLayout->rowPitch = image->surface.level[level].pitch_bytes;
 962         pLayout->arrayPitch = image->surface.level[level].slice_size;
 963         pLayout->depthPitch = image->surface.level[level].slice_size;
 964         pLayout->size = image->surface.level[level].slice_size;
 965         if (image->type == VK_IMAGE_TYPE_3D)
 966                 pLayout->size *= image->surface.level[level].nblk_z;
 967 }
 968
 969
 970 VkResult
 971 radv_CreateImageView(VkDevice _device,
 972                      const VkImageViewCreateInfo *pCreateInfo,
 973                      const VkAllocationCallbacks *pAllocator,
 974                      VkImageView *pView)
 975 {
 976         RADV_FROM_HANDLE(radv_device, device, _device);
 977         struct radv_image_view *view;
 978
 979         view = vk_alloc2(&device->alloc, pAllocator, sizeof(*view), 8,
 980                            VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
 981         if (view == NULL)
 982                 return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY);
 983
 984         radv_image_view_init(view, device, pCreateInfo, NULL, ~0);
 985
 986         *pView = radv_image_view_to_handle(view);
 987
 988         return VK_SUCCESS;
 989 }
 990
 991 void
 992 radv_DestroyImageView(VkDevice _device, VkImageView _iview,
 993                       const VkAllocationCallbacks *pAllocator)
 994 {
 995         RADV_FROM_HANDLE(radv_device, device, _device);
 996         RADV_FROM_HANDLE(radv_image_view, iview, _iview);
 997
 998         if (!iview)
 999                 return;
1000         vk_free2(&device->alloc, pAllocator, iview);
1001 }
1002
1003 void radv_buffer_view_init(struct radv_buffer_view *view,
1004                            struct radv_device *device,
1005                            const VkBufferViewCreateInfo* pCreateInfo,
1006                            struct radv_cmd_buffer *cmd_buffer)
1007 {
1008         RADV_FROM_HANDLE(radv_buffer, buffer, pCreateInfo->buffer);
1009
1010         view->bo = buffer->bo;
1011         view->range = pCreateInfo->range == VK_WHOLE_SIZE ?
1012                 buffer->size - pCreateInfo->offset : pCreateInfo->range;
1013         view->vk_format = pCreateInfo->format;
1014
1015         radv_make_buffer_descriptor(device, buffer, view->vk_format,
1016                                     pCreateInfo->offset, view->range, view->state);
1017 }
1018
1019 VkResult
1020 radv_CreateBufferView(VkDevice _device,
1021                       const VkBufferViewCreateInfo *pCreateInfo,
1022                       const VkAllocationCallbacks *pAllocator,
1023                       VkBufferView *pView)
1024 {
1025         RADV_FROM_HANDLE(radv_device, device, _device);
1026         struct radv_buffer_view *view;
1027
1028         view = vk_alloc2(&device->alloc, pAllocator, sizeof(*view), 8,
1029                            VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
1030         if (!view)
1031                 return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY);
1032
1033         radv_buffer_view_init(view, device, pCreateInfo, NULL);
1034
1035         *pView = radv_buffer_view_to_handle(view);
1036
1037         return VK_SUCCESS;
1038 }
1039
1040 void
1041 radv_DestroyBufferView(VkDevice _device, VkBufferView bufferView,
1042                        const VkAllocationCallbacks *pAllocator)
1043 {
1044         RADV_FROM_HANDLE(radv_device, device, _device);
1045         RADV_FROM_HANDLE(radv_buffer_view, view, bufferView);
1046
1047         if (!view)
1048                 return;
1049
1050         vk_free2(&device->alloc, pAllocator, view);
1051 }