src/amd/vulkan/radv_device.c

   1 /*
   2  * Copyright © 2016 Red Hat.
   3  * Copyright © 2016 Bas Nieuwenhuizen
   4  *
   5  * based in part on anv driver which is:
   6  * Copyright © 2015 Intel Corporation
   7  *
   8  * Permission is hereby granted, free of charge, to any person obtaining a
   9  * copy of this software and associated documentation files (the "Software"),
  10  * to deal in the Software without restriction, including without limitation
  11  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  12  * and/or sell copies of the Software, and to permit persons to whom the
  13  * Software is furnished to do so, subject to the following conditions:
  14  *
  15  * The above copyright notice and this permission notice (including the next
  16  * paragraph) shall be included in all copies or substantial portions of the
  17  * Software.
  18  *
  19  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  20  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  21  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  22  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  23  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  24  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  25  * IN THE SOFTWARE.
  26  */
  27
  28 #include <stdbool.h>
  29 #include <string.h>
  30 #include <unistd.h>
  31 #include <fcntl.h>
  32 #include "radv_private.h"
  33 #include "util/strtod.h"
  34
  35 #include <xf86drm.h>
  36 #include <amdgpu.h>
  37 #include <amdgpu_drm.h>
  38 #include "amdgpu_id.h"
  39 #include "winsys/amdgpu/radv_amdgpu_winsys_public.h"
  40 #include "ac_llvm_util.h"
  41 #include "vk_format.h"
  42 #include "sid.h"
  43 #include "radv_timestamp.h"
  44 #include "util/debug.h"
  45 struct radv_dispatch_table dtable;
  46
  47 static VkResult
  48 radv_physical_device_init(struct radv_physical_device *device,
  49                           struct radv_instance *instance,
  50                           const char *path)
  51 {
  52         VkResult result;
  53         drmVersionPtr version;
  54         int fd;
  55
  56         fd = open(path, O_RDWR | O_CLOEXEC);
  57         if (fd < 0)
  58                 return vk_errorf(VK_ERROR_INCOMPATIBLE_DRIVER,
  59                                  "failed to open %s: %m", path);
  60
  61         version = drmGetVersion(fd);
  62         if (!version) {
  63                 close(fd);
  64                 return vk_errorf(VK_ERROR_INCOMPATIBLE_DRIVER,
  65                                  "failed to get version %s: %m", path);
  66         }
  67
  68         if (strcmp(version->name, "amdgpu")) {
  69                 drmFreeVersion(version);
  70                 close(fd);
  71                 return VK_ERROR_INCOMPATIBLE_DRIVER;
  72         }
  73         drmFreeVersion(version);
  74
  75         device->_loader_data.loaderMagic = ICD_LOADER_MAGIC;
  76         device->instance = instance;
  77         assert(strlen(path) < ARRAY_SIZE(device->path));
  78         strncpy(device->path, path, ARRAY_SIZE(device->path));
  79
  80         device->ws = radv_amdgpu_winsys_create(fd);
  81         if (!device->ws) {
  82                 result = VK_ERROR_INCOMPATIBLE_DRIVER;
  83                 goto fail;
  84         }
  85         device->ws->query_info(device->ws, &device->rad_info);
  86         result = radv_init_wsi(device);
  87         if (result != VK_SUCCESS) {
  88                 device->ws->destroy(device->ws);
  89                 goto fail;
  90         }
  91
  92         fprintf(stderr, "WARNING: radv is not a conformant vulkan implementation, testing use only.\n");
  93         device->name = device->rad_info.name;
  94         close(fd);
  95         return VK_SUCCESS;
  96
  97 fail:
  98         close(fd);
  99         return result;
 100 }
 101
 102 static void
 103 radv_physical_device_finish(struct radv_physical_device *device)
 104 {
 105         radv_finish_wsi(device);
 106         device->ws->destroy(device->ws);
 107 }
 108
 109 static const VkExtensionProperties global_extensions[] = {
 110         {
 111                 .extensionName = VK_KHR_SURFACE_EXTENSION_NAME,
 112                 .specVersion = 25,
 113         },
 114 #ifdef VK_USE_PLATFORM_XCB_KHR
 115         {
 116                 .extensionName = VK_KHR_XCB_SURFACE_EXTENSION_NAME,
 117                 .specVersion = 6,
 118         },
 119 #endif
 120 #ifdef VK_USE_PLATFORM_XLIB_KHR
 121         {
 122                 .extensionName = VK_KHR_XLIB_SURFACE_EXTENSION_NAME,
 123                 .specVersion = 6,
 124         },
 125 #endif
 126 #ifdef VK_USE_PLATFORM_WAYLAND_KHR
 127         {
 128                 .extensionName = VK_KHR_WAYLAND_SURFACE_EXTENSION_NAME,
 129                 .specVersion = 5,
 130         },
 131 #endif
 132 };
 133
 134 static const VkExtensionProperties device_extensions[] = {
 135         {
 136                 .extensionName = VK_KHR_SWAPCHAIN_EXTENSION_NAME,
 137                 .specVersion = 68,
 138         },
 139 };
 140
 141 static void *
 142 default_alloc_func(void *pUserData, size_t size, size_t align,
 143                    VkSystemAllocationScope allocationScope)
 144 {
 145         return malloc(size);
 146 }
 147
 148 static void *
 149 default_realloc_func(void *pUserData, void *pOriginal, size_t size,
 150                      size_t align, VkSystemAllocationScope allocationScope)
 151 {
 152         return realloc(pOriginal, size);
 153 }
 154
 155 static void
 156 default_free_func(void *pUserData, void *pMemory)
 157 {
 158         free(pMemory);
 159 }
 160
 161 static const VkAllocationCallbacks default_alloc = {
 162         .pUserData = NULL,
 163         .pfnAllocation = default_alloc_func,
 164         .pfnReallocation = default_realloc_func,
 165         .pfnFree = default_free_func,
 166 };
 167
 168 VkResult radv_CreateInstance(
 169         const VkInstanceCreateInfo*                 pCreateInfo,
 170         const VkAllocationCallbacks*                pAllocator,
 171         VkInstance*                                 pInstance)
 172 {
 173         struct radv_instance *instance;
 174
 175         assert(pCreateInfo->sType == VK_STRUCTURE_TYPE_INSTANCE_CREATE_INFO);
 176
 177         uint32_t client_version;
 178         if (pCreateInfo->pApplicationInfo &&
 179             pCreateInfo->pApplicationInfo->apiVersion != 0) {
 180                 client_version = pCreateInfo->pApplicationInfo->apiVersion;
 181         } else {
 182                 client_version = VK_MAKE_VERSION(1, 0, 0);
 183         }
 184
 185         if (VK_MAKE_VERSION(1, 0, 0) > client_version ||
 186             client_version > VK_MAKE_VERSION(1, 0, 0xfff)) {
 187                 return vk_errorf(VK_ERROR_INCOMPATIBLE_DRIVER,
 188                                  "Client requested version %d.%d.%d",
 189                                  VK_VERSION_MAJOR(client_version),
 190                                  VK_VERSION_MINOR(client_version),
 191                                  VK_VERSION_PATCH(client_version));
 192         }
 193
 194         for (uint32_t i = 0; i < pCreateInfo->enabledExtensionCount; i++) {
 195                 bool found = false;
 196                 for (uint32_t j = 0; j < ARRAY_SIZE(global_extensions); j++) {
 197                         if (strcmp(pCreateInfo->ppEnabledExtensionNames[i],
 198                                    global_extensions[j].extensionName) == 0) {
 199                                 found = true;
 200                                 break;
 201                         }
 202                 }
 203                 if (!found)
 204                         return vk_error(VK_ERROR_EXTENSION_NOT_PRESENT);
 205         }
 206
 207         instance = vk_alloc2(&default_alloc, pAllocator, sizeof(*instance), 8,
 208                                VK_SYSTEM_ALLOCATION_SCOPE_INSTANCE);
 209         if (!instance)
 210                 return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY);
 211
 212         instance->_loader_data.loaderMagic = ICD_LOADER_MAGIC;
 213
 214         if (pAllocator)
 215                 instance->alloc = *pAllocator;
 216         else
 217                 instance->alloc = default_alloc;
 218
 219         instance->apiVersion = client_version;
 220         instance->physicalDeviceCount = -1;
 221
 222         _mesa_locale_init();
 223
 224         VG(VALGRIND_CREATE_MEMPOOL(instance, 0, false));
 225
 226         *pInstance = radv_instance_to_handle(instance);
 227
 228         return VK_SUCCESS;
 229 }
 230
 231 void radv_DestroyInstance(
 232         VkInstance                                  _instance,
 233         const VkAllocationCallbacks*                pAllocator)
 234 {
 235         RADV_FROM_HANDLE(radv_instance, instance, _instance);
 236
 237         if (instance->physicalDeviceCount > 0) {
 238                 /* We support at most one physical device. */
 239                 assert(instance->physicalDeviceCount == 1);
 240                 radv_physical_device_finish(&instance->physicalDevice);
 241         }
 242
 243         VG(VALGRIND_DESTROY_MEMPOOL(instance));
 244
 245         _mesa_locale_fini();
 246
 247         vk_free(&instance->alloc, instance);
 248 }
 249
 250 VkResult radv_EnumeratePhysicalDevices(
 251         VkInstance                                  _instance,
 252         uint32_t*                                   pPhysicalDeviceCount,
 253         VkPhysicalDevice*                           pPhysicalDevices)
 254 {
 255         RADV_FROM_HANDLE(radv_instance, instance, _instance);
 256         VkResult result;
 257
 258         if (instance->physicalDeviceCount < 0) {
 259                 char path[20];
 260                 for (unsigned i = 0; i < 8; i++) {
 261                         snprintf(path, sizeof(path), "/dev/dri/renderD%d", 128 + i);
 262                         result = radv_physical_device_init(&instance->physicalDevice,
 263                                                            instance, path);
 264                         if (result != VK_ERROR_INCOMPATIBLE_DRIVER)
 265                                 break;
 266                 }
 267
 268                 if (result == VK_ERROR_INCOMPATIBLE_DRIVER) {
 269                         instance->physicalDeviceCount = 0;
 270                 } else if (result == VK_SUCCESS) {
 271                         instance->physicalDeviceCount = 1;
 272                 } else {
 273                         return result;
 274                 }
 275         }
 276
 277         /* pPhysicalDeviceCount is an out parameter if pPhysicalDevices is NULL;
 278          * otherwise it's an inout parameter.
 279          *
 280          * The Vulkan spec (git aaed022) says:
 281          *
 282          *    pPhysicalDeviceCount is a pointer to an unsigned integer variable
 283          *    that is initialized with the number of devices the application is
 284          *    prepared to receive handles to. pname:pPhysicalDevices is pointer to
 285          *    an array of at least this many VkPhysicalDevice handles [...].
 286          *
 287          *    Upon success, if pPhysicalDevices is NULL, vkEnumeratePhysicalDevices
 288          *    overwrites the contents of the variable pointed to by
 289          *    pPhysicalDeviceCount with the number of physical devices in in the
 290          *    instance; otherwise, vkEnumeratePhysicalDevices overwrites
 291          *    pPhysicalDeviceCount with the number of physical handles written to
 292          *    pPhysicalDevices.
 293          */
 294         if (!pPhysicalDevices) {
 295                 *pPhysicalDeviceCount = instance->physicalDeviceCount;
 296         } else if (*pPhysicalDeviceCount >= 1) {
 297                 pPhysicalDevices[0] = radv_physical_device_to_handle(&instance->physicalDevice);
 298                 *pPhysicalDeviceCount = 1;
 299         } else if (*pPhysicalDeviceCount < instance->physicalDeviceCount) {
 300                 return VK_INCOMPLETE;
 301         } else {
 302                 *pPhysicalDeviceCount = 0;
 303         }
 304
 305         return VK_SUCCESS;
 306 }
 307
 308 void radv_GetPhysicalDeviceFeatures(
 309         VkPhysicalDevice                            physicalDevice,
 310         VkPhysicalDeviceFeatures*                   pFeatures)
 311 {
 312         //   RADV_FROM_HANDLE(radv_physical_device, pdevice, physicalDevice);
 313
 314         memset(pFeatures, 0, sizeof(*pFeatures));
 315
 316         *pFeatures = (VkPhysicalDeviceFeatures) {
 317                 .robustBufferAccess                       = true,
 318                 .fullDrawIndexUint32                      = true,
 319                 .imageCubeArray                           = true,
 320                 .independentBlend                         = true,
 321                 .geometryShader                           = false,
 322                 .tessellationShader                       = false,
 323                 .sampleRateShading                        = false,
 324                 .dualSrcBlend                             = true,
 325                 .logicOp                                  = true,
 326                 .multiDrawIndirect                        = true,
 327                 .drawIndirectFirstInstance                = true,
 328                 .depthClamp                               = true,
 329                 .depthBiasClamp                           = true,
 330                 .fillModeNonSolid                         = true,
 331                 .depthBounds                              = true,
 332                 .wideLines                                = true,
 333                 .largePoints                              = true,
 334                 .alphaToOne                               = true,
 335                 .multiViewport                            = false,
 336                 .samplerAnisotropy                        = false, /* FINISHME */
 337                 .textureCompressionETC2                   = false,
 338                 .textureCompressionASTC_LDR               = false,
 339                 .textureCompressionBC                     = true,
 340                 .occlusionQueryPrecise                    = true,
 341                 .pipelineStatisticsQuery                  = false,
 342                 .vertexPipelineStoresAndAtomics           = true,
 343                 .fragmentStoresAndAtomics                 = true,
 344                 .shaderTessellationAndGeometryPointSize   = true,
 345                 .shaderImageGatherExtended                = false,
 346                 .shaderStorageImageExtendedFormats        = false,
 347                 .shaderStorageImageMultisample            = false,
 348                 .shaderUniformBufferArrayDynamicIndexing  = true,
 349                 .shaderSampledImageArrayDynamicIndexing   = true,
 350                 .shaderStorageBufferArrayDynamicIndexing  = true,
 351                 .shaderStorageImageArrayDynamicIndexing   = true,
 352                 .shaderStorageImageReadWithoutFormat      = false,
 353                 .shaderStorageImageWriteWithoutFormat     = true,
 354                 .shaderClipDistance                       = true,
 355                 .shaderCullDistance                       = true,
 356                 .shaderFloat64                            = false,
 357                 .shaderInt64                              = false,
 358                 .shaderInt16                              = false,
 359                 .alphaToOne                               = true,
 360                 .variableMultisampleRate                  = false,
 361                 .inheritedQueries                         = false,
 362         };
 363 }
 364
 365 void
 366 radv_device_get_cache_uuid(void *uuid)
 367 {
 368         memset(uuid, 0, VK_UUID_SIZE);
 369         snprintf(uuid, VK_UUID_SIZE, "radv-%s", RADV_TIMESTAMP);
 370 }
 371
 372 void radv_GetPhysicalDeviceProperties(
 373         VkPhysicalDevice                            physicalDevice,
 374         VkPhysicalDeviceProperties*                 pProperties)
 375 {
 376         RADV_FROM_HANDLE(radv_physical_device, pdevice, physicalDevice);
 377         VkSampleCountFlags sample_counts = 0xf;
 378         VkPhysicalDeviceLimits limits = {
 379                 .maxImageDimension1D                      = (1 << 14),
 380                 .maxImageDimension2D                      = (1 << 14),
 381                 .maxImageDimension3D                      = (1 << 11),
 382                 .maxImageDimensionCube                    = (1 << 14),
 383                 .maxImageArrayLayers                      = (1 << 11),
 384                 .maxTexelBufferElements                   = 128 * 1024 * 1024,
 385                 .maxUniformBufferRange                    = UINT32_MAX,
 386                 .maxStorageBufferRange                    = UINT32_MAX,
 387                 .maxPushConstantsSize                     = MAX_PUSH_CONSTANTS_SIZE,
 388                 .maxMemoryAllocationCount                 = UINT32_MAX,
 389                 .maxSamplerAllocationCount                = 64 * 1024,
 390                 .bufferImageGranularity                   = 64, /* A cache line */
 391                 .sparseAddressSpaceSize                   = 0,
 392                 .maxBoundDescriptorSets                   = MAX_SETS,
 393                 .maxPerStageDescriptorSamplers            = 64,
 394                 .maxPerStageDescriptorUniformBuffers      = 64,
 395                 .maxPerStageDescriptorStorageBuffers      = 64,
 396                 .maxPerStageDescriptorSampledImages       = 64,
 397                 .maxPerStageDescriptorStorageImages       = 64,
 398                 .maxPerStageDescriptorInputAttachments    = 64,
 399                 .maxPerStageResources                     = 128,
 400                 .maxDescriptorSetSamplers                 = 256,
 401                 .maxDescriptorSetUniformBuffers           = 256,
 402                 .maxDescriptorSetUniformBuffersDynamic    = 256,
 403                 .maxDescriptorSetStorageBuffers           = 256,
 404                 .maxDescriptorSetStorageBuffersDynamic    = 256,
 405                 .maxDescriptorSetSampledImages            = 256,
 406                 .maxDescriptorSetStorageImages            = 256,
 407                 .maxDescriptorSetInputAttachments         = 256,
 408                 .maxVertexInputAttributes                 = 32,
 409                 .maxVertexInputBindings                   = 32,
 410                 .maxVertexInputAttributeOffset            = 2047,
 411                 .maxVertexInputBindingStride              = 2048,
 412                 .maxVertexOutputComponents                = 128,
 413                 .maxTessellationGenerationLevel           = 0,
 414                 .maxTessellationPatchSize                 = 0,
 415                 .maxTessellationControlPerVertexInputComponents = 0,
 416                 .maxTessellationControlPerVertexOutputComponents = 0,
 417                 .maxTessellationControlPerPatchOutputComponents = 0,
 418                 .maxTessellationControlTotalOutputComponents = 0,
 419                 .maxTessellationEvaluationInputComponents = 0,
 420                 .maxTessellationEvaluationOutputComponents = 0,
 421                 .maxGeometryShaderInvocations             = 32,
 422                 .maxGeometryInputComponents               = 64,
 423                 .maxGeometryOutputComponents              = 128,
 424                 .maxGeometryOutputVertices                = 256,
 425                 .maxGeometryTotalOutputComponents         = 1024,
 426                 .maxFragmentInputComponents               = 128,
 427                 .maxFragmentOutputAttachments             = 8,
 428                 .maxFragmentDualSrcAttachments            = 2,
 429                 .maxFragmentCombinedOutputResources       = 8,
 430                 .maxComputeSharedMemorySize               = 32768,
 431                 .maxComputeWorkGroupCount                 = { 65535, 65535, 65535 },
 432                 .maxComputeWorkGroupInvocations           = 16 * 1024,
 433                 .maxComputeWorkGroupSize = {
 434                         16 * 1024/*devinfo->max_cs_threads*/,
 435                         16 * 1024,
 436                         16 * 1024
 437                 },
 438                 .subPixelPrecisionBits                    = 4 /* FIXME */,
 439                 .subTexelPrecisionBits                    = 4 /* FIXME */,
 440                 .mipmapPrecisionBits                      = 4 /* FIXME */,
 441                 .maxDrawIndexedIndexValue                 = UINT32_MAX,
 442                 .maxDrawIndirectCount                     = UINT32_MAX,
 443                 .maxSamplerLodBias                        = 16,
 444                 .maxSamplerAnisotropy                     = 16,
 445                 .maxViewports                             = MAX_VIEWPORTS,
 446                 .maxViewportDimensions                    = { (1 << 14), (1 << 14) },
 447                 .viewportBoundsRange                      = { INT16_MIN, INT16_MAX },
 448                 .viewportSubPixelBits                     = 13, /* We take a float? */
 449                 .minMemoryMapAlignment                    = 4096, /* A page */
 450                 .minTexelBufferOffsetAlignment            = 1,
 451                 .minUniformBufferOffsetAlignment          = 4,
 452                 .minStorageBufferOffsetAlignment          = 4,
 453                 .minTexelOffset                           = -8,
 454                 .maxTexelOffset                           = 7,
 455                 .minTexelGatherOffset                     = -8,
 456                 .maxTexelGatherOffset                     = 7,
 457                 .minInterpolationOffset                   = 0, /* FIXME */
 458                 .maxInterpolationOffset                   = 0, /* FIXME */
 459                 .subPixelInterpolationOffsetBits          = 0, /* FIXME */
 460                 .maxFramebufferWidth                      = (1 << 14),
 461                 .maxFramebufferHeight                     = (1 << 14),
 462                 .maxFramebufferLayers                     = (1 << 10),
 463                 .framebufferColorSampleCounts             = sample_counts,
 464                 .framebufferDepthSampleCounts             = sample_counts,
 465                 .framebufferStencilSampleCounts           = sample_counts,
 466                 .framebufferNoAttachmentsSampleCounts     = sample_counts,
 467                 .maxColorAttachments                      = MAX_RTS,
 468                 .sampledImageColorSampleCounts            = sample_counts,
 469                 .sampledImageIntegerSampleCounts          = VK_SAMPLE_COUNT_1_BIT,
 470                 .sampledImageDepthSampleCounts            = sample_counts,
 471                 .sampledImageStencilSampleCounts          = sample_counts,
 472                 .storageImageSampleCounts                 = VK_SAMPLE_COUNT_1_BIT,
 473                 .maxSampleMaskWords                       = 1,
 474                 .timestampComputeAndGraphics              = false,
 475                 .timestampPeriod                          = 100000.0 / pdevice->rad_info.clock_crystal_freq,
 476                 .maxClipDistances                         = 8,
 477                 .maxCullDistances                         = 8,
 478                 .maxCombinedClipAndCullDistances          = 8,
 479                 .discreteQueuePriorities                  = 1,
 480                 .pointSizeRange                           = { 0.125, 255.875 },
 481                 .lineWidthRange                           = { 0.0, 7.9921875 },
 482                 .pointSizeGranularity                     = (1.0 / 8.0),
 483                 .lineWidthGranularity                     = (1.0 / 128.0),
 484                 .strictLines                              = false, /* FINISHME */
 485                 .standardSampleLocations                  = true,
 486                 .optimalBufferCopyOffsetAlignment         = 128,
 487                 .optimalBufferCopyRowPitchAlignment       = 128,
 488                 .nonCoherentAtomSize                      = 64,
 489         };
 490
 491         *pProperties = (VkPhysicalDeviceProperties) {
 492                 .apiVersion = VK_MAKE_VERSION(1, 0, 5),
 493                 .driverVersion = 1,
 494                 .vendorID = 0x1002,
 495                 .deviceID = pdevice->rad_info.pci_id,
 496                 .deviceType = VK_PHYSICAL_DEVICE_TYPE_DISCRETE_GPU,
 497                 .limits = limits,
 498                 .sparseProperties = {0}, /* Broadwell doesn't do sparse. */
 499         };
 500
 501         strcpy(pProperties->deviceName, pdevice->name);
 502         radv_device_get_cache_uuid(pProperties->pipelineCacheUUID);
 503 }
 504
 505 void radv_GetPhysicalDeviceQueueFamilyProperties(
 506         VkPhysicalDevice                            physicalDevice,
 507         uint32_t*                                   pCount,
 508         VkQueueFamilyProperties*                    pQueueFamilyProperties)
 509 {
 510         if (pQueueFamilyProperties == NULL) {
 511                 *pCount = 1;
 512                 return;
 513         }
 514         assert(*pCount >= 1);
 515
 516         *pQueueFamilyProperties = (VkQueueFamilyProperties) {
 517                 .queueFlags = VK_QUEUE_GRAPHICS_BIT |
 518                 VK_QUEUE_COMPUTE_BIT |
 519                 VK_QUEUE_TRANSFER_BIT,
 520                 .queueCount = 1,
 521                 .timestampValidBits = 64,
 522                 .minImageTransferGranularity = (VkExtent3D) { 1, 1, 1 },
 523         };
 524 }
 525
 526 void radv_GetPhysicalDeviceMemoryProperties(
 527         VkPhysicalDevice                            physicalDevice,
 528         VkPhysicalDeviceMemoryProperties*           pMemoryProperties)
 529 {
 530         RADV_FROM_HANDLE(radv_physical_device, physical_device, physicalDevice);
 531
 532         pMemoryProperties->memoryTypeCount = 3;
 533         pMemoryProperties->memoryTypes[0] = (VkMemoryType) {
 534                 .propertyFlags = VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT,
 535                 .heapIndex = 0,
 536         };
 537         pMemoryProperties->memoryTypes[1] = (VkMemoryType) {
 538                 .propertyFlags = VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT |
 539                 VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT |
 540                 VK_MEMORY_PROPERTY_HOST_COHERENT_BIT,
 541                 .heapIndex = 0,
 542         };
 543         pMemoryProperties->memoryTypes[2] = (VkMemoryType) {
 544                 .propertyFlags = VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT|
 545                 VK_MEMORY_PROPERTY_HOST_COHERENT_BIT |
 546                 VK_MEMORY_PROPERTY_HOST_CACHED_BIT,
 547                 .heapIndex = 1,
 548         };
 549
 550         pMemoryProperties->memoryHeapCount = 2;
 551         pMemoryProperties->memoryHeaps[0] = (VkMemoryHeap) {
 552                 .size = physical_device->rad_info.vram_size,
 553                 .flags = VK_MEMORY_HEAP_DEVICE_LOCAL_BIT,
 554         };
 555         pMemoryProperties->memoryHeaps[1] = (VkMemoryHeap) {
 556                 .size = physical_device->rad_info.gart_size,
 557                 .flags = 0,
 558         };
 559 }
 560
 561 static VkResult
 562 radv_queue_init(struct radv_device *device, struct radv_queue *queue)
 563 {
 564         queue->_loader_data.loaderMagic = ICD_LOADER_MAGIC;
 565         queue->device = device;
 566
 567         return VK_SUCCESS;
 568 }
 569
 570 static void
 571 radv_queue_finish(struct radv_queue *queue)
 572 {
 573 }
 574
 575 VkResult radv_CreateDevice(
 576         VkPhysicalDevice                            physicalDevice,
 577         const VkDeviceCreateInfo*                   pCreateInfo,
 578         const VkAllocationCallbacks*                pAllocator,
 579         VkDevice*                                   pDevice)
 580 {
 581         RADV_FROM_HANDLE(radv_physical_device, physical_device, physicalDevice);
 582         VkResult result;
 583         struct radv_device *device;
 584
 585         for (uint32_t i = 0; i < pCreateInfo->enabledExtensionCount; i++) {
 586                 bool found = false;
 587                 for (uint32_t j = 0; j < ARRAY_SIZE(device_extensions); j++) {
 588                         if (strcmp(pCreateInfo->ppEnabledExtensionNames[i],
 589                                    device_extensions[j].extensionName) == 0) {
 590                                 found = true;
 591                                 break;
 592                         }
 593                 }
 594                 if (!found)
 595                         return vk_error(VK_ERROR_EXTENSION_NOT_PRESENT);
 596         }
 597
 598         device = vk_alloc2(&physical_device->instance->alloc, pAllocator,
 599                              sizeof(*device), 8,
 600                              VK_SYSTEM_ALLOCATION_SCOPE_DEVICE);
 601         if (!device)
 602                 return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY);
 603
 604         device->_loader_data.loaderMagic = ICD_LOADER_MAGIC;
 605         device->instance = physical_device->instance;
 606
 607         device->ws = physical_device->ws;
 608         if (pAllocator)
 609                 device->alloc = *pAllocator;
 610         else
 611                 device->alloc = physical_device->instance->alloc;
 612
 613         device->hw_ctx = device->ws->ctx_create(device->ws);
 614         if (!device->hw_ctx) {
 615                 result = VK_ERROR_OUT_OF_HOST_MEMORY;
 616                 goto fail_free;
 617         }
 618
 619         radv_queue_init(device, &device->queue);
 620
 621         result = radv_device_init_meta(device);
 622         if (result != VK_SUCCESS) {
 623                 device->ws->ctx_destroy(device->hw_ctx);
 624                 goto fail_free;
 625         }
 626         device->allow_fast_clears = env_var_as_boolean("RADV_FAST_CLEARS", false);
 627         device->allow_dcc = !env_var_as_boolean("RADV_DCC_DISABLE", false);
 628
 629         if (device->allow_fast_clears && device->allow_dcc)
 630                 radv_finishme("DCC fast clears have not been tested\n");
 631
 632         radv_device_init_msaa(device);
 633         device->empty_cs = device->ws->cs_create(device->ws, RING_GFX);
 634         radeon_emit(device->empty_cs, PKT3(PKT3_CONTEXT_CONTROL, 1, 0));
 635         radeon_emit(device->empty_cs, CONTEXT_CONTROL_LOAD_ENABLE(1));
 636         radeon_emit(device->empty_cs, CONTEXT_CONTROL_SHADOW_ENABLE(1));
 637         device->ws->cs_finalize(device->empty_cs);
 638         *pDevice = radv_device_to_handle(device);
 639         return VK_SUCCESS;
 640 fail_free:
 641         vk_free(&device->alloc, device);
 642         return result;
 643 }
 644
 645 void radv_DestroyDevice(
 646         VkDevice                                    _device,
 647         const VkAllocationCallbacks*                pAllocator)
 648 {
 649         RADV_FROM_HANDLE(radv_device, device, _device);
 650
 651         device->ws->ctx_destroy(device->hw_ctx);
 652         radv_queue_finish(&device->queue);
 653         radv_device_finish_meta(device);
 654
 655         vk_free(&device->alloc, device);
 656 }
 657
 658 VkResult radv_EnumerateInstanceExtensionProperties(
 659         const char*                                 pLayerName,
 660         uint32_t*                                   pPropertyCount,
 661         VkExtensionProperties*                      pProperties)
 662 {
 663         if (pProperties == NULL) {
 664                 *pPropertyCount = ARRAY_SIZE(global_extensions);
 665                 return VK_SUCCESS;
 666         }
 667
 668         *pPropertyCount = MIN2(*pPropertyCount, ARRAY_SIZE(global_extensions));
 669         typed_memcpy(pProperties, global_extensions, *pPropertyCount);
 670
 671         if (*pPropertyCount < ARRAY_SIZE(global_extensions))
 672                 return VK_INCOMPLETE;
 673
 674         return VK_SUCCESS;
 675 }
 676
 677 VkResult radv_EnumerateDeviceExtensionProperties(
 678         VkPhysicalDevice                            physicalDevice,
 679         const char*                                 pLayerName,
 680         uint32_t*                                   pPropertyCount,
 681         VkExtensionProperties*                      pProperties)
 682 {
 683         if (pProperties == NULL) {
 684                 *pPropertyCount = ARRAY_SIZE(device_extensions);
 685                 return VK_SUCCESS;
 686         }
 687
 688         *pPropertyCount = MIN2(*pPropertyCount, ARRAY_SIZE(device_extensions));
 689         typed_memcpy(pProperties, device_extensions, *pPropertyCount);
 690
 691         if (*pPropertyCount < ARRAY_SIZE(device_extensions))
 692                 return VK_INCOMPLETE;
 693
 694         return VK_SUCCESS;
 695 }
 696
 697 VkResult radv_EnumerateInstanceLayerProperties(
 698         uint32_t*                                   pPropertyCount,
 699         VkLayerProperties*                          pProperties)
 700 {
 701         if (pProperties == NULL) {
 702                 *pPropertyCount = 0;
 703                 return VK_SUCCESS;
 704         }
 705
 706         /* None supported at this time */
 707         return vk_error(VK_ERROR_LAYER_NOT_PRESENT);
 708 }
 709
 710 VkResult radv_EnumerateDeviceLayerProperties(
 711         VkPhysicalDevice                            physicalDevice,
 712         uint32_t*                                   pPropertyCount,
 713         VkLayerProperties*                          pProperties)
 714 {
 715         if (pProperties == NULL) {
 716                 *pPropertyCount = 0;
 717                 return VK_SUCCESS;
 718         }
 719
 720         /* None supported at this time */
 721         return vk_error(VK_ERROR_LAYER_NOT_PRESENT);
 722 }
 723
 724 void radv_GetDeviceQueue(
 725         VkDevice                                    _device,
 726         uint32_t                                    queueNodeIndex,
 727         uint32_t                                    queueIndex,
 728         VkQueue*                                    pQueue)
 729 {
 730         RADV_FROM_HANDLE(radv_device, device, _device);
 731
 732         assert(queueIndex == 0);
 733
 734         *pQueue = radv_queue_to_handle(&device->queue);
 735 }
 736
 737 VkResult radv_QueueSubmit(
 738         VkQueue                                     _queue,
 739         uint32_t                                    submitCount,
 740         const VkSubmitInfo*                         pSubmits,
 741         VkFence                                     _fence)
 742 {
 743         RADV_FROM_HANDLE(radv_queue, queue, _queue);
 744         RADV_FROM_HANDLE(radv_fence, fence, _fence);
 745         struct radeon_winsys_fence *base_fence = fence ? fence->fence : NULL;
 746         struct radeon_winsys_ctx *ctx = queue->device->hw_ctx;
 747         int ret;
 748
 749         for (uint32_t i = 0; i < submitCount; i++) {
 750                 struct radeon_winsys_cs **cs_array;
 751                 bool can_patch = true;
 752
 753                 if (!pSubmits[i].commandBufferCount)
 754                         continue;
 755
 756                 cs_array = malloc(sizeof(struct radeon_winsys_cs *) *
 757                                                 pSubmits[i].commandBufferCount);
 758
 759                 for (uint32_t j = 0; j < pSubmits[i].commandBufferCount; j++) {
 760                         RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer,
 761                                          pSubmits[i].pCommandBuffers[j]);
 762                         assert(cmd_buffer->level == VK_COMMAND_BUFFER_LEVEL_PRIMARY);
 763
 764                         cs_array[j] = cmd_buffer->cs;
 765                         if ((cmd_buffer->usage_flags & VK_COMMAND_BUFFER_USAGE_SIMULTANEOUS_USE_BIT))
 766                                 can_patch = false;
 767                 }
 768                 ret = queue->device->ws->cs_submit(ctx, cs_array,
 769                                                    pSubmits[i].commandBufferCount,
 770                                                    can_patch, base_fence);
 771                 if (ret)
 772                         radv_loge("failed to submit CS %d\n", i);
 773                 free(cs_array);
 774         }
 775
 776         if (fence) {
 777                 if (!submitCount)
 778                         ret = queue->device->ws->cs_submit(ctx, &queue->device->empty_cs,
 779                                                            1, false, base_fence);
 780
 781                 fence->submitted = true;
 782         }
 783
 784         return VK_SUCCESS;
 785 }
 786
 787 VkResult radv_QueueWaitIdle(
 788         VkQueue                                     _queue)
 789 {
 790         RADV_FROM_HANDLE(radv_queue, queue, _queue);
 791
 792         queue->device->ws->ctx_wait_idle(queue->device->hw_ctx);
 793         return VK_SUCCESS;
 794 }
 795
 796 VkResult radv_DeviceWaitIdle(
 797         VkDevice                                    _device)
 798 {
 799         RADV_FROM_HANDLE(radv_device, device, _device);
 800
 801         device->ws->ctx_wait_idle(device->hw_ctx);
 802         return VK_SUCCESS;
 803 }
 804
 805 PFN_vkVoidFunction radv_GetInstanceProcAddr(
 806         VkInstance                                  instance,
 807         const char*                                 pName)
 808 {
 809         return radv_lookup_entrypoint(pName);
 810 }
 811
 812 /* The loader wants us to expose a second GetInstanceProcAddr function
 813  * to work around certain LD_PRELOAD issues seen in apps.
 814  */
 815 PUBLIC
 816 VKAPI_ATTR PFN_vkVoidFunction VKAPI_CALL vk_icdGetInstanceProcAddr(
 817         VkInstance                                  instance,
 818         const char*                                 pName);
 819
 820 PUBLIC
 821 VKAPI_ATTR PFN_vkVoidFunction VKAPI_CALL vk_icdGetInstanceProcAddr(
 822         VkInstance                                  instance,
 823         const char*                                 pName)
 824 {
 825         return radv_GetInstanceProcAddr(instance, pName);
 826 }
 827
 828 PFN_vkVoidFunction radv_GetDeviceProcAddr(
 829         VkDevice                                    device,
 830         const char*                                 pName)
 831 {
 832         return radv_lookup_entrypoint(pName);
 833 }
 834
 835 VkResult radv_AllocateMemory(
 836         VkDevice                                    _device,
 837         const VkMemoryAllocateInfo*                 pAllocateInfo,
 838         const VkAllocationCallbacks*                pAllocator,
 839         VkDeviceMemory*                             pMem)
 840 {
 841         RADV_FROM_HANDLE(radv_device, device, _device);
 842         struct radv_device_memory *mem;
 843         VkResult result;
 844         enum radeon_bo_domain domain;
 845         uint32_t flags = 0;
 846         assert(pAllocateInfo->sType == VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO);
 847
 848         if (pAllocateInfo->allocationSize == 0) {
 849                 /* Apparently, this is allowed */
 850                 *pMem = VK_NULL_HANDLE;
 851                 return VK_SUCCESS;
 852         }
 853
 854         mem = vk_alloc2(&device->alloc, pAllocator, sizeof(*mem), 8,
 855                           VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
 856         if (mem == NULL)
 857                 return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY);
 858
 859         uint64_t alloc_size = align_u64(pAllocateInfo->allocationSize, 4096);
 860         if (pAllocateInfo->memoryTypeIndex == 2)
 861                 domain = RADEON_DOMAIN_GTT;
 862         else
 863                 domain = RADEON_DOMAIN_VRAM;
 864
 865         if (pAllocateInfo->memoryTypeIndex == 0)
 866                 flags |= RADEON_FLAG_NO_CPU_ACCESS;
 867         else
 868                 flags |= RADEON_FLAG_CPU_ACCESS;
 869         mem->bo = device->ws->buffer_create(device->ws, alloc_size, 32768,
 870                                                domain, flags);
 871
 872         if (!mem->bo) {
 873                 result = VK_ERROR_OUT_OF_DEVICE_MEMORY;
 874                 goto fail;
 875         }
 876         mem->type_index = pAllocateInfo->memoryTypeIndex;
 877
 878         *pMem = radv_device_memory_to_handle(mem);
 879
 880         return VK_SUCCESS;
 881
 882 fail:
 883         vk_free2(&device->alloc, pAllocator, mem);
 884
 885         return result;
 886 }
 887
 888 void radv_FreeMemory(
 889         VkDevice                                    _device,
 890         VkDeviceMemory                              _mem,
 891         const VkAllocationCallbacks*                pAllocator)
 892 {
 893         RADV_FROM_HANDLE(radv_device, device, _device);
 894         RADV_FROM_HANDLE(radv_device_memory, mem, _mem);
 895
 896         if (mem == NULL)
 897                 return;
 898
 899         device->ws->buffer_destroy(mem->bo);
 900         mem->bo = NULL;
 901
 902         vk_free2(&device->alloc, pAllocator, mem);
 903 }
 904
 905 VkResult radv_MapMemory(
 906         VkDevice                                    _device,
 907         VkDeviceMemory                              _memory,
 908         VkDeviceSize                                offset,
 909         VkDeviceSize                                size,
 910         VkMemoryMapFlags                            flags,
 911         void**                                      ppData)
 912 {
 913         RADV_FROM_HANDLE(radv_device, device, _device);
 914         RADV_FROM_HANDLE(radv_device_memory, mem, _memory);
 915
 916         if (mem == NULL) {
 917                 *ppData = NULL;
 918                 return VK_SUCCESS;
 919         }
 920
 921         *ppData = device->ws->buffer_map(mem->bo);
 922         if (*ppData) {
 923                 *ppData += offset;
 924                 return VK_SUCCESS;
 925         }
 926
 927         return VK_ERROR_MEMORY_MAP_FAILED;
 928 }
 929
 930 void radv_UnmapMemory(
 931         VkDevice                                    _device,
 932         VkDeviceMemory                              _memory)
 933 {
 934         RADV_FROM_HANDLE(radv_device, device, _device);
 935         RADV_FROM_HANDLE(radv_device_memory, mem, _memory);
 936
 937         if (mem == NULL)
 938                 return;
 939
 940         device->ws->buffer_unmap(mem->bo);
 941 }
 942
 943 VkResult radv_FlushMappedMemoryRanges(
 944         VkDevice                                    _device,
 945         uint32_t                                    memoryRangeCount,
 946         const VkMappedMemoryRange*                  pMemoryRanges)
 947 {
 948         return VK_SUCCESS;
 949 }
 950
 951 VkResult radv_InvalidateMappedMemoryRanges(
 952         VkDevice                                    _device,
 953         uint32_t                                    memoryRangeCount,
 954         const VkMappedMemoryRange*                  pMemoryRanges)
 955 {
 956         return VK_SUCCESS;
 957 }
 958
 959 void radv_GetBufferMemoryRequirements(
 960         VkDevice                                    device,
 961         VkBuffer                                    _buffer,
 962         VkMemoryRequirements*                       pMemoryRequirements)
 963 {
 964         RADV_FROM_HANDLE(radv_buffer, buffer, _buffer);
 965
 966         /* The Vulkan spec (git aaed022) says:
 967          *
 968          *    memoryTypeBits is a bitfield and contains one bit set for every
 969          *    supported memory type for the resource. The bit `1<<i` is set if and
 970          *    only if the memory type `i` in the VkPhysicalDeviceMemoryProperties
 971          *    structure for the physical device is supported.
 972          *
 973          * We support exactly one memory type.
 974          */
 975         pMemoryRequirements->memoryTypeBits = 0x7;
 976
 977         pMemoryRequirements->size = buffer->size;
 978         pMemoryRequirements->alignment = 16;
 979 }
 980
 981 void radv_GetImageMemoryRequirements(
 982         VkDevice                                    device,
 983         VkImage                                     _image,
 984         VkMemoryRequirements*                       pMemoryRequirements)
 985 {
 986         RADV_FROM_HANDLE(radv_image, image, _image);
 987
 988         /* The Vulkan spec (git aaed022) says:
 989          *
 990          *    memoryTypeBits is a bitfield and contains one bit set for every
 991          *    supported memory type for the resource. The bit `1<<i` is set if and
 992          *    only if the memory type `i` in the VkPhysicalDeviceMemoryProperties
 993          *    structure for the physical device is supported.
 994          *
 995          * We support exactly one memory type.
 996          */
 997         pMemoryRequirements->memoryTypeBits = 0x7;
 998
 999         pMemoryRequirements->size = image->size;
1000         pMemoryRequirements->alignment = image->alignment;
1001 }
1002
1003 void radv_GetImageSparseMemoryRequirements(
1004         VkDevice                                    device,
1005         VkImage                                     image,
1006         uint32_t*                                   pSparseMemoryRequirementCount,
1007         VkSparseImageMemoryRequirements*            pSparseMemoryRequirements)
1008 {
1009         stub();
1010 }
1011
1012 void radv_GetDeviceMemoryCommitment(
1013         VkDevice                                    device,
1014         VkDeviceMemory                              memory,
1015         VkDeviceSize*                               pCommittedMemoryInBytes)
1016 {
1017         *pCommittedMemoryInBytes = 0;
1018 }
1019
1020 VkResult radv_BindBufferMemory(
1021         VkDevice                                    device,
1022         VkBuffer                                    _buffer,
1023         VkDeviceMemory                              _memory,
1024         VkDeviceSize                                memoryOffset)
1025 {
1026         RADV_FROM_HANDLE(radv_device_memory, mem, _memory);
1027         RADV_FROM_HANDLE(radv_buffer, buffer, _buffer);
1028
1029         if (mem) {
1030                 buffer->bo = mem->bo;
1031                 buffer->offset = memoryOffset;
1032         } else {
1033                 buffer->bo = NULL;
1034                 buffer->offset = 0;
1035         }
1036
1037         return VK_SUCCESS;
1038 }
1039
1040 VkResult radv_BindImageMemory(
1041         VkDevice                                    device,
1042         VkImage                                     _image,
1043         VkDeviceMemory                              _memory,
1044         VkDeviceSize                                memoryOffset)
1045 {
1046         RADV_FROM_HANDLE(radv_device_memory, mem, _memory);
1047         RADV_FROM_HANDLE(radv_image, image, _image);
1048
1049         if (mem) {
1050                 image->bo = mem->bo;
1051                 image->offset = memoryOffset;
1052         } else {
1053                 image->bo = NULL;
1054                 image->offset = 0;
1055         }
1056
1057         return VK_SUCCESS;
1058 }
1059
1060 VkResult radv_QueueBindSparse(
1061         VkQueue                                     queue,
1062         uint32_t                                    bindInfoCount,
1063         const VkBindSparseInfo*                     pBindInfo,
1064         VkFence                                     fence)
1065 {
1066         stub_return(VK_ERROR_INCOMPATIBLE_DRIVER);
1067 }
1068
1069 VkResult radv_CreateFence(
1070         VkDevice                                    _device,
1071         const VkFenceCreateInfo*                    pCreateInfo,
1072         const VkAllocationCallbacks*                pAllocator,
1073         VkFence*                                    pFence)
1074 {
1075         RADV_FROM_HANDLE(radv_device, device, _device);
1076         struct radv_fence *fence = vk_alloc2(&device->alloc, pAllocator,
1077                                                sizeof(*fence), 8,
1078                                                VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
1079
1080         if (!fence)
1081                 return VK_ERROR_OUT_OF_HOST_MEMORY;
1082
1083         memset(fence, 0, sizeof(*fence));
1084         fence->submitted = false;
1085         fence->signalled = !!(pCreateInfo->flags & VK_FENCE_CREATE_SIGNALED_BIT);
1086         fence->fence = device->ws->create_fence();
1087
1088
1089         *pFence = radv_fence_to_handle(fence);
1090
1091         return VK_SUCCESS;
1092 }
1093
1094 void radv_DestroyFence(
1095         VkDevice                                    _device,
1096         VkFence                                     _fence,
1097         const VkAllocationCallbacks*                pAllocator)
1098 {
1099         RADV_FROM_HANDLE(radv_device, device, _device);
1100         RADV_FROM_HANDLE(radv_fence, fence, _fence);
1101
1102         if (!fence)
1103                 return;
1104         device->ws->destroy_fence(fence->fence);
1105         vk_free2(&device->alloc, pAllocator, fence);
1106 }
1107
1108 static uint64_t radv_get_absolute_timeout(uint64_t timeout)
1109 {
1110         uint64_t current_time;
1111         struct timespec tv;
1112
1113         clock_gettime(CLOCK_MONOTONIC, &tv);
1114         current_time = tv.tv_nsec + tv.tv_sec*1000000000ull;
1115
1116         timeout = MIN2(UINT64_MAX - current_time, timeout);
1117
1118         return current_time + timeout;
1119 }
1120
1121 VkResult radv_WaitForFences(
1122         VkDevice                                    _device,
1123         uint32_t                                    fenceCount,
1124         const VkFence*                              pFences,
1125         VkBool32                                    waitAll,
1126         uint64_t                                    timeout)
1127 {
1128         RADV_FROM_HANDLE(radv_device, device, _device);
1129         timeout = radv_get_absolute_timeout(timeout);
1130
1131         if (!waitAll && fenceCount > 1) {
1132                 fprintf(stderr, "radv: WaitForFences without waitAll not implemented yet\n");
1133         }
1134
1135         for (uint32_t i = 0; i < fenceCount; ++i) {
1136                 RADV_FROM_HANDLE(radv_fence, fence, pFences[i]);
1137                 bool expired = false;
1138
1139                 if (fence->signalled)
1140                         continue;
1141
1142                 if (!fence->submitted)
1143                         return VK_TIMEOUT;
1144
1145                 expired = device->ws->fence_wait(device->ws, fence->fence, true, timeout);
1146                 if (!expired)
1147                         return VK_TIMEOUT;
1148
1149                 fence->signalled = true;
1150         }
1151
1152         return VK_SUCCESS;
1153 }
1154
1155 VkResult radv_ResetFences(VkDevice device,
1156                           uint32_t fenceCount,
1157                           const VkFence *pFences)
1158 {
1159         for (unsigned i = 0; i < fenceCount; ++i) {
1160                 RADV_FROM_HANDLE(radv_fence, fence, pFences[i]);
1161                 fence->submitted = fence->signalled = false;
1162         }
1163
1164         return VK_SUCCESS;
1165 }
1166
1167 VkResult radv_GetFenceStatus(VkDevice _device, VkFence _fence)
1168 {
1169         RADV_FROM_HANDLE(radv_device, device, _device);
1170         RADV_FROM_HANDLE(radv_fence, fence, _fence);
1171
1172         if (fence->signalled)
1173                 return VK_SUCCESS;
1174         if (!fence->submitted)
1175                 return VK_NOT_READY;
1176
1177         if (!device->ws->fence_wait(device->ws, fence->fence, false, 0))
1178                 return VK_NOT_READY;
1179
1180         return VK_SUCCESS;
1181 }
1182
1183
1184 // Queue semaphore functions
1185
1186 VkResult radv_CreateSemaphore(
1187         VkDevice                                    device,
1188         const VkSemaphoreCreateInfo*                pCreateInfo,
1189         const VkAllocationCallbacks*                pAllocator,
1190         VkSemaphore*                                pSemaphore)
1191 {
1192         /* The DRM execbuffer ioctl always execute in-oder, even between different
1193          * rings. As such, there's nothing to do for the user space semaphore.
1194          */
1195
1196         *pSemaphore = (VkSemaphore)1;
1197
1198         return VK_SUCCESS;
1199 }
1200
1201 void radv_DestroySemaphore(
1202         VkDevice                                    device,
1203         VkSemaphore                                 semaphore,
1204         const VkAllocationCallbacks*                pAllocator)
1205 {
1206 }
1207
1208 VkResult radv_CreateEvent(
1209         VkDevice                                    _device,
1210         const VkEventCreateInfo*                    pCreateInfo,
1211         const VkAllocationCallbacks*                pAllocator,
1212         VkEvent*                                    pEvent)
1213 {
1214         RADV_FROM_HANDLE(radv_device, device, _device);
1215         struct radv_event *event = vk_alloc2(&device->alloc, pAllocator,
1216                                                sizeof(*event), 8,
1217                                                VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
1218
1219         if (!event)
1220                 return VK_ERROR_OUT_OF_HOST_MEMORY;
1221
1222         event->bo = device->ws->buffer_create(device->ws, 8, 8,
1223                                               RADEON_DOMAIN_GTT,
1224                                               RADEON_FLAG_CPU_ACCESS);
1225         if (!event->bo) {
1226                 vk_free2(&device->alloc, pAllocator, event);
1227                 return VK_ERROR_OUT_OF_DEVICE_MEMORY;
1228         }
1229
1230         event->map = (uint64_t*)device->ws->buffer_map(event->bo);
1231
1232         *pEvent = radv_event_to_handle(event);
1233
1234         return VK_SUCCESS;
1235 }
1236
1237 void radv_DestroyEvent(
1238         VkDevice                                    _device,
1239         VkEvent                                     _event,
1240         const VkAllocationCallbacks*                pAllocator)
1241 {
1242         RADV_FROM_HANDLE(radv_device, device, _device);
1243         RADV_FROM_HANDLE(radv_event, event, _event);
1244
1245         if (!event)
1246                 return;
1247         device->ws->buffer_destroy(event->bo);
1248         vk_free2(&device->alloc, pAllocator, event);
1249 }
1250
1251 VkResult radv_GetEventStatus(
1252         VkDevice                                    _device,
1253         VkEvent                                     _event)
1254 {
1255         RADV_FROM_HANDLE(radv_event, event, _event);
1256
1257         if (*event->map == 1)
1258                 return VK_EVENT_SET;
1259         return VK_EVENT_RESET;
1260 }
1261
1262 VkResult radv_SetEvent(
1263         VkDevice                                    _device,
1264         VkEvent                                     _event)
1265 {
1266         RADV_FROM_HANDLE(radv_event, event, _event);
1267         *event->map = 1;
1268
1269         return VK_SUCCESS;
1270 }
1271
1272 VkResult radv_ResetEvent(
1273     VkDevice                                    _device,
1274     VkEvent                                     _event)
1275 {
1276         RADV_FROM_HANDLE(radv_event, event, _event);
1277         *event->map = 0;
1278
1279         return VK_SUCCESS;
1280 }
1281
1282 VkResult radv_CreateBuffer(
1283         VkDevice                                    _device,
1284         const VkBufferCreateInfo*                   pCreateInfo,
1285         const VkAllocationCallbacks*                pAllocator,
1286         VkBuffer*                                   pBuffer)
1287 {
1288         RADV_FROM_HANDLE(radv_device, device, _device);
1289         struct radv_buffer *buffer;
1290
1291         assert(pCreateInfo->sType == VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO);
1292
1293         buffer = vk_alloc2(&device->alloc, pAllocator, sizeof(*buffer), 8,
1294                              VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
1295         if (buffer == NULL)
1296                 return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY);
1297
1298         buffer->size = pCreateInfo->size;
1299         buffer->usage = pCreateInfo->usage;
1300         buffer->bo = NULL;
1301         buffer->offset = 0;
1302
1303         *pBuffer = radv_buffer_to_handle(buffer);
1304
1305         return VK_SUCCESS;
1306 }
1307
1308 void radv_DestroyBuffer(
1309         VkDevice                                    _device,
1310         VkBuffer                                    _buffer,
1311         const VkAllocationCallbacks*                pAllocator)
1312 {
1313         RADV_FROM_HANDLE(radv_device, device, _device);
1314         RADV_FROM_HANDLE(radv_buffer, buffer, _buffer);
1315
1316         if (!buffer)
1317                 return;
1318
1319         vk_free2(&device->alloc, pAllocator, buffer);
1320 }
1321
1322 static inline unsigned
1323 si_tile_mode_index(const struct radv_image *image, unsigned level, bool stencil)
1324 {
1325         if (stencil)
1326                 return image->surface.stencil_tiling_index[level];
1327         else
1328                 return image->surface.tiling_index[level];
1329 }
1330
1331 static void
1332 radv_initialise_color_surface(struct radv_device *device,
1333                               struct radv_color_buffer_info *cb,
1334                               struct radv_image_view *iview)
1335 {
1336         const struct vk_format_description *desc;
1337         unsigned ntype, format, swap, endian;
1338         unsigned blend_clamp = 0, blend_bypass = 0;
1339         unsigned pitch_tile_max, slice_tile_max, tile_mode_index;
1340         uint64_t va;
1341         const struct radeon_surf *surf = &iview->image->surface;
1342         const struct radeon_surf_level *level_info = &surf->level[iview->base_mip];
1343
1344         desc = vk_format_description(iview->vk_format);
1345
1346         memset(cb, 0, sizeof(*cb));
1347
1348         va = device->ws->buffer_get_va(iview->bo) + iview->image->offset;
1349         va += level_info->offset;
1350         cb->cb_color_base = va >> 8;
1351
1352         /* CMASK variables */
1353         va = device->ws->buffer_get_va(iview->bo) + iview->image->offset;
1354         va += iview->image->cmask.offset;
1355         cb->cb_color_cmask = va >> 8;
1356         cb->cb_color_cmask_slice = iview->image->cmask.slice_tile_max;
1357
1358         va = device->ws->buffer_get_va(iview->bo) + iview->image->offset;
1359         va += iview->image->dcc_offset;
1360         cb->cb_dcc_base = va >> 8;
1361
1362         cb->cb_color_view = S_028C6C_SLICE_START(iview->base_layer) |
1363                 S_028C6C_SLICE_MAX(iview->base_layer + iview->extent.depth - 1);
1364
1365         cb->micro_tile_mode = iview->image->surface.micro_tile_mode;
1366         pitch_tile_max = level_info->nblk_x / 8 - 1;
1367         slice_tile_max = (level_info->nblk_x * level_info->nblk_y) / 64 - 1;
1368         tile_mode_index = si_tile_mode_index(iview->image, iview->base_mip, false);
1369
1370         cb->cb_color_pitch = S_028C64_TILE_MAX(pitch_tile_max);
1371         cb->cb_color_slice = S_028C68_TILE_MAX(slice_tile_max);
1372
1373         /* Intensity is implemented as Red, so treat it that way. */
1374         cb->cb_color_attrib = S_028C74_FORCE_DST_ALPHA_1(desc->swizzle[3] == VK_SWIZZLE_1) |
1375                 S_028C74_TILE_MODE_INDEX(tile_mode_index);
1376
1377         if (iview->image->samples > 1) {
1378                 unsigned log_samples = util_logbase2(iview->image->samples);
1379
1380                 cb->cb_color_attrib |= S_028C74_NUM_SAMPLES(log_samples) |
1381                         S_028C74_NUM_FRAGMENTS(log_samples);
1382         }
1383
1384         if (iview->image->fmask.size) {
1385                 va = device->ws->buffer_get_va(iview->bo) + iview->image->offset + iview->image->fmask.offset;
1386                 if (device->instance->physicalDevice.rad_info.chip_class >= CIK)
1387                         cb->cb_color_pitch |= S_028C64_FMASK_TILE_MAX(iview->image->fmask.pitch_in_pixels / 8 - 1);
1388                 cb->cb_color_attrib |= S_028C74_FMASK_TILE_MODE_INDEX(iview->image->fmask.tile_mode_index);
1389                 cb->cb_color_fmask = va >> 8;
1390                 cb->cb_color_fmask_slice = S_028C88_TILE_MAX(iview->image->fmask.slice_tile_max);
1391         } else {
1392                 /* This must be set for fast clear to work without FMASK. */
1393                 if (device->instance->physicalDevice.rad_info.chip_class >= CIK)
1394                         cb->cb_color_pitch |= S_028C64_FMASK_TILE_MAX(pitch_tile_max);
1395                 cb->cb_color_attrib |= S_028C74_FMASK_TILE_MODE_INDEX(tile_mode_index);
1396                 cb->cb_color_fmask = cb->cb_color_base;
1397                 cb->cb_color_fmask_slice = S_028C88_TILE_MAX(slice_tile_max);
1398         }
1399
1400         ntype = radv_translate_color_numformat(iview->vk_format,
1401                                                desc,
1402                                                vk_format_get_first_non_void_channel(iview->vk_format));
1403         format = radv_translate_colorformat(iview->vk_format);
1404         if (format == V_028C70_COLOR_INVALID || ntype == ~0u)
1405                 radv_finishme("Illegal color\n");
1406         swap = radv_translate_colorswap(iview->vk_format, FALSE);
1407         endian = radv_colorformat_endian_swap(format);
1408
1409         /* blend clamp should be set for all NORM/SRGB types */
1410         if (ntype == V_028C70_NUMBER_UNORM ||
1411             ntype == V_028C70_NUMBER_SNORM ||
1412             ntype == V_028C70_NUMBER_SRGB)
1413                 blend_clamp = 1;
1414
1415         /* set blend bypass according to docs if SINT/UINT or
1416            8/24 COLOR variants */
1417         if (ntype == V_028C70_NUMBER_UINT || ntype == V_028C70_NUMBER_SINT ||
1418             format == V_028C70_COLOR_8_24 || format == V_028C70_COLOR_24_8 ||
1419             format == V_028C70_COLOR_X24_8_32_FLOAT) {
1420                 blend_clamp = 0;
1421                 blend_bypass = 1;
1422         }
1423 #if 0
1424         if ((ntype == V_028C70_NUMBER_UINT || ntype == V_028C70_NUMBER_SINT) &&
1425             (format == V_028C70_COLOR_8 ||
1426              format == V_028C70_COLOR_8_8 ||
1427              format == V_028C70_COLOR_8_8_8_8))
1428                 ->color_is_int8 = true;
1429 #endif
1430         cb->cb_color_info = S_028C70_FORMAT(format) |
1431                 S_028C70_COMP_SWAP(swap) |
1432                 S_028C70_BLEND_CLAMP(blend_clamp) |
1433                 S_028C70_BLEND_BYPASS(blend_bypass) |
1434                 S_028C70_SIMPLE_FLOAT(1) |
1435                 S_028C70_ROUND_MODE(ntype != V_028C70_NUMBER_UNORM &&
1436                                     ntype != V_028C70_NUMBER_SNORM &&
1437                                     ntype != V_028C70_NUMBER_SRGB &&
1438                                     format != V_028C70_COLOR_8_24 &&
1439                                     format != V_028C70_COLOR_24_8) |
1440                 S_028C70_NUMBER_TYPE(ntype) |
1441                 S_028C70_ENDIAN(endian);
1442         if (iview->image->samples > 1)
1443                 if (iview->image->fmask.size)
1444                         cb->cb_color_info |= S_028C70_COMPRESSION(1);
1445
1446         if (iview->image->cmask.size && device->allow_fast_clears)
1447                 cb->cb_color_info |= S_028C70_FAST_CLEAR(1);
1448
1449         if (iview->image->surface.dcc_size && level_info->dcc_enabled)
1450                 cb->cb_color_info |= S_028C70_DCC_ENABLE(1);
1451
1452         if (device->instance->physicalDevice.rad_info.chip_class >= VI) {
1453                 unsigned max_uncompressed_block_size = 2;
1454                 if (iview->image->samples > 1) {
1455                         if (iview->image->surface.bpe == 1)
1456                                 max_uncompressed_block_size = 0;
1457                         else if (iview->image->surface.bpe == 2)
1458                                 max_uncompressed_block_size = 1;
1459                 }
1460
1461                 cb->cb_dcc_control = S_028C78_MAX_UNCOMPRESSED_BLOCK_SIZE(max_uncompressed_block_size) |
1462                         S_028C78_INDEPENDENT_64B_BLOCKS(1);
1463         }
1464
1465         /* This must be set for fast clear to work without FMASK. */
1466         if (!iview->image->fmask.size &&
1467             device->instance->physicalDevice.rad_info.chip_class == SI) {
1468                 unsigned bankh = util_logbase2(iview->image->surface.bankh);
1469                 cb->cb_color_attrib |= S_028C74_FMASK_BANK_HEIGHT(bankh);
1470         }
1471 }
1472
1473 static void
1474 radv_initialise_ds_surface(struct radv_device *device,
1475                            struct radv_ds_buffer_info *ds,
1476                            struct radv_image_view *iview)
1477 {
1478         unsigned level = iview->base_mip;
1479         unsigned format;
1480         uint64_t va, s_offs, z_offs;
1481         const struct radeon_surf_level *level_info = &iview->image->surface.level[level];
1482         memset(ds, 0, sizeof(*ds));
1483         switch (iview->vk_format) {
1484         case VK_FORMAT_D24_UNORM_S8_UINT:
1485         case VK_FORMAT_X8_D24_UNORM_PACK32:
1486                 ds->pa_su_poly_offset_db_fmt_cntl = S_028B78_POLY_OFFSET_NEG_NUM_DB_BITS(-24);
1487                 ds->offset_scale = 2.0f;
1488                 break;
1489         case VK_FORMAT_D16_UNORM:
1490         case VK_FORMAT_D16_UNORM_S8_UINT:
1491                 ds->pa_su_poly_offset_db_fmt_cntl = S_028B78_POLY_OFFSET_NEG_NUM_DB_BITS(-16);
1492                 ds->offset_scale = 4.0f;
1493                 break;
1494         case VK_FORMAT_D32_SFLOAT:
1495         case VK_FORMAT_D32_SFLOAT_S8_UINT:
1496                 ds->pa_su_poly_offset_db_fmt_cntl = S_028B78_POLY_OFFSET_NEG_NUM_DB_BITS(-23) |
1497                         S_028B78_POLY_OFFSET_DB_IS_FLOAT_FMT(1);
1498                 ds->offset_scale = 1.0f;
1499                 break;
1500         default:
1501                 break;
1502         }
1503
1504         format = radv_translate_dbformat(iview->vk_format);
1505         if (format == V_028040_Z_INVALID) {
1506                 fprintf(stderr, "Invalid DB format: %d, disabling DB.\n", iview->vk_format);
1507         }
1508
1509         va = device->ws->buffer_get_va(iview->bo) + iview->image->offset;
1510         s_offs = z_offs = va;
1511         z_offs += iview->image->surface.level[level].offset;
1512         s_offs += iview->image->surface.stencil_level[level].offset;
1513
1514         ds->db_depth_view = S_028008_SLICE_START(iview->base_layer) |
1515                 S_028008_SLICE_MAX(iview->base_layer + iview->extent.depth - 1);
1516         ds->db_depth_info = S_02803C_ADDR5_SWIZZLE_MASK(1);
1517         ds->db_z_info = S_028040_FORMAT(format) | S_028040_ZRANGE_PRECISION(1);
1518
1519         if (iview->image->samples > 1)
1520                 ds->db_z_info |= S_028040_NUM_SAMPLES(util_logbase2(iview->image->samples));
1521
1522         if (iview->image->surface.flags & RADEON_SURF_SBUFFER)
1523                 ds->db_stencil_info = S_028044_FORMAT(V_028044_STENCIL_8);
1524         else
1525                 ds->db_stencil_info = S_028044_FORMAT(V_028044_STENCIL_INVALID);
1526
1527         if (device->instance->physicalDevice.rad_info.chip_class >= CIK) {
1528                 struct radeon_info *info = &device->instance->physicalDevice.rad_info;
1529                 unsigned tiling_index = iview->image->surface.tiling_index[level];
1530                 unsigned stencil_index = iview->image->surface.stencil_tiling_index[level];
1531                 unsigned macro_index = iview->image->surface.macro_tile_index;
1532                 unsigned tile_mode = info->si_tile_mode_array[tiling_index];
1533                 unsigned stencil_tile_mode = info->si_tile_mode_array[stencil_index];
1534                 unsigned macro_mode = info->cik_macrotile_mode_array[macro_index];
1535
1536                 ds->db_depth_info |=
1537                         S_02803C_ARRAY_MODE(G_009910_ARRAY_MODE(tile_mode)) |
1538                         S_02803C_PIPE_CONFIG(G_009910_PIPE_CONFIG(tile_mode)) |
1539                         S_02803C_BANK_WIDTH(G_009990_BANK_WIDTH(macro_mode)) |
1540                         S_02803C_BANK_HEIGHT(G_009990_BANK_HEIGHT(macro_mode)) |
1541                         S_02803C_MACRO_TILE_ASPECT(G_009990_MACRO_TILE_ASPECT(macro_mode)) |
1542                         S_02803C_NUM_BANKS(G_009990_NUM_BANKS(macro_mode));
1543                 ds->db_z_info |= S_028040_TILE_SPLIT(G_009910_TILE_SPLIT(tile_mode));
1544                 ds->db_stencil_info |= S_028044_TILE_SPLIT(G_009910_TILE_SPLIT(stencil_tile_mode));
1545         } else {
1546                 unsigned tile_mode_index = si_tile_mode_index(iview->image, level, false);
1547                 ds->db_z_info |= S_028040_TILE_MODE_INDEX(tile_mode_index);
1548                 tile_mode_index = si_tile_mode_index(iview->image, level, true);
1549                 ds->db_stencil_info |= S_028044_TILE_MODE_INDEX(tile_mode_index);
1550         }
1551
1552         if (iview->image->htile.size && !level) {
1553                 ds->db_z_info |= S_028040_TILE_SURFACE_ENABLE(1) |
1554                         S_028040_ALLOW_EXPCLEAR(1);
1555
1556                 if (iview->image->surface.flags & RADEON_SURF_SBUFFER) {
1557                         /* Workaround: For a not yet understood reason, the
1558                          * combination of MSAA, fast stencil clear and stencil
1559                          * decompress messes with subsequent stencil buffer
1560                          * uses. Problem was reproduced on Verde, Bonaire,
1561                          * Tonga, and Carrizo.
1562                          *
1563                          * Disabling EXPCLEAR works around the problem.
1564                          *
1565                          * Check piglit's arb_texture_multisample-stencil-clear
1566                          * test if you want to try changing this.
1567                          */
1568                         if (iview->image->samples <= 1)
1569                                 ds->db_stencil_info |= S_028044_ALLOW_EXPCLEAR(1);
1570                 } else
1571                         /* Use all of the htile_buffer for depth if there's no stencil. */
1572                         ds->db_stencil_info |= S_028044_TILE_STENCIL_DISABLE(1);
1573
1574                 va = device->ws->buffer_get_va(iview->bo) + iview->image->offset +
1575                      iview->image->htile.offset;
1576                 ds->db_htile_data_base = va >> 8;
1577                 ds->db_htile_surface = S_028ABC_FULL_CACHE(1);
1578         } else {
1579                 ds->db_htile_data_base = 0;
1580                 ds->db_htile_surface = 0;
1581         }
1582
1583         ds->db_z_read_base = ds->db_z_write_base = z_offs >> 8;
1584         ds->db_stencil_read_base = ds->db_stencil_write_base = s_offs >> 8;
1585
1586         ds->db_depth_size = S_028058_PITCH_TILE_MAX((level_info->nblk_x / 8) - 1) |
1587                 S_028058_HEIGHT_TILE_MAX((level_info->nblk_y / 8) - 1);
1588         ds->db_depth_slice = S_02805C_SLICE_TILE_MAX((level_info->nblk_x * level_info->nblk_y) / 64 - 1);
1589 }
1590
1591 VkResult radv_CreateFramebuffer(
1592         VkDevice                                    _device,
1593         const VkFramebufferCreateInfo*              pCreateInfo,
1594         const VkAllocationCallbacks*                pAllocator,
1595         VkFramebuffer*                              pFramebuffer)
1596 {
1597         RADV_FROM_HANDLE(radv_device, device, _device);
1598         struct radv_framebuffer *framebuffer;
1599
1600         assert(pCreateInfo->sType == VK_STRUCTURE_TYPE_FRAMEBUFFER_CREATE_INFO);
1601
1602         size_t size = sizeof(*framebuffer) +
1603                 sizeof(struct radv_attachment_info) * pCreateInfo->attachmentCount;
1604         framebuffer = vk_alloc2(&device->alloc, pAllocator, size, 8,
1605                                   VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
1606         if (framebuffer == NULL)
1607                 return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY);
1608
1609         framebuffer->attachment_count = pCreateInfo->attachmentCount;
1610         for (uint32_t i = 0; i < pCreateInfo->attachmentCount; i++) {
1611                 VkImageView _iview = pCreateInfo->pAttachments[i];
1612                 struct radv_image_view *iview = radv_image_view_from_handle(_iview);
1613                 framebuffer->attachments[i].attachment = iview;
1614                 if (iview->aspect_mask & VK_IMAGE_ASPECT_COLOR_BIT) {
1615                         radv_initialise_color_surface(device, &framebuffer->attachments[i].cb, iview);
1616                 } else if (iview->aspect_mask & (VK_IMAGE_ASPECT_DEPTH_BIT | VK_IMAGE_ASPECT_STENCIL_BIT)) {
1617                         radv_initialise_ds_surface(device, &framebuffer->attachments[i].ds, iview);
1618                 }
1619         }
1620
1621         framebuffer->width = pCreateInfo->width;
1622         framebuffer->height = pCreateInfo->height;
1623         framebuffer->layers = pCreateInfo->layers;
1624
1625         *pFramebuffer = radv_framebuffer_to_handle(framebuffer);
1626         return VK_SUCCESS;
1627 }
1628
1629 void radv_DestroyFramebuffer(
1630         VkDevice                                    _device,
1631         VkFramebuffer                               _fb,
1632         const VkAllocationCallbacks*                pAllocator)
1633 {
1634         RADV_FROM_HANDLE(radv_device, device, _device);
1635         RADV_FROM_HANDLE(radv_framebuffer, fb, _fb);
1636
1637         if (!fb)
1638                 return;
1639         vk_free2(&device->alloc, pAllocator, fb);
1640 }
1641
1642 static unsigned radv_tex_wrap(VkSamplerAddressMode address_mode)
1643 {
1644         switch (address_mode) {
1645         case VK_SAMPLER_ADDRESS_MODE_REPEAT:
1646                 return V_008F30_SQ_TEX_WRAP;
1647         case VK_SAMPLER_ADDRESS_MODE_MIRRORED_REPEAT:
1648                 return V_008F30_SQ_TEX_MIRROR;
1649         case VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_EDGE:
1650                 return V_008F30_SQ_TEX_CLAMP_LAST_TEXEL;
1651         case VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_BORDER:
1652                 return V_008F30_SQ_TEX_CLAMP_BORDER;
1653         case VK_SAMPLER_ADDRESS_MODE_MIRROR_CLAMP_TO_EDGE:
1654                 return V_008F30_SQ_TEX_MIRROR_ONCE_LAST_TEXEL;
1655         default:
1656                 unreachable("illegal tex wrap mode");
1657                 break;
1658         }
1659 }
1660
1661 static unsigned
1662 radv_tex_compare(VkCompareOp op)
1663 {
1664         switch (op) {
1665         case VK_COMPARE_OP_NEVER:
1666                 return V_008F30_SQ_TEX_DEPTH_COMPARE_NEVER;
1667         case VK_COMPARE_OP_LESS:
1668                 return V_008F30_SQ_TEX_DEPTH_COMPARE_LESS;
1669         case VK_COMPARE_OP_EQUAL:
1670                 return V_008F30_SQ_TEX_DEPTH_COMPARE_EQUAL;
1671         case VK_COMPARE_OP_LESS_OR_EQUAL:
1672                 return V_008F30_SQ_TEX_DEPTH_COMPARE_LESSEQUAL;
1673         case VK_COMPARE_OP_GREATER:
1674                 return V_008F30_SQ_TEX_DEPTH_COMPARE_GREATER;
1675         case VK_COMPARE_OP_NOT_EQUAL:
1676                 return V_008F30_SQ_TEX_DEPTH_COMPARE_NOTEQUAL;
1677         case VK_COMPARE_OP_GREATER_OR_EQUAL:
1678                 return V_008F30_SQ_TEX_DEPTH_COMPARE_GREATEREQUAL;
1679         case VK_COMPARE_OP_ALWAYS:
1680                 return V_008F30_SQ_TEX_DEPTH_COMPARE_ALWAYS;
1681         default:
1682                 unreachable("illegal compare mode");
1683                 break;
1684         }
1685 }
1686
1687 static unsigned
1688 radv_tex_filter(VkFilter filter, unsigned max_ansio)
1689 {
1690         switch (filter) {
1691         case VK_FILTER_NEAREST:
1692                 return (max_ansio > 1 ? V_008F38_SQ_TEX_XY_FILTER_ANISO_POINT :
1693                         V_008F38_SQ_TEX_XY_FILTER_POINT);
1694         case VK_FILTER_LINEAR:
1695                 return (max_ansio > 1 ? V_008F38_SQ_TEX_XY_FILTER_ANISO_BILINEAR :
1696                         V_008F38_SQ_TEX_XY_FILTER_BILINEAR);
1697         case VK_FILTER_CUBIC_IMG:
1698         default:
1699                 fprintf(stderr, "illegal texture filter");
1700                 return 0;
1701         }
1702 }
1703
1704 static unsigned
1705 radv_tex_mipfilter(VkSamplerMipmapMode mode)
1706 {
1707         switch (mode) {
1708         case VK_SAMPLER_MIPMAP_MODE_NEAREST:
1709                 return V_008F38_SQ_TEX_Z_FILTER_POINT;
1710         case VK_SAMPLER_MIPMAP_MODE_LINEAR:
1711                 return V_008F38_SQ_TEX_Z_FILTER_LINEAR;
1712         default:
1713                 return V_008F38_SQ_TEX_Z_FILTER_NONE;
1714         }
1715 }
1716
1717 static unsigned
1718 radv_tex_bordercolor(VkBorderColor bcolor)
1719 {
1720         switch (bcolor) {
1721         case VK_BORDER_COLOR_FLOAT_TRANSPARENT_BLACK:
1722         case VK_BORDER_COLOR_INT_TRANSPARENT_BLACK:
1723                 return V_008F3C_SQ_TEX_BORDER_COLOR_TRANS_BLACK;
1724         case VK_BORDER_COLOR_FLOAT_OPAQUE_BLACK:
1725         case VK_BORDER_COLOR_INT_OPAQUE_BLACK:
1726                 return V_008F3C_SQ_TEX_BORDER_COLOR_OPAQUE_BLACK;
1727         case VK_BORDER_COLOR_FLOAT_OPAQUE_WHITE:
1728         case VK_BORDER_COLOR_INT_OPAQUE_WHITE:
1729                 return V_008F3C_SQ_TEX_BORDER_COLOR_OPAQUE_WHITE;
1730         default:
1731                 break;
1732         }
1733         return 0;
1734 }
1735
1736 static unsigned
1737 radv_tex_aniso_filter(unsigned filter)
1738 {
1739         if (filter < 2)
1740                 return 0;
1741         if (filter < 4)
1742                 return 1;
1743         if (filter < 8)
1744                 return 2;
1745         if (filter < 16)
1746                 return 3;
1747         return 4;
1748 }
1749
1750 static void
1751 radv_init_sampler(struct radv_device *device,
1752                   struct radv_sampler *sampler,
1753                   const VkSamplerCreateInfo *pCreateInfo)
1754 {
1755         uint32_t max_aniso = pCreateInfo->anisotropyEnable && pCreateInfo->maxAnisotropy > 1.0 ?
1756                                         (uint32_t) pCreateInfo->maxAnisotropy : 0;
1757         uint32_t max_aniso_ratio = radv_tex_aniso_filter(max_aniso);
1758         bool is_vi;
1759         is_vi = (device->instance->physicalDevice.rad_info.chip_class >= VI);
1760
1761         if (!is_vi && max_aniso > 0) {
1762                 radv_finishme("Anisotropic filtering must be disabled manually "
1763                               "by the shader on SI-CI when BASE_LEVEL == LAST_LEVEL\n");
1764                 max_aniso = max_aniso_ratio = 0;
1765         }
1766
1767         sampler->state[0] = (S_008F30_CLAMP_X(radv_tex_wrap(pCreateInfo->addressModeU)) |
1768                              S_008F30_CLAMP_Y(radv_tex_wrap(pCreateInfo->addressModeV)) |
1769                              S_008F30_CLAMP_Z(radv_tex_wrap(pCreateInfo->addressModeW)) |
1770                              S_008F30_MAX_ANISO_RATIO(max_aniso_ratio) |
1771                              S_008F30_DEPTH_COMPARE_FUNC(radv_tex_compare(pCreateInfo->compareOp)) |
1772                              S_008F30_FORCE_UNNORMALIZED(pCreateInfo->unnormalizedCoordinates ? 1 : 0) |
1773                              S_008F30_ANISO_THRESHOLD(max_aniso_ratio >> 1) |
1774                              S_008F30_ANISO_BIAS(max_aniso_ratio) |
1775                              S_008F30_DISABLE_CUBE_WRAP(0) |
1776                              S_008F30_COMPAT_MODE(is_vi));
1777         sampler->state[1] = (S_008F34_MIN_LOD(S_FIXED(CLAMP(pCreateInfo->minLod, 0, 15), 8)) |
1778                              S_008F34_MAX_LOD(S_FIXED(CLAMP(pCreateInfo->maxLod, 0, 15), 8)) |
1779                              S_008F34_PERF_MIP(max_aniso_ratio ? max_aniso_ratio + 6 : 0));
1780         sampler->state[2] = (S_008F38_LOD_BIAS(S_FIXED(CLAMP(pCreateInfo->mipLodBias, -16, 16), 8)) |
1781                              S_008F38_XY_MAG_FILTER(radv_tex_filter(pCreateInfo->magFilter, max_aniso)) |
1782                              S_008F38_XY_MIN_FILTER(radv_tex_filter(pCreateInfo->minFilter, max_aniso)) |
1783                              S_008F38_MIP_FILTER(radv_tex_mipfilter(pCreateInfo->mipmapMode)) |
1784                              S_008F38_MIP_POINT_PRECLAMP(1) |
1785                              S_008F38_DISABLE_LSB_CEIL(1) |
1786                              S_008F38_FILTER_PREC_FIX(1) |
1787                              S_008F38_ANISO_OVERRIDE(is_vi));
1788         sampler->state[3] = (S_008F3C_BORDER_COLOR_PTR(0) |
1789                              S_008F3C_BORDER_COLOR_TYPE(radv_tex_bordercolor(pCreateInfo->borderColor)));
1790 }
1791
1792 VkResult radv_CreateSampler(
1793         VkDevice                                    _device,
1794         const VkSamplerCreateInfo*                  pCreateInfo,
1795         const VkAllocationCallbacks*                pAllocator,
1796         VkSampler*                                  pSampler)
1797 {
1798         RADV_FROM_HANDLE(radv_device, device, _device);
1799         struct radv_sampler *sampler;
1800
1801         assert(pCreateInfo->sType == VK_STRUCTURE_TYPE_SAMPLER_CREATE_INFO);
1802
1803         sampler = vk_alloc2(&device->alloc, pAllocator, sizeof(*sampler), 8,
1804                               VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
1805         if (!sampler)
1806                 return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY);
1807
1808         radv_init_sampler(device, sampler, pCreateInfo);
1809         *pSampler = radv_sampler_to_handle(sampler);
1810
1811         return VK_SUCCESS;
1812 }
1813
1814 void radv_DestroySampler(
1815         VkDevice                                    _device,
1816         VkSampler                                   _sampler,
1817         const VkAllocationCallbacks*                pAllocator)
1818 {
1819         RADV_FROM_HANDLE(radv_device, device, _device);
1820         RADV_FROM_HANDLE(radv_sampler, sampler, _sampler);
1821
1822         if (!sampler)
1823                 return;
1824         vk_free2(&device->alloc, pAllocator, sampler);
1825 }