src/gallium/drivers/radeon/r600_texture.c

   1 /*
   2  * Copyright 2010 Jerome Glisse <glisse@freedesktop.org>
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * on the rights to use, copy, modify, merge, publish, distribute, sub
   8  * license, and/or sell copies of the Software, and to permit persons to whom
   9  * the Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
  18  * THE AUTHOR(S) AND/OR THEIR SUPPLIERS BE LIABLE FOR ANY CLAIM,
  19  * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
  20  * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
  21  * USE OR OTHER DEALINGS IN THE SOFTWARE.
  22  *
  23  * Authors:
  24  *      Jerome Glisse
  25  *      Corbin Simpson
  26  */
  27 #include "r600_pipe_common.h"
  28 #include "r600_cs.h"
  29 #include "util/u_format.h"
  30 #include "util/u_memory.h"
  31 #include "util/u_pack_color.h"
  32 #include <errno.h>
  33 #include <inttypes.h>
  34
  35 static void r600_texture_discard_dcc(struct r600_common_screen *rscreen,
  36                                      struct r600_texture *rtex);
  37 static void r600_texture_discard_cmask(struct r600_common_screen *rscreen,
  38                                        struct r600_texture *rtex);
  39
  40
  41 static bool range_covers_whole_texture(struct pipe_resource *tex,
  42                                        unsigned level, unsigned x, unsigned y,
  43                                        unsigned z, unsigned width,
  44                                        unsigned height, unsigned depth)
  45 {
  46         return x == 0 && y == 0 && z == 0 &&
  47                width == u_minify(tex->width0, level) &&
  48                height == u_minify(tex->height0, level) &&
  49                depth == util_max_layer(tex, level) + 1;
  50 }
  51
  52 bool r600_prepare_for_dma_blit(struct r600_common_context *rctx,
  53                                struct r600_texture *rdst,
  54                                unsigned dst_level, unsigned dstx,
  55                                unsigned dsty, unsigned dstz,
  56                                struct r600_texture *rsrc,
  57                                unsigned src_level,
  58                                const struct pipe_box *src_box)
  59 {
  60         if (!rctx->dma.cs)
  61                 return false;
  62
  63         if (util_format_get_blocksizebits(rdst->resource.b.b.format) !=
  64             util_format_get_blocksizebits(rsrc->resource.b.b.format))
  65                 return false;
  66
  67         /* MSAA: Blits don't exist in the real world. */
  68         if (rsrc->resource.b.b.nr_samples > 1 ||
  69             rdst->resource.b.b.nr_samples > 1)
  70                 return false;
  71
  72         /* Depth-stencil surfaces:
  73          *   When dst is linear, the DB->CB copy preserves HTILE.
  74          *   When dst is tiled, the 3D path must be used to update HTILE.
  75          */
  76         if (rsrc->is_depth || rdst->is_depth)
  77                 return false;
  78
  79         /* DCC as:
  80          *   src: Use the 3D path. DCC decompression is expensive.
  81          *   dst: If overwriting the whole texture, discard DCC and use SDMA.
  82          *        Otherwise, use the 3D path.
  83          */
  84         if (rsrc->dcc_offset)
  85                 return false;
  86
  87         if (rdst->dcc_offset) {
  88                 /* We can't discard DCC if the texture has been exported. */
  89                 if (rdst->resource.is_shared ||
  90                     !range_covers_whole_texture(&rdst->resource.b.b, dst_level,
  91                                                 dstx, dsty, dstz, src_box->width,
  92                                                 src_box->height, src_box->depth))
  93                         return false;
  94
  95                 r600_texture_discard_dcc(rctx->screen, rdst);
  96         }
  97
  98         /* CMASK as:
  99          *   src: Both texture and SDMA paths need decompression. Use SDMA.
 100          *   dst: If overwriting the whole texture, discard CMASK and use
 101          *        SDMA. Otherwise, use the 3D path.
 102          */
 103         if (rdst->cmask.size && rdst->dirty_level_mask & (1 << dst_level)) {
 104                 if (!range_covers_whole_texture(&rdst->resource.b.b, dst_level,
 105                                                 dstx, dsty, dstz, src_box->width,
 106                                                 src_box->height, src_box->depth))
 107                         return false;
 108
 109                 r600_texture_discard_cmask(rctx->screen, rdst);
 110         }
 111
 112         /* All requirements are met. Prepare textures for SDMA. */
 113         if (rsrc->cmask.size && rsrc->dirty_level_mask & (1 << src_level))
 114                 rctx->b.flush_resource(&rctx->b, &rsrc->resource.b.b);
 115
 116         assert(!(rsrc->dirty_level_mask & (1 << src_level)));
 117         assert(!(rdst->dirty_level_mask & (1 << dst_level)));
 118
 119         return true;
 120 }
 121
 122 /* Same as resource_copy_region, except that both upsampling and downsampling are allowed. */
 123 static void r600_copy_region_with_blit(struct pipe_context *pipe,
 124                                        struct pipe_resource *dst,
 125                                        unsigned dst_level,
 126                                        unsigned dstx, unsigned dsty, unsigned dstz,
 127                                        struct pipe_resource *src,
 128                                        unsigned src_level,
 129                                        const struct pipe_box *src_box)
 130 {
 131         struct pipe_blit_info blit;
 132
 133         memset(&blit, 0, sizeof(blit));
 134         blit.src.resource = src;
 135         blit.src.format = src->format;
 136         blit.src.level = src_level;
 137         blit.src.box = *src_box;
 138         blit.dst.resource = dst;
 139         blit.dst.format = dst->format;
 140         blit.dst.level = dst_level;
 141         blit.dst.box.x = dstx;
 142         blit.dst.box.y = dsty;
 143         blit.dst.box.z = dstz;
 144         blit.dst.box.width = src_box->width;
 145         blit.dst.box.height = src_box->height;
 146         blit.dst.box.depth = src_box->depth;
 147         blit.mask = util_format_get_mask(src->format) &
 148                     util_format_get_mask(dst->format);
 149         blit.filter = PIPE_TEX_FILTER_NEAREST;
 150
 151         if (blit.mask) {
 152                 pipe->blit(pipe, &blit);
 153         }
 154 }
 155
 156 /* Copy from a full GPU texture to a transfer's staging one. */
 157 static void r600_copy_to_staging_texture(struct pipe_context *ctx, struct r600_transfer *rtransfer)
 158 {
 159         struct r600_common_context *rctx = (struct r600_common_context*)ctx;
 160         struct pipe_transfer *transfer = (struct pipe_transfer*)rtransfer;
 161         struct pipe_resource *dst = &rtransfer->staging->b.b;
 162         struct pipe_resource *src = transfer->resource;
 163
 164         if (src->nr_samples > 1) {
 165                 r600_copy_region_with_blit(ctx, dst, 0, 0, 0, 0,
 166                                            src, transfer->level, &transfer->box);
 167                 return;
 168         }
 169
 170         rctx->dma_copy(ctx, dst, 0, 0, 0, 0, src, transfer->level,
 171                        &transfer->box);
 172 }
 173
 174 /* Copy from a transfer's staging texture to a full GPU one. */
 175 static void r600_copy_from_staging_texture(struct pipe_context *ctx, struct r600_transfer *rtransfer)
 176 {
 177         struct r600_common_context *rctx = (struct r600_common_context*)ctx;
 178         struct pipe_transfer *transfer = (struct pipe_transfer*)rtransfer;
 179         struct pipe_resource *dst = transfer->resource;
 180         struct pipe_resource *src = &rtransfer->staging->b.b;
 181         struct pipe_box sbox;
 182
 183         u_box_3d(0, 0, 0, transfer->box.width, transfer->box.height, transfer->box.depth, &sbox);
 184
 185         if (dst->nr_samples > 1) {
 186                 r600_copy_region_with_blit(ctx, dst, transfer->level,
 187                                            transfer->box.x, transfer->box.y, transfer->box.z,
 188                                            src, 0, &sbox);
 189                 return;
 190         }
 191
 192         rctx->dma_copy(ctx, dst, transfer->level,
 193                        transfer->box.x, transfer->box.y, transfer->box.z,
 194                        src, 0, &sbox);
 195 }
 196
 197 static unsigned r600_texture_get_offset(struct r600_texture *rtex, unsigned level,
 198                                         const struct pipe_box *box)
 199 {
 200         enum pipe_format format = rtex->resource.b.b.format;
 201
 202         return rtex->surface.level[level].offset +
 203                box->z * rtex->surface.level[level].slice_size +
 204                box->y / util_format_get_blockheight(format) * rtex->surface.level[level].pitch_bytes +
 205                box->x / util_format_get_blockwidth(format) * util_format_get_blocksize(format);
 206 }
 207
 208 static int r600_init_surface(struct r600_common_screen *rscreen,
 209                              struct radeon_surf *surface,
 210                              const struct pipe_resource *ptex,
 211                              unsigned array_mode,
 212                              bool is_flushed_depth)
 213 {
 214         const struct util_format_description *desc =
 215                 util_format_description(ptex->format);
 216         bool is_depth, is_stencil;
 217
 218         is_depth = util_format_has_depth(desc);
 219         is_stencil = util_format_has_stencil(desc);
 220
 221         surface->npix_x = ptex->width0;
 222         surface->npix_y = ptex->height0;
 223         surface->npix_z = ptex->depth0;
 224         surface->blk_w = util_format_get_blockwidth(ptex->format);
 225         surface->blk_h = util_format_get_blockheight(ptex->format);
 226         surface->blk_d = 1;
 227         surface->array_size = 1;
 228         surface->last_level = ptex->last_level;
 229
 230         if (rscreen->chip_class >= EVERGREEN && !is_flushed_depth &&
 231             ptex->format == PIPE_FORMAT_Z32_FLOAT_S8X24_UINT) {
 232                 surface->bpe = 4; /* stencil is allocated separately on evergreen */
 233         } else {
 234                 surface->bpe = util_format_get_blocksize(ptex->format);
 235                 /* align byte per element on dword */
 236                 if (surface->bpe == 3) {
 237                         surface->bpe = 4;
 238                 }
 239         }
 240
 241         surface->nsamples = ptex->nr_samples ? ptex->nr_samples : 1;
 242         surface->flags = RADEON_SURF_SET(array_mode, MODE);
 243
 244         switch (ptex->target) {
 245         case PIPE_TEXTURE_1D:
 246                 surface->flags |= RADEON_SURF_SET(RADEON_SURF_TYPE_1D, TYPE);
 247                 break;
 248         case PIPE_TEXTURE_RECT:
 249         case PIPE_TEXTURE_2D:
 250                 surface->flags |= RADEON_SURF_SET(RADEON_SURF_TYPE_2D, TYPE);
 251                 break;
 252         case PIPE_TEXTURE_3D:
 253                 surface->flags |= RADEON_SURF_SET(RADEON_SURF_TYPE_3D, TYPE);
 254                 break;
 255         case PIPE_TEXTURE_1D_ARRAY:
 256                 surface->flags |= RADEON_SURF_SET(RADEON_SURF_TYPE_1D_ARRAY, TYPE);
 257                 surface->array_size = ptex->array_size;
 258                 break;
 259         case PIPE_TEXTURE_CUBE_ARRAY: /* cube array layout like 2d array */
 260                 assert(ptex->array_size % 6 == 0);
 261         case PIPE_TEXTURE_2D_ARRAY:
 262                 surface->flags |= RADEON_SURF_SET(RADEON_SURF_TYPE_2D_ARRAY, TYPE);
 263                 surface->array_size = ptex->array_size;
 264                 break;
 265         case PIPE_TEXTURE_CUBE:
 266                 surface->flags |= RADEON_SURF_SET(RADEON_SURF_TYPE_CUBEMAP, TYPE);
 267                 break;
 268         case PIPE_BUFFER:
 269         default:
 270                 return -EINVAL;
 271         }
 272         if (ptex->bind & PIPE_BIND_SCANOUT) {
 273                 surface->flags |= RADEON_SURF_SCANOUT;
 274         }
 275
 276         if (!is_flushed_depth && is_depth) {
 277                 surface->flags |= RADEON_SURF_ZBUFFER;
 278
 279                 if (is_stencil) {
 280                         surface->flags |= RADEON_SURF_SBUFFER |
 281                                           RADEON_SURF_HAS_SBUFFER_MIPTREE;
 282                 }
 283         }
 284         if (rscreen->chip_class >= SI) {
 285                 surface->flags |= RADEON_SURF_HAS_TILE_MODE_INDEX;
 286         }
 287         return 0;
 288 }
 289
 290 static int r600_setup_surface(struct pipe_screen *screen,
 291                               struct r600_texture *rtex,
 292                               unsigned pitch_in_bytes_override,
 293                               unsigned offset)
 294 {
 295         struct r600_common_screen *rscreen = (struct r600_common_screen*)screen;
 296         unsigned i;
 297         int r;
 298
 299         r = rscreen->ws->surface_init(rscreen->ws, &rtex->surface);
 300         if (r) {
 301                 return r;
 302         }
 303
 304         rtex->size = rtex->surface.bo_size;
 305
 306         if (pitch_in_bytes_override && pitch_in_bytes_override != rtex->surface.level[0].pitch_bytes) {
 307                 /* old ddx on evergreen over estimate alignment for 1d, only 1 level
 308                  * for those
 309                  */
 310                 rtex->surface.level[0].nblk_x = pitch_in_bytes_override / rtex->surface.bpe;
 311                 rtex->surface.level[0].pitch_bytes = pitch_in_bytes_override;
 312                 rtex->surface.level[0].slice_size = pitch_in_bytes_override * rtex->surface.level[0].nblk_y;
 313         }
 314
 315         if (offset) {
 316                 for (i = 0; i < ARRAY_SIZE(rtex->surface.level); ++i)
 317                         rtex->surface.level[i].offset += offset;
 318         }
 319         return 0;
 320 }
 321
 322 static void r600_texture_init_metadata(struct r600_texture *rtex,
 323                                        struct radeon_bo_metadata *metadata)
 324 {
 325         struct radeon_surf *surface = &rtex->surface;
 326
 327         memset(metadata, 0, sizeof(*metadata));
 328         metadata->microtile = surface->level[0].mode >= RADEON_SURF_MODE_1D ?
 329                                    RADEON_LAYOUT_TILED : RADEON_LAYOUT_LINEAR;
 330         metadata->macrotile = surface->level[0].mode >= RADEON_SURF_MODE_2D ?
 331                                    RADEON_LAYOUT_TILED : RADEON_LAYOUT_LINEAR;
 332         metadata->pipe_config = surface->pipe_config;
 333         metadata->bankw = surface->bankw;
 334         metadata->bankh = surface->bankh;
 335         metadata->tile_split = surface->tile_split;
 336         metadata->mtilea = surface->mtilea;
 337         metadata->num_banks = surface->num_banks;
 338         metadata->stride = surface->level[0].pitch_bytes;
 339         metadata->scanout = (surface->flags & RADEON_SURF_SCANOUT) != 0;
 340 }
 341
 342 static void r600_dirty_all_framebuffer_states(struct r600_common_screen *rscreen)
 343 {
 344         p_atomic_inc(&rscreen->dirty_fb_counter);
 345 }
 346
 347 static void r600_eliminate_fast_color_clear(struct r600_common_screen *rscreen,
 348                                       struct r600_texture *rtex)
 349 {
 350         struct pipe_context *ctx = rscreen->aux_context;
 351
 352         pipe_mutex_lock(rscreen->aux_context_lock);
 353         ctx->flush_resource(ctx, &rtex->resource.b.b);
 354         ctx->flush(ctx, NULL, 0);
 355         pipe_mutex_unlock(rscreen->aux_context_lock);
 356 }
 357
 358 static void r600_texture_discard_cmask(struct r600_common_screen *rscreen,
 359                                        struct r600_texture *rtex)
 360 {
 361         if (!rtex->cmask.size)
 362                 return;
 363
 364         assert(rtex->resource.b.b.nr_samples <= 1);
 365
 366         /* Disable CMASK. */
 367         memset(&rtex->cmask, 0, sizeof(rtex->cmask));
 368         rtex->cmask.base_address_reg = rtex->resource.gpu_address >> 8;
 369
 370         if (rscreen->chip_class >= SI)
 371                 rtex->cb_color_info &= ~SI_S_028C70_FAST_CLEAR(1);
 372         else
 373                 rtex->cb_color_info &= ~EG_S_028C70_FAST_CLEAR(1);
 374
 375         if (rtex->cmask_buffer != &rtex->resource)
 376             pipe_resource_reference((struct pipe_resource**)&rtex->cmask_buffer, NULL);
 377
 378         /* Notify all contexts about the change. */
 379         r600_dirty_all_framebuffer_states(rscreen);
 380         p_atomic_inc(&rscreen->compressed_colortex_counter);
 381 }
 382
 383 static void r600_texture_discard_dcc(struct r600_common_screen *rscreen,
 384                                      struct r600_texture *rtex)
 385 {
 386         /* Disable DCC. */
 387         rtex->dcc_offset = 0;
 388         rtex->cb_color_info &= ~VI_S_028C70_DCC_ENABLE(1);
 389
 390         /* Notify all contexts about the change. */
 391         r600_dirty_all_framebuffer_states(rscreen);
 392 }
 393
 394 void r600_texture_disable_dcc(struct r600_common_screen *rscreen,
 395                               struct r600_texture *rtex)
 396 {
 397         struct r600_common_context *rctx =
 398                 (struct r600_common_context *)rscreen->aux_context;
 399
 400         if (!rtex->dcc_offset)
 401                 return;
 402
 403         /* Decompress DCC. */
 404         pipe_mutex_lock(rscreen->aux_context_lock);
 405         rctx->decompress_dcc(&rctx->b, rtex);
 406         rctx->b.flush(&rctx->b, NULL, 0);
 407         pipe_mutex_unlock(rscreen->aux_context_lock);
 408
 409         r600_texture_discard_dcc(rscreen, rtex);
 410 }
 411
 412 static boolean r600_texture_get_handle(struct pipe_screen* screen,
 413                                        struct pipe_resource *resource,
 414                                        struct winsys_handle *whandle,
 415                                        unsigned usage)
 416 {
 417         struct r600_common_screen *rscreen = (struct r600_common_screen*)screen;
 418         struct r600_resource *res = (struct r600_resource*)resource;
 419         struct r600_texture *rtex = (struct r600_texture*)resource;
 420         struct radeon_bo_metadata metadata;
 421         bool update_metadata = false;
 422
 423         /* This is not supported now, but it might be required for OpenCL
 424          * interop in the future.
 425          */
 426         if (resource->target != PIPE_BUFFER &&
 427             (resource->nr_samples > 1 || rtex->is_depth))
 428                 return false;
 429
 430         if (resource->target != PIPE_BUFFER) {
 431                 /* Since shader image stores don't support DCC on VI,
 432                  * disable it for external clients that want write
 433                  * access.
 434                  */
 435                 if (usage & PIPE_HANDLE_USAGE_WRITE && rtex->dcc_offset) {
 436                         r600_texture_disable_dcc(rscreen, rtex);
 437                         update_metadata = true;
 438                 }
 439
 440                 if (!(usage & PIPE_HANDLE_USAGE_EXPLICIT_FLUSH) &&
 441                     rtex->cmask.size) {
 442                         /* Eliminate fast clear (both CMASK and DCC) */
 443                         r600_eliminate_fast_color_clear(rscreen, rtex);
 444
 445                         /* Disable CMASK if flush_resource isn't going
 446                          * to be called.
 447                          */
 448                         r600_texture_discard_cmask(rscreen, rtex);
 449                         update_metadata = true;
 450                 }
 451
 452                 /* Set metadata. */
 453                 if (!res->is_shared || update_metadata) {
 454                         r600_texture_init_metadata(rtex, &metadata);
 455                         if (rscreen->query_opaque_metadata)
 456                                 rscreen->query_opaque_metadata(rscreen, rtex,
 457                                                                &metadata);
 458
 459                         rscreen->ws->buffer_set_metadata(res->buf, &metadata);
 460                 }
 461         }
 462
 463         if (res->is_shared) {
 464                 /* USAGE_EXPLICIT_FLUSH must be cleared if at least one user
 465                  * doesn't set it.
 466                  */
 467                 res->external_usage |= usage & ~PIPE_HANDLE_USAGE_EXPLICIT_FLUSH;
 468                 if (!(usage & PIPE_HANDLE_USAGE_EXPLICIT_FLUSH))
 469                         res->external_usage &= ~PIPE_HANDLE_USAGE_EXPLICIT_FLUSH;
 470         } else {
 471                 res->is_shared = true;
 472                 res->external_usage = usage;
 473         }
 474
 475         return rscreen->ws->buffer_get_handle(res->buf,
 476                                               rtex->surface.level[0].pitch_bytes,
 477                                               rtex->surface.level[0].offset,
 478                                               rtex->surface.level[0].slice_size,
 479                                               whandle);
 480 }
 481
 482 static void r600_texture_destroy(struct pipe_screen *screen,
 483                                  struct pipe_resource *ptex)
 484 {
 485         struct r600_texture *rtex = (struct r600_texture*)ptex;
 486         struct r600_resource *resource = &rtex->resource;
 487
 488         if (rtex->flushed_depth_texture)
 489                 pipe_resource_reference((struct pipe_resource **)&rtex->flushed_depth_texture, NULL);
 490
 491         pipe_resource_reference((struct pipe_resource**)&rtex->htile_buffer, NULL);
 492         if (rtex->cmask_buffer != &rtex->resource) {
 493             pipe_resource_reference((struct pipe_resource**)&rtex->cmask_buffer, NULL);
 494         }
 495         pb_reference(&resource->buf, NULL);
 496         FREE(rtex);
 497 }
 498
 499 static const struct u_resource_vtbl r600_texture_vtbl;
 500
 501 /* The number of samples can be specified independently of the texture. */
 502 void r600_texture_get_fmask_info(struct r600_common_screen *rscreen,
 503                                  struct r600_texture *rtex,
 504                                  unsigned nr_samples,
 505                                  struct r600_fmask_info *out)
 506 {
 507         /* FMASK is allocated like an ordinary texture. */
 508         struct radeon_surf fmask = rtex->surface;
 509
 510         memset(out, 0, sizeof(*out));
 511
 512         fmask.bo_alignment = 0;
 513         fmask.bo_size = 0;
 514         fmask.nsamples = 1;
 515         fmask.flags |= RADEON_SURF_FMASK;
 516
 517         /* Force 2D tiling if it wasn't set. This may occur when creating
 518          * FMASK for MSAA resolve on R6xx. On R6xx, the single-sample
 519          * destination buffer must have an FMASK too. */
 520         fmask.flags = RADEON_SURF_CLR(fmask.flags, MODE);
 521         fmask.flags |= RADEON_SURF_SET(RADEON_SURF_MODE_2D, MODE);
 522
 523         if (rscreen->chip_class >= SI) {
 524                 fmask.flags |= RADEON_SURF_HAS_TILE_MODE_INDEX;
 525         }
 526
 527         switch (nr_samples) {
 528         case 2:
 529         case 4:
 530                 fmask.bpe = 1;
 531                 if (rscreen->chip_class <= CAYMAN) {
 532                         fmask.bankh = 4;
 533                 }
 534                 break;
 535         case 8:
 536                 fmask.bpe = 4;
 537                 break;
 538         default:
 539                 R600_ERR("Invalid sample count for FMASK allocation.\n");
 540                 return;
 541         }
 542
 543         /* Overallocate FMASK on R600-R700 to fix colorbuffer corruption.
 544          * This can be fixed by writing a separate FMASK allocator specifically
 545          * for R600-R700 asics. */
 546         if (rscreen->chip_class <= R700) {
 547                 fmask.bpe *= 2;
 548         }
 549
 550         if (rscreen->ws->surface_init(rscreen->ws, &fmask)) {
 551                 R600_ERR("Got error in surface_init while allocating FMASK.\n");
 552                 return;
 553         }
 554
 555         assert(fmask.level[0].mode == RADEON_SURF_MODE_2D);
 556
 557         out->slice_tile_max = (fmask.level[0].nblk_x * fmask.level[0].nblk_y) / 64;
 558         if (out->slice_tile_max)
 559                 out->slice_tile_max -= 1;
 560
 561         out->tile_mode_index = fmask.tiling_index[0];
 562         out->pitch_in_pixels = fmask.level[0].nblk_x;
 563         out->bank_height = fmask.bankh;
 564         out->alignment = MAX2(256, fmask.bo_alignment);
 565         out->size = fmask.bo_size;
 566 }
 567
 568 static void r600_texture_allocate_fmask(struct r600_common_screen *rscreen,
 569                                         struct r600_texture *rtex)
 570 {
 571         r600_texture_get_fmask_info(rscreen, rtex,
 572                                     rtex->resource.b.b.nr_samples, &rtex->fmask);
 573
 574         rtex->fmask.offset = align64(rtex->size, rtex->fmask.alignment);
 575         rtex->size = rtex->fmask.offset + rtex->fmask.size;
 576 }
 577
 578 void r600_texture_get_cmask_info(struct r600_common_screen *rscreen,
 579                                  struct r600_texture *rtex,
 580                                  struct r600_cmask_info *out)
 581 {
 582         unsigned cmask_tile_width = 8;
 583         unsigned cmask_tile_height = 8;
 584         unsigned cmask_tile_elements = cmask_tile_width * cmask_tile_height;
 585         unsigned element_bits = 4;
 586         unsigned cmask_cache_bits = 1024;
 587         unsigned num_pipes = rscreen->info.num_tile_pipes;
 588         unsigned pipe_interleave_bytes = rscreen->info.pipe_interleave_bytes;
 589
 590         unsigned elements_per_macro_tile = (cmask_cache_bits / element_bits) * num_pipes;
 591         unsigned pixels_per_macro_tile = elements_per_macro_tile * cmask_tile_elements;
 592         unsigned sqrt_pixels_per_macro_tile = sqrt(pixels_per_macro_tile);
 593         unsigned macro_tile_width = util_next_power_of_two(sqrt_pixels_per_macro_tile);
 594         unsigned macro_tile_height = pixels_per_macro_tile / macro_tile_width;
 595
 596         unsigned pitch_elements = align(rtex->surface.npix_x, macro_tile_width);
 597         unsigned height = align(rtex->surface.npix_y, macro_tile_height);
 598
 599         unsigned base_align = num_pipes * pipe_interleave_bytes;
 600         unsigned slice_bytes =
 601                 ((pitch_elements * height * element_bits + 7) / 8) / cmask_tile_elements;
 602
 603         assert(macro_tile_width % 128 == 0);
 604         assert(macro_tile_height % 128 == 0);
 605
 606         out->pitch = pitch_elements;
 607         out->height = height;
 608         out->xalign = macro_tile_width;
 609         out->yalign = macro_tile_height;
 610         out->slice_tile_max = ((pitch_elements * height) / (128*128)) - 1;
 611         out->alignment = MAX2(256, base_align);
 612         out->size = (util_max_layer(&rtex->resource.b.b, 0) + 1) *
 613                     align(slice_bytes, base_align);
 614 }
 615
 616 static void si_texture_get_cmask_info(struct r600_common_screen *rscreen,
 617                                       struct r600_texture *rtex,
 618                                       struct r600_cmask_info *out)
 619 {
 620         unsigned pipe_interleave_bytes = rscreen->info.pipe_interleave_bytes;
 621         unsigned num_pipes = rscreen->info.num_tile_pipes;
 622         unsigned cl_width, cl_height;
 623
 624         switch (num_pipes) {
 625         case 2:
 626                 cl_width = 32;
 627                 cl_height = 16;
 628                 break;
 629         case 4:
 630                 cl_width = 32;
 631                 cl_height = 32;
 632                 break;
 633         case 8:
 634                 cl_width = 64;
 635                 cl_height = 32;
 636                 break;
 637         case 16: /* Hawaii */
 638                 cl_width = 64;
 639                 cl_height = 64;
 640                 break;
 641         default:
 642                 assert(0);
 643                 return;
 644         }
 645
 646         unsigned base_align = num_pipes * pipe_interleave_bytes;
 647
 648         unsigned width = align(rtex->surface.npix_x, cl_width*8);
 649         unsigned height = align(rtex->surface.npix_y, cl_height*8);
 650         unsigned slice_elements = (width * height) / (8*8);
 651
 652         /* Each element of CMASK is a nibble. */
 653         unsigned slice_bytes = slice_elements / 2;
 654
 655         out->pitch = width;
 656         out->height = height;
 657         out->xalign = cl_width * 8;
 658         out->yalign = cl_height * 8;
 659         out->slice_tile_max = (width * height) / (128*128);
 660         if (out->slice_tile_max)
 661                 out->slice_tile_max -= 1;
 662
 663         out->alignment = MAX2(256, base_align);
 664         out->size = (util_max_layer(&rtex->resource.b.b, 0) + 1) *
 665                     align(slice_bytes, base_align);
 666 }
 667
 668 static void r600_texture_allocate_cmask(struct r600_common_screen *rscreen,
 669                                         struct r600_texture *rtex)
 670 {
 671         if (rscreen->chip_class >= SI) {
 672                 si_texture_get_cmask_info(rscreen, rtex, &rtex->cmask);
 673         } else {
 674                 r600_texture_get_cmask_info(rscreen, rtex, &rtex->cmask);
 675         }
 676
 677         rtex->cmask.offset = align64(rtex->size, rtex->cmask.alignment);
 678         rtex->size = rtex->cmask.offset + rtex->cmask.size;
 679
 680         if (rscreen->chip_class >= SI)
 681                 rtex->cb_color_info |= SI_S_028C70_FAST_CLEAR(1);
 682         else
 683                 rtex->cb_color_info |= EG_S_028C70_FAST_CLEAR(1);
 684 }
 685
 686 static void r600_texture_alloc_cmask_separate(struct r600_common_screen *rscreen,
 687                                               struct r600_texture *rtex)
 688 {
 689         if (rtex->cmask_buffer)
 690                 return;
 691
 692         assert(rtex->cmask.size == 0);
 693
 694         if (rscreen->chip_class >= SI) {
 695                 si_texture_get_cmask_info(rscreen, rtex, &rtex->cmask);
 696         } else {
 697                 r600_texture_get_cmask_info(rscreen, rtex, &rtex->cmask);
 698         }
 699
 700         rtex->cmask_buffer = (struct r600_resource *)
 701                 pipe_buffer_create(&rscreen->b, PIPE_BIND_CUSTOM,
 702                                    PIPE_USAGE_DEFAULT, rtex->cmask.size);
 703         if (rtex->cmask_buffer == NULL) {
 704                 rtex->cmask.size = 0;
 705                 return;
 706         }
 707
 708         /* update colorbuffer state bits */
 709         rtex->cmask.base_address_reg = rtex->cmask_buffer->gpu_address >> 8;
 710
 711         if (rscreen->chip_class >= SI)
 712                 rtex->cb_color_info |= SI_S_028C70_FAST_CLEAR(1);
 713         else
 714                 rtex->cb_color_info |= EG_S_028C70_FAST_CLEAR(1);
 715
 716         p_atomic_inc(&rscreen->compressed_colortex_counter);
 717 }
 718
 719 static unsigned r600_texture_get_htile_size(struct r600_common_screen *rscreen,
 720                                             struct r600_texture *rtex)
 721 {
 722         unsigned cl_width, cl_height, width, height;
 723         unsigned slice_elements, slice_bytes, pipe_interleave_bytes, base_align;
 724         unsigned num_pipes = rscreen->info.num_tile_pipes;
 725
 726         if (rscreen->chip_class <= EVERGREEN &&
 727             rscreen->info.drm_major == 2 && rscreen->info.drm_minor < 26)
 728                 return 0;
 729
 730         /* HW bug on R6xx. */
 731         if (rscreen->chip_class == R600 &&
 732             (rtex->surface.level[0].npix_x > 7680 ||
 733              rtex->surface.level[0].npix_y > 7680))
 734                 return 0;
 735
 736         /* HTILE is broken with 1D tiling on old kernels and CIK. */
 737         if (rscreen->chip_class >= CIK &&
 738             rtex->surface.level[0].mode == RADEON_SURF_MODE_1D &&
 739             rscreen->info.drm_major == 2 && rscreen->info.drm_minor < 38)
 740                 return 0;
 741
 742         /* Overalign HTILE on P2 configs to work around GPU hangs in
 743          * piglit/depthstencil-render-miplevels 585.
 744          *
 745          * This has been confirmed to help Kabini & Stoney, where the hangs
 746          * are always reproducible. I think I have seen the test hang
 747          * on Carrizo too, though it was very rare there.
 748          */
 749         if (rscreen->chip_class >= CIK && num_pipes < 4)
 750                 num_pipes = 4;
 751
 752         switch (num_pipes) {
 753         case 1:
 754                 cl_width = 32;
 755                 cl_height = 16;
 756                 break;
 757         case 2:
 758                 cl_width = 32;
 759                 cl_height = 32;
 760                 break;
 761         case 4:
 762                 cl_width = 64;
 763                 cl_height = 32;
 764                 break;
 765         case 8:
 766                 cl_width = 64;
 767                 cl_height = 64;
 768                 break;
 769         case 16:
 770                 cl_width = 128;
 771                 cl_height = 64;
 772                 break;
 773         default:
 774                 assert(0);
 775                 return 0;
 776         }
 777
 778         width = align(rtex->surface.npix_x, cl_width * 8);
 779         height = align(rtex->surface.npix_y, cl_height * 8);
 780
 781         slice_elements = (width * height) / (8 * 8);
 782         slice_bytes = slice_elements * 4;
 783
 784         pipe_interleave_bytes = rscreen->info.pipe_interleave_bytes;
 785         base_align = num_pipes * pipe_interleave_bytes;
 786
 787         rtex->htile.pitch = width;
 788         rtex->htile.height = height;
 789         rtex->htile.xalign = cl_width * 8;
 790         rtex->htile.yalign = cl_height * 8;
 791
 792         return (util_max_layer(&rtex->resource.b.b, 0) + 1) *
 793                 align(slice_bytes, base_align);
 794 }
 795
 796 static void r600_texture_allocate_htile(struct r600_common_screen *rscreen,
 797                                         struct r600_texture *rtex)
 798 {
 799         unsigned htile_size = r600_texture_get_htile_size(rscreen, rtex);
 800
 801         if (!htile_size)
 802                 return;
 803
 804         rtex->htile_buffer = (struct r600_resource*)
 805                              pipe_buffer_create(&rscreen->b, PIPE_BIND_CUSTOM,
 806                                                 PIPE_USAGE_DEFAULT, htile_size);
 807         if (rtex->htile_buffer == NULL) {
 808                 /* this is not a fatal error as we can still keep rendering
 809                  * without htile buffer */
 810                 R600_ERR("Failed to create buffer object for htile buffer.\n");
 811         } else {
 812                 r600_screen_clear_buffer(rscreen, &rtex->htile_buffer->b.b, 0,
 813                                          htile_size, 0, R600_COHERENCY_NONE);
 814         }
 815 }
 816
 817 void r600_print_texture_info(struct r600_texture *rtex, FILE *f)
 818 {
 819         int i;
 820
 821         fprintf(f, "  Info: npix_x=%u, npix_y=%u, npix_z=%u, blk_w=%u, "
 822                 "blk_h=%u, blk_d=%u, array_size=%u, last_level=%u, "
 823                 "bpe=%u, nsamples=%u, flags=0x%x, %s\n",
 824                 rtex->surface.npix_x, rtex->surface.npix_y,
 825                 rtex->surface.npix_z, rtex->surface.blk_w,
 826                 rtex->surface.blk_h, rtex->surface.blk_d,
 827                 rtex->surface.array_size, rtex->surface.last_level,
 828                 rtex->surface.bpe, rtex->surface.nsamples,
 829                 rtex->surface.flags, util_format_short_name(rtex->resource.b.b.format));
 830
 831         fprintf(f, "  Layout: size=%"PRIu64", alignment=%"PRIu64", bankw=%u, "
 832                 "bankh=%u, nbanks=%u, mtilea=%u, tilesplit=%u, pipeconfig=%u, scanout=%u\n",
 833                 rtex->surface.bo_size, rtex->surface.bo_alignment, rtex->surface.bankw,
 834                 rtex->surface.bankh, rtex->surface.num_banks, rtex->surface.mtilea,
 835                 rtex->surface.tile_split, rtex->surface.pipe_config,
 836                 (rtex->surface.flags & RADEON_SURF_SCANOUT) != 0);
 837
 838         if (rtex->fmask.size)
 839                 fprintf(f, "  FMask: offset=%"PRIu64", size=%"PRIu64", alignment=%u, pitch_in_pixels=%u, "
 840                         "bankh=%u, slice_tile_max=%u, tile_mode_index=%u\n",
 841                         rtex->fmask.offset, rtex->fmask.size, rtex->fmask.alignment,
 842                         rtex->fmask.pitch_in_pixels, rtex->fmask.bank_height,
 843                         rtex->fmask.slice_tile_max, rtex->fmask.tile_mode_index);
 844
 845         if (rtex->cmask.size)
 846                 fprintf(f, "  CMask: offset=%"PRIu64", size=%"PRIu64", alignment=%u, pitch=%u, "
 847                         "height=%u, xalign=%u, yalign=%u, slice_tile_max=%u\n",
 848                         rtex->cmask.offset, rtex->cmask.size, rtex->cmask.alignment,
 849                         rtex->cmask.pitch, rtex->cmask.height, rtex->cmask.xalign,
 850                         rtex->cmask.yalign, rtex->cmask.slice_tile_max);
 851
 852         if (rtex->htile_buffer)
 853                 fprintf(f, "  HTile: size=%u, alignment=%u, pitch=%u, height=%u, "
 854                         "xalign=%u, yalign=%u\n",
 855                         rtex->htile_buffer->b.b.width0,
 856                         rtex->htile_buffer->buf->alignment, rtex->htile.pitch,
 857                         rtex->htile.height, rtex->htile.xalign, rtex->htile.yalign);
 858
 859         if (rtex->dcc_offset) {
 860                 fprintf(f, "  DCC: offset=%"PRIu64", size=%"PRIu64", alignment=%"PRIu64"\n",
 861                         rtex->dcc_offset, rtex->surface.dcc_size,
 862                         rtex->surface.dcc_alignment);
 863                 for (i = 0; i <= rtex->surface.last_level; i++)
 864                         fprintf(f, "  DCCLevel[%i]: offset=%"PRIu64"\n",
 865                                 i, rtex->surface.level[i].dcc_offset);
 866         }
 867
 868         for (i = 0; i <= rtex->surface.last_level; i++)
 869                 fprintf(f, "  Level[%i]: offset=%"PRIu64", slice_size=%"PRIu64", "
 870                         "npix_x=%u, npix_y=%u, npix_z=%u, nblk_x=%u, nblk_y=%u, "
 871                         "nblk_z=%u, pitch_bytes=%u, mode=%u\n",
 872                         i, rtex->surface.level[i].offset,
 873                         rtex->surface.level[i].slice_size,
 874                         u_minify(rtex->resource.b.b.width0, i),
 875                         u_minify(rtex->resource.b.b.height0, i),
 876                         u_minify(rtex->resource.b.b.depth0, i),
 877                         rtex->surface.level[i].nblk_x,
 878                         rtex->surface.level[i].nblk_y,
 879                         rtex->surface.level[i].nblk_z,
 880                         rtex->surface.level[i].pitch_bytes,
 881                         rtex->surface.level[i].mode);
 882
 883         if (rtex->surface.flags & RADEON_SURF_SBUFFER) {
 884                 for (i = 0; i <= rtex->surface.last_level; i++) {
 885                         fprintf(f, "  StencilLayout: tilesplit=%u\n",
 886                                 rtex->surface.stencil_tile_split);
 887                         fprintf(f, "  StencilLevel[%i]: offset=%"PRIu64", "
 888                                 "slice_size=%"PRIu64", npix_x=%u, "
 889                                 "npix_y=%u, npix_z=%u, nblk_x=%u, nblk_y=%u, "
 890                                 "nblk_z=%u, pitch_bytes=%u, mode=%u\n",
 891                                 i, rtex->surface.stencil_level[i].offset,
 892                                 rtex->surface.stencil_level[i].slice_size,
 893                                 u_minify(rtex->resource.b.b.width0, i),
 894                                 u_minify(rtex->resource.b.b.height0, i),
 895                                 u_minify(rtex->resource.b.b.depth0, i),
 896                                 rtex->surface.stencil_level[i].nblk_x,
 897                                 rtex->surface.stencil_level[i].nblk_y,
 898                                 rtex->surface.stencil_level[i].nblk_z,
 899                                 rtex->surface.stencil_level[i].pitch_bytes,
 900                                 rtex->surface.stencil_level[i].mode);
 901                 }
 902         }
 903 }
 904
 905 /* Common processing for r600_texture_create and r600_texture_from_handle */
 906 static struct r600_texture *
 907 r600_texture_create_object(struct pipe_screen *screen,
 908                            const struct pipe_resource *base,
 909                            unsigned pitch_in_bytes_override,
 910                            unsigned offset,
 911                            struct pb_buffer *buf,
 912                            struct radeon_surf *surface)
 913 {
 914         struct r600_texture *rtex;
 915         struct r600_resource *resource;
 916         struct r600_common_screen *rscreen = (struct r600_common_screen*)screen;
 917
 918         rtex = CALLOC_STRUCT(r600_texture);
 919         if (!rtex)
 920                 return NULL;
 921
 922         resource = &rtex->resource;
 923         resource->b.b = *base;
 924         resource->b.vtbl = &r600_texture_vtbl;
 925         pipe_reference_init(&resource->b.b.reference, 1);
 926         resource->b.b.screen = screen;
 927
 928         /* don't include stencil-only formats which we don't support for rendering */
 929         rtex->is_depth = util_format_has_depth(util_format_description(rtex->resource.b.b.format));
 930
 931         rtex->surface = *surface;
 932         if (r600_setup_surface(screen, rtex, pitch_in_bytes_override, offset)) {
 933                 FREE(rtex);
 934                 return NULL;
 935         }
 936
 937         /* Tiled depth textures utilize the non-displayable tile order.
 938          * This must be done after r600_setup_surface.
 939          * Applies to R600-Cayman. */
 940         rtex->non_disp_tiling = rtex->is_depth && rtex->surface.level[0].mode >= RADEON_SURF_MODE_1D;
 941
 942         if (rtex->is_depth) {
 943                 if (!(base->flags & (R600_RESOURCE_FLAG_TRANSFER |
 944                                      R600_RESOURCE_FLAG_FLUSHED_DEPTH)) &&
 945                     !(rscreen->debug_flags & DBG_NO_HYPERZ)) {
 946
 947                         r600_texture_allocate_htile(rscreen, rtex);
 948                 }
 949         } else {
 950                 if (base->nr_samples > 1) {
 951                         if (!buf) {
 952                                 r600_texture_allocate_fmask(rscreen, rtex);
 953                                 r600_texture_allocate_cmask(rscreen, rtex);
 954                                 rtex->cmask_buffer = &rtex->resource;
 955                         }
 956                         if (!rtex->fmask.size || !rtex->cmask.size) {
 957                                 FREE(rtex);
 958                                 return NULL;
 959                         }
 960                 }
 961
 962                 if (!buf && rtex->surface.dcc_size &&
 963                     !(rscreen->debug_flags & DBG_NO_DCC)) {
 964                         /* Reserve space for the DCC buffer. */
 965                         rtex->dcc_offset = align64(rtex->size, rtex->surface.dcc_alignment);
 966                         rtex->size = rtex->dcc_offset + rtex->surface.dcc_size;
 967                         rtex->cb_color_info |= VI_S_028C70_DCC_ENABLE(1);
 968                 }
 969         }
 970
 971         /* Now create the backing buffer. */
 972         if (!buf) {
 973                 if (!r600_init_resource(rscreen, resource, rtex->size,
 974                                         rtex->surface.bo_alignment)) {
 975                         FREE(rtex);
 976                         return NULL;
 977                 }
 978         } else {
 979                 resource->buf = buf;
 980                 resource->gpu_address = rscreen->ws->buffer_get_virtual_address(resource->buf);
 981                 resource->domains = rscreen->ws->buffer_get_initial_domain(resource->buf);
 982         }
 983
 984         if (rtex->cmask.size) {
 985                 /* Initialize the cmask to 0xCC (= compressed state). */
 986                 r600_screen_clear_buffer(rscreen, &rtex->cmask_buffer->b.b,
 987                                          rtex->cmask.offset, rtex->cmask.size,
 988                                          0xCCCCCCCC, R600_COHERENCY_NONE);
 989         }
 990         if (rtex->dcc_offset) {
 991                 r600_screen_clear_buffer(rscreen, &rtex->resource.b.b,
 992                                          rtex->dcc_offset,
 993                                          rtex->surface.dcc_size,
 994                                          0xFFFFFFFF, R600_COHERENCY_NONE);
 995         }
 996
 997         /* Initialize the CMASK base register value. */
 998         rtex->cmask.base_address_reg =
 999                 (rtex->resource.gpu_address + rtex->cmask.offset) >> 8;
1000
1001         if (rscreen->debug_flags & DBG_VM) {
1002                 fprintf(stderr, "VM start=0x%"PRIX64"  end=0x%"PRIX64" | Texture %ix%ix%i, %i levels, %i samples, %s\n",
1003                         rtex->resource.gpu_address,
1004                         rtex->resource.gpu_address + rtex->resource.buf->size,
1005                         base->width0, base->height0, util_max_layer(base, 0)+1, base->last_level+1,
1006                         base->nr_samples ? base->nr_samples : 1, util_format_short_name(base->format));
1007         }
1008
1009         if (rscreen->debug_flags & DBG_TEX) {
1010                 puts("Texture:");
1011                 r600_print_texture_info(rtex, stdout);
1012         }
1013
1014         return rtex;
1015 }
1016
1017 static unsigned r600_choose_tiling(struct r600_common_screen *rscreen,
1018                                    const struct pipe_resource *templ)
1019 {
1020         const struct util_format_description *desc = util_format_description(templ->format);
1021         bool force_tiling = templ->flags & R600_RESOURCE_FLAG_FORCE_TILING;
1022
1023         /* MSAA resources must be 2D tiled. */
1024         if (templ->nr_samples > 1)
1025                 return RADEON_SURF_MODE_2D;
1026
1027         /* Transfer resources should be linear. */
1028         if (templ->flags & R600_RESOURCE_FLAG_TRANSFER)
1029                 return RADEON_SURF_MODE_LINEAR_ALIGNED;
1030
1031         /* r600g: force tiling on TEXTURE_2D and TEXTURE_3D compute resources. */
1032         if (rscreen->chip_class >= R600 && rscreen->chip_class <= CAYMAN &&
1033             (templ->bind & PIPE_BIND_COMPUTE_RESOURCE) &&
1034             (templ->target == PIPE_TEXTURE_2D ||
1035              templ->target == PIPE_TEXTURE_3D))
1036                 force_tiling = true;
1037
1038         /* Handle common candidates for the linear mode.
1039          * Compressed textures and DB surfaces must always be tiled.
1040          */
1041         if (!force_tiling && !util_format_is_compressed(templ->format) &&
1042             (!util_format_is_depth_or_stencil(templ->format) ||
1043              templ->flags & R600_RESOURCE_FLAG_FLUSHED_DEPTH)) {
1044                 if (rscreen->debug_flags & DBG_NO_TILING)
1045                         return RADEON_SURF_MODE_LINEAR_ALIGNED;
1046
1047                 /* Tiling doesn't work with the 422 (SUBSAMPLED) formats on R600+. */
1048                 if (desc->layout == UTIL_FORMAT_LAYOUT_SUBSAMPLED)
1049                         return RADEON_SURF_MODE_LINEAR_ALIGNED;
1050
1051                 /* Cursors are linear on SI.
1052                  * (XXX double-check, maybe also use RADEON_SURF_SCANOUT) */
1053                 if (rscreen->chip_class >= SI &&
1054                     (templ->bind & PIPE_BIND_CURSOR))
1055                         return RADEON_SURF_MODE_LINEAR_ALIGNED;
1056
1057                 if (templ->bind & PIPE_BIND_LINEAR)
1058                         return RADEON_SURF_MODE_LINEAR_ALIGNED;
1059
1060                 /* Textures with a very small height are recommended to be linear. */
1061                 if (templ->target == PIPE_TEXTURE_1D ||
1062                     templ->target == PIPE_TEXTURE_1D_ARRAY ||
1063                     templ->height0 <= 4)
1064                         return RADEON_SURF_MODE_LINEAR_ALIGNED;
1065
1066                 /* Textures likely to be mapped often. */
1067                 if (templ->usage == PIPE_USAGE_STAGING ||
1068                     templ->usage == PIPE_USAGE_STREAM)
1069                         return RADEON_SURF_MODE_LINEAR_ALIGNED;
1070         }
1071
1072         /* Make small textures 1D tiled. */
1073         if (templ->width0 <= 16 || templ->height0 <= 16 ||
1074             (rscreen->debug_flags & DBG_NO_2D_TILING))
1075                 return RADEON_SURF_MODE_1D;
1076
1077         /* The allocator will switch to 1D if needed. */
1078         return RADEON_SURF_MODE_2D;
1079 }
1080
1081 struct pipe_resource *r600_texture_create(struct pipe_screen *screen,
1082                                           const struct pipe_resource *templ)
1083 {
1084         struct r600_common_screen *rscreen = (struct r600_common_screen*)screen;
1085         struct radeon_surf surface = {0};
1086         int r;
1087
1088         r = r600_init_surface(rscreen, &surface, templ,
1089                               r600_choose_tiling(rscreen, templ),
1090                               templ->flags & R600_RESOURCE_FLAG_FLUSHED_DEPTH);
1091         if (r) {
1092                 return NULL;
1093         }
1094         r = rscreen->ws->surface_best(rscreen->ws, &surface);
1095         if (r) {
1096                 return NULL;
1097         }
1098         return (struct pipe_resource *)r600_texture_create_object(screen, templ, 0,
1099                                                                   0, NULL, &surface);
1100 }
1101
1102 static struct pipe_resource *r600_texture_from_handle(struct pipe_screen *screen,
1103                                                       const struct pipe_resource *templ,
1104                                                       struct winsys_handle *whandle,
1105                                                       unsigned usage)
1106 {
1107         struct r600_common_screen *rscreen = (struct r600_common_screen*)screen;
1108         struct pb_buffer *buf = NULL;
1109         unsigned stride = 0, offset = 0;
1110         unsigned array_mode;
1111         struct radeon_surf surface;
1112         int r;
1113         struct radeon_bo_metadata metadata = {};
1114         struct r600_texture *rtex;
1115
1116         /* Support only 2D textures without mipmaps */
1117         if ((templ->target != PIPE_TEXTURE_2D && templ->target != PIPE_TEXTURE_RECT) ||
1118               templ->depth0 != 1 || templ->last_level != 0)
1119                 return NULL;
1120
1121         buf = rscreen->ws->buffer_from_handle(rscreen->ws, whandle, &stride, &offset);
1122         if (!buf)
1123                 return NULL;
1124
1125         rscreen->ws->buffer_get_metadata(buf, &metadata);
1126
1127         surface.pipe_config = metadata.pipe_config;
1128         surface.bankw = metadata.bankw;
1129         surface.bankh = metadata.bankh;
1130         surface.tile_split = metadata.tile_split;
1131         surface.mtilea = metadata.mtilea;
1132         surface.num_banks = metadata.num_banks;
1133
1134         if (metadata.macrotile == RADEON_LAYOUT_TILED)
1135                 array_mode = RADEON_SURF_MODE_2D;
1136         else if (metadata.microtile == RADEON_LAYOUT_TILED)
1137                 array_mode = RADEON_SURF_MODE_1D;
1138         else
1139                 array_mode = RADEON_SURF_MODE_LINEAR_ALIGNED;
1140
1141         r = r600_init_surface(rscreen, &surface, templ, array_mode, false);
1142         if (r) {
1143                 return NULL;
1144         }
1145
1146         if (metadata.scanout)
1147                 surface.flags |= RADEON_SURF_SCANOUT;
1148
1149         rtex = r600_texture_create_object(screen, templ, stride,
1150                                           offset, buf, &surface);
1151         if (!rtex)
1152                 return NULL;
1153
1154         rtex->resource.is_shared = true;
1155         rtex->resource.external_usage = usage;
1156         return &rtex->resource.b.b;
1157 }
1158
1159 bool r600_init_flushed_depth_texture(struct pipe_context *ctx,
1160                                      struct pipe_resource *texture,
1161                                      struct r600_texture **staging)
1162 {
1163         struct r600_texture *rtex = (struct r600_texture*)texture;
1164         struct pipe_resource resource;
1165         struct r600_texture **flushed_depth_texture = staging ?
1166                         staging : &rtex->flushed_depth_texture;
1167
1168         if (!staging && rtex->flushed_depth_texture)
1169                 return true; /* it's ready */
1170
1171         resource.target = texture->target;
1172         resource.format = texture->format;
1173         resource.width0 = texture->width0;
1174         resource.height0 = texture->height0;
1175         resource.depth0 = texture->depth0;
1176         resource.array_size = texture->array_size;
1177         resource.last_level = texture->last_level;
1178         resource.nr_samples = texture->nr_samples;
1179         resource.usage = staging ? PIPE_USAGE_STAGING : PIPE_USAGE_DEFAULT;
1180         resource.bind = texture->bind & ~PIPE_BIND_DEPTH_STENCIL;
1181         resource.flags = texture->flags | R600_RESOURCE_FLAG_FLUSHED_DEPTH;
1182
1183         if (staging)
1184                 resource.flags |= R600_RESOURCE_FLAG_TRANSFER;
1185
1186         *flushed_depth_texture = (struct r600_texture *)ctx->screen->resource_create(ctx->screen, &resource);
1187         if (*flushed_depth_texture == NULL) {
1188                 R600_ERR("failed to create temporary texture to hold flushed depth\n");
1189                 return false;
1190         }
1191
1192         (*flushed_depth_texture)->is_flushing_texture = TRUE;
1193         (*flushed_depth_texture)->non_disp_tiling = false;
1194         return true;
1195 }
1196
1197 /**
1198  * Initialize the pipe_resource descriptor to be of the same size as the box,
1199  * which is supposed to hold a subregion of the texture "orig" at the given
1200  * mipmap level.
1201  */
1202 static void r600_init_temp_resource_from_box(struct pipe_resource *res,
1203                                              struct pipe_resource *orig,
1204                                              const struct pipe_box *box,
1205                                              unsigned level, unsigned flags)
1206 {
1207         memset(res, 0, sizeof(*res));
1208         res->format = orig->format;
1209         res->width0 = box->width;
1210         res->height0 = box->height;
1211         res->depth0 = 1;
1212         res->array_size = 1;
1213         res->usage = flags & R600_RESOURCE_FLAG_TRANSFER ? PIPE_USAGE_STAGING : PIPE_USAGE_DEFAULT;
1214         res->flags = flags;
1215
1216         /* We must set the correct texture target and dimensions for a 3D box. */
1217         if (box->depth > 1 && util_max_layer(orig, level) > 0) {
1218                 res->target = PIPE_TEXTURE_2D_ARRAY;
1219                 res->array_size = box->depth;
1220         } else {
1221                 res->target = PIPE_TEXTURE_2D;
1222         }
1223 }
1224
1225 static void *r600_texture_transfer_map(struct pipe_context *ctx,
1226                                        struct pipe_resource *texture,
1227                                        unsigned level,
1228                                        unsigned usage,
1229                                        const struct pipe_box *box,
1230                                        struct pipe_transfer **ptransfer)
1231 {
1232         struct r600_common_context *rctx = (struct r600_common_context*)ctx;
1233         struct r600_texture *rtex = (struct r600_texture*)texture;
1234         struct r600_transfer *trans;
1235         boolean use_staging_texture = FALSE;
1236         struct r600_resource *buf;
1237         unsigned offset = 0;
1238         char *map;
1239
1240         /* We cannot map a tiled texture directly because the data is
1241          * in a different order, therefore we do detiling using a blit.
1242          *
1243          * Also, use a temporary in GTT memory for read transfers, as
1244          * the CPU is much happier reading out of cached system memory
1245          * than uncached VRAM.
1246          */
1247         if (rtex->surface.level[0].mode >= RADEON_SURF_MODE_1D) {
1248                 use_staging_texture = TRUE;
1249         } else if ((usage & PIPE_TRANSFER_READ) &&
1250                    rtex->resource.domains & RADEON_DOMAIN_VRAM) {
1251                 /* Untiled buffers in VRAM, which is slow for CPU reads */
1252                 use_staging_texture = TRUE;
1253         } else if (!(usage & PIPE_TRANSFER_READ) &&
1254             (r600_rings_is_buffer_referenced(rctx, rtex->resource.buf, RADEON_USAGE_READWRITE) ||
1255              !rctx->ws->buffer_wait(rtex->resource.buf, 0, RADEON_USAGE_READWRITE))) {
1256                 /* Use a staging texture for uploads if the underlying BO is busy. */
1257                 use_staging_texture = TRUE;
1258         }
1259
1260         if (texture->flags & R600_RESOURCE_FLAG_TRANSFER) {
1261                 use_staging_texture = FALSE;
1262         }
1263
1264         trans = CALLOC_STRUCT(r600_transfer);
1265         if (!trans)
1266                 return NULL;
1267         trans->transfer.resource = texture;
1268         trans->transfer.level = level;
1269         trans->transfer.usage = usage;
1270         trans->transfer.box = *box;
1271
1272         if (rtex->is_depth) {
1273                 struct r600_texture *staging_depth;
1274
1275                 if (rtex->resource.b.b.nr_samples > 1) {
1276                         /* MSAA depth buffers need to be converted to single sample buffers.
1277                          *
1278                          * Mapping MSAA depth buffers can occur if ReadPixels is called
1279                          * with a multisample GLX visual.
1280                          *
1281                          * First downsample the depth buffer to a temporary texture,
1282                          * then decompress the temporary one to staging.
1283                          *
1284                          * Only the region being mapped is transfered.
1285                          */
1286                         struct pipe_resource resource;
1287
1288                         r600_init_temp_resource_from_box(&resource, texture, box, level, 0);
1289
1290                         if (!r600_init_flushed_depth_texture(ctx, &resource, &staging_depth)) {
1291                                 R600_ERR("failed to create temporary texture to hold untiled copy\n");
1292                                 FREE(trans);
1293                                 return NULL;
1294                         }
1295
1296                         if (usage & PIPE_TRANSFER_READ) {
1297                                 struct pipe_resource *temp = ctx->screen->resource_create(ctx->screen, &resource);
1298                                 if (!temp) {
1299                                         R600_ERR("failed to create a temporary depth texture\n");
1300                                         FREE(trans);
1301                                         return NULL;
1302                                 }
1303
1304                                 r600_copy_region_with_blit(ctx, temp, 0, 0, 0, 0, texture, level, box);
1305                                 rctx->blit_decompress_depth(ctx, (struct r600_texture*)temp, staging_depth,
1306                                                             0, 0, 0, box->depth, 0, 0);
1307                                 pipe_resource_reference(&temp, NULL);
1308                         }
1309                 }
1310                 else {
1311                         /* XXX: only readback the rectangle which is being mapped? */
1312                         /* XXX: when discard is true, no need to read back from depth texture */
1313                         if (!r600_init_flushed_depth_texture(ctx, texture, &staging_depth)) {
1314                                 R600_ERR("failed to create temporary texture to hold untiled copy\n");
1315                                 FREE(trans);
1316                                 return NULL;
1317                         }
1318
1319                         rctx->blit_decompress_depth(ctx, rtex, staging_depth,
1320                                                     level, level,
1321                                                     box->z, box->z + box->depth - 1,
1322                                                     0, 0);
1323
1324                         offset = r600_texture_get_offset(staging_depth, level, box);
1325                 }
1326
1327                 trans->transfer.stride = staging_depth->surface.level[level].pitch_bytes;
1328                 trans->transfer.layer_stride = staging_depth->surface.level[level].slice_size;
1329                 trans->staging = (struct r600_resource*)staging_depth;
1330         } else if (use_staging_texture) {
1331                 struct pipe_resource resource;
1332                 struct r600_texture *staging;
1333
1334                 r600_init_temp_resource_from_box(&resource, texture, box, level,
1335                                                  R600_RESOURCE_FLAG_TRANSFER);
1336                 resource.usage = (usage & PIPE_TRANSFER_READ) ?
1337                         PIPE_USAGE_STAGING : PIPE_USAGE_STREAM;
1338
1339                 /* Create the temporary texture. */
1340                 staging = (struct r600_texture*)ctx->screen->resource_create(ctx->screen, &resource);
1341                 if (!staging) {
1342                         R600_ERR("failed to create temporary texture to hold untiled copy\n");
1343                         FREE(trans);
1344                         return NULL;
1345                 }
1346                 trans->staging = &staging->resource;
1347                 trans->transfer.stride = staging->surface.level[0].pitch_bytes;
1348                 trans->transfer.layer_stride = staging->surface.level[0].slice_size;
1349                 if (usage & PIPE_TRANSFER_READ) {
1350                         r600_copy_to_staging_texture(ctx, trans);
1351                 }
1352         } else {
1353                 /* the resource is mapped directly */
1354                 trans->transfer.stride = rtex->surface.level[level].pitch_bytes;
1355                 trans->transfer.layer_stride = rtex->surface.level[level].slice_size;
1356                 offset = r600_texture_get_offset(rtex, level, box);
1357         }
1358
1359         if (trans->staging) {
1360                 buf = trans->staging;
1361                 if (!rtex->is_depth && !(usage & PIPE_TRANSFER_READ))
1362                         usage |= PIPE_TRANSFER_UNSYNCHRONIZED;
1363         } else {
1364                 buf = &rtex->resource;
1365         }
1366
1367         if (!(map = r600_buffer_map_sync_with_rings(rctx, buf, usage))) {
1368                 pipe_resource_reference((struct pipe_resource**)&trans->staging, NULL);
1369                 FREE(trans);
1370                 return NULL;
1371         }
1372
1373         *ptransfer = &trans->transfer;
1374         return map + offset;
1375 }
1376
1377 static void r600_texture_transfer_unmap(struct pipe_context *ctx,
1378                                         struct pipe_transfer* transfer)
1379 {
1380         struct r600_transfer *rtransfer = (struct r600_transfer*)transfer;
1381         struct pipe_resource *texture = transfer->resource;
1382         struct r600_texture *rtex = (struct r600_texture*)texture;
1383
1384         if ((transfer->usage & PIPE_TRANSFER_WRITE) && rtransfer->staging) {
1385                 if (rtex->is_depth && rtex->resource.b.b.nr_samples <= 1) {
1386                         ctx->resource_copy_region(ctx, texture, transfer->level,
1387                                                   transfer->box.x, transfer->box.y, transfer->box.z,
1388                                                   &rtransfer->staging->b.b, transfer->level,
1389                                                   &transfer->box);
1390                 } else {
1391                         r600_copy_from_staging_texture(ctx, rtransfer);
1392                 }
1393         }
1394
1395         if (rtransfer->staging)
1396                 pipe_resource_reference((struct pipe_resource**)&rtransfer->staging, NULL);
1397
1398         FREE(transfer);
1399 }
1400
1401 static const struct u_resource_vtbl r600_texture_vtbl =
1402 {
1403         NULL,                           /* get_handle */
1404         r600_texture_destroy,           /* resource_destroy */
1405         r600_texture_transfer_map,      /* transfer_map */
1406         u_default_transfer_flush_region, /* transfer_flush_region */
1407         r600_texture_transfer_unmap,    /* transfer_unmap */
1408         NULL                            /* transfer_inline_write */
1409 };
1410
1411 struct pipe_surface *r600_create_surface_custom(struct pipe_context *pipe,
1412                                                 struct pipe_resource *texture,
1413                                                 const struct pipe_surface *templ,
1414                                                 unsigned width, unsigned height)
1415 {
1416         struct r600_surface *surface = CALLOC_STRUCT(r600_surface);
1417
1418         if (!surface)
1419                 return NULL;
1420
1421         assert(templ->u.tex.first_layer <= util_max_layer(texture, templ->u.tex.level));
1422         assert(templ->u.tex.last_layer <= util_max_layer(texture, templ->u.tex.level));
1423
1424         pipe_reference_init(&surface->base.reference, 1);
1425         pipe_resource_reference(&surface->base.texture, texture);
1426         surface->base.context = pipe;
1427         surface->base.format = templ->format;
1428         surface->base.width = width;
1429         surface->base.height = height;
1430         surface->base.u = templ->u;
1431         return &surface->base;
1432 }
1433
1434 static struct pipe_surface *r600_create_surface(struct pipe_context *pipe,
1435                                                 struct pipe_resource *tex,
1436                                                 const struct pipe_surface *templ)
1437 {
1438         unsigned level = templ->u.tex.level;
1439         unsigned width = u_minify(tex->width0, level);
1440         unsigned height = u_minify(tex->height0, level);
1441
1442         if (tex->target != PIPE_BUFFER && templ->format != tex->format) {
1443                 const struct util_format_description *tex_desc
1444                         = util_format_description(tex->format);
1445                 const struct util_format_description *templ_desc
1446                         = util_format_description(templ->format);
1447
1448                 assert(tex_desc->block.bits == templ_desc->block.bits);
1449
1450                 /* Adjust size of surface if and only if the block width or
1451                  * height is changed. */
1452                 if (tex_desc->block.width != templ_desc->block.width ||
1453                     tex_desc->block.height != templ_desc->block.height) {
1454                         unsigned nblks_x = util_format_get_nblocksx(tex->format, width);
1455                         unsigned nblks_y = util_format_get_nblocksy(tex->format, height);
1456
1457                         width = nblks_x * templ_desc->block.width;
1458                         height = nblks_y * templ_desc->block.height;
1459                 }
1460         }
1461
1462         return r600_create_surface_custom(pipe, tex, templ, width, height);
1463 }
1464
1465 static void r600_surface_destroy(struct pipe_context *pipe,
1466                                  struct pipe_surface *surface)
1467 {
1468         struct r600_surface *surf = (struct r600_surface*)surface;
1469         pipe_resource_reference((struct pipe_resource**)&surf->cb_buffer_fmask, NULL);
1470         pipe_resource_reference((struct pipe_resource**)&surf->cb_buffer_cmask, NULL);
1471         pipe_resource_reference(&surface->texture, NULL);
1472         FREE(surface);
1473 }
1474
1475 unsigned r600_translate_colorswap(enum pipe_format format, bool do_endian_swap)
1476 {
1477         const struct util_format_description *desc = util_format_description(format);
1478
1479 #define HAS_SWIZZLE(chan,swz) (desc->swizzle[chan] == PIPE_SWIZZLE_##swz)
1480
1481         if (format == PIPE_FORMAT_R11G11B10_FLOAT) /* isn't plain */
1482                 return V_0280A0_SWAP_STD;
1483
1484         if (desc->layout != UTIL_FORMAT_LAYOUT_PLAIN)
1485                 return ~0U;
1486
1487         switch (desc->nr_channels) {
1488         case 1:
1489                 if (HAS_SWIZZLE(0,X))
1490                         return V_0280A0_SWAP_STD; /* X___ */
1491                 else if (HAS_SWIZZLE(3,X))
1492                         return V_0280A0_SWAP_ALT_REV; /* ___X */
1493                 break;
1494         case 2:
1495                 if ((HAS_SWIZZLE(0,X) && HAS_SWIZZLE(1,Y)) ||
1496                     (HAS_SWIZZLE(0,X) && HAS_SWIZZLE(1,NONE)) ||
1497                     (HAS_SWIZZLE(0,NONE) && HAS_SWIZZLE(1,Y)))
1498                         return V_0280A0_SWAP_STD; /* XY__ */
1499                 else if ((HAS_SWIZZLE(0,Y) && HAS_SWIZZLE(1,X)) ||
1500                          (HAS_SWIZZLE(0,Y) && HAS_SWIZZLE(1,NONE)) ||
1501                          (HAS_SWIZZLE(0,NONE) && HAS_SWIZZLE(1,X)))
1502                         /* YX__ */
1503                         return (do_endian_swap ? V_0280A0_SWAP_STD : V_0280A0_SWAP_STD_REV);
1504                 else if (HAS_SWIZZLE(0,X) && HAS_SWIZZLE(3,Y))
1505                         return V_0280A0_SWAP_ALT; /* X__Y */
1506                 else if (HAS_SWIZZLE(0,Y) && HAS_SWIZZLE(3,X))
1507                         return V_0280A0_SWAP_ALT_REV; /* Y__X */
1508                 break;
1509         case 3:
1510                 if (HAS_SWIZZLE(0,X))
1511                         return (do_endian_swap ? V_0280A0_SWAP_STD_REV : V_0280A0_SWAP_STD);
1512                 else if (HAS_SWIZZLE(0,Z))
1513                         return V_0280A0_SWAP_STD_REV; /* ZYX */
1514                 break;
1515         case 4:
1516                 /* check the middle channels, the 1st and 4th channel can be NONE */
1517                 if (HAS_SWIZZLE(1,Y) && HAS_SWIZZLE(2,Z)) {
1518                         return V_0280A0_SWAP_STD; /* XYZW */
1519                 } else if (HAS_SWIZZLE(1,Z) && HAS_SWIZZLE(2,Y)) {
1520                         return V_0280A0_SWAP_STD_REV; /* WZYX */
1521                 } else if (HAS_SWIZZLE(1,Y) && HAS_SWIZZLE(2,X)) {
1522                         return V_0280A0_SWAP_ALT; /* ZYXW */
1523                 } else if (HAS_SWIZZLE(1,Z) && HAS_SWIZZLE(2,W)) {
1524                         /* YZWX */
1525                         if (desc->is_array)
1526                                 return V_0280A0_SWAP_ALT_REV;
1527                         else
1528                                 return (do_endian_swap ? V_0280A0_SWAP_ALT : V_0280A0_SWAP_ALT_REV);
1529                 }
1530                 break;
1531         }
1532         return ~0U;
1533 }
1534
1535 static void evergreen_set_clear_color(struct r600_texture *rtex,
1536                                       enum pipe_format surface_format,
1537                                       const union pipe_color_union *color)
1538 {
1539         union util_color uc;
1540
1541         memset(&uc, 0, sizeof(uc));
1542
1543         if (util_format_is_pure_uint(surface_format)) {
1544                 util_format_write_4ui(surface_format, color->ui, 0, &uc, 0, 0, 0, 1, 1);
1545         } else if (util_format_is_pure_sint(surface_format)) {
1546                 util_format_write_4i(surface_format, color->i, 0, &uc, 0, 0, 0, 1, 1);
1547         } else {
1548                 util_pack_color(color->f, surface_format, &uc);
1549         }
1550
1551         memcpy(rtex->color_clear_value, &uc, 2 * sizeof(uint32_t));
1552 }
1553
1554 static void vi_get_fast_clear_parameters(enum pipe_format surface_format,
1555                                          const union pipe_color_union *color,
1556                                          uint32_t* reset_value,
1557                                          bool* clear_words_needed)
1558 {
1559         bool values[4] = {};
1560         int i;
1561         bool main_value = false;
1562         bool extra_value = false;
1563         int extra_channel;
1564         const struct util_format_description *desc = util_format_description(surface_format);
1565
1566         *clear_words_needed = true;
1567         *reset_value = 0x20202020U;
1568
1569         /* If we want to clear without needing a fast clear eliminate step, we
1570          * can set each channel to 0 or 1 (or 0/max for integer formats). We
1571          * have two sets of flags, one for the last or first channel(extra) and
1572          * one for the other channels(main).
1573          */
1574
1575         if (surface_format == PIPE_FORMAT_R11G11B10_FLOAT ||
1576             surface_format == PIPE_FORMAT_B5G6R5_UNORM ||
1577             surface_format == PIPE_FORMAT_B5G6R5_SRGB) {
1578                 extra_channel = -1;
1579         } else if (desc->layout == UTIL_FORMAT_LAYOUT_PLAIN) {
1580                 if(r600_translate_colorswap(surface_format, FALSE) <= 1)
1581                         extra_channel = desc->nr_channels - 1;
1582                 else
1583                         extra_channel = 0;
1584         } else
1585                 return;
1586
1587         for (i = 0; i < 4; ++i) {
1588                 int index = desc->swizzle[i] - PIPE_SWIZZLE_X;
1589
1590                 if (desc->swizzle[i] < PIPE_SWIZZLE_X ||
1591                     desc->swizzle[i] > PIPE_SWIZZLE_W)
1592                         continue;
1593
1594                 if (util_format_is_pure_sint(surface_format)) {
1595                         values[i] = color->i[i] != 0;
1596                         if (color->i[i] != 0 && color->i[i] != INT32_MAX)
1597                                 return;
1598                 } else if (util_format_is_pure_uint(surface_format)) {
1599                         values[i] = color->ui[i] != 0U;
1600                         if (color->ui[i] != 0U && color->ui[i] != UINT32_MAX)
1601                                 return;
1602                 } else {
1603                         values[i] = color->f[i] != 0.0F;
1604                         if (color->f[i] != 0.0F && color->f[i] != 1.0F)
1605                                 return;
1606                 }
1607
1608                 if (index == extra_channel)
1609                         extra_value = values[i];
1610                 else
1611                         main_value = values[i];
1612         }
1613
1614         for (int i = 0; i < 4; ++i)
1615                 if (values[i] != main_value &&
1616                     desc->swizzle[i] - PIPE_SWIZZLE_X != extra_channel &&
1617                     desc->swizzle[i] >= PIPE_SWIZZLE_X &&
1618                     desc->swizzle[i] <= PIPE_SWIZZLE_W)
1619                         return;
1620
1621         *clear_words_needed = false;
1622         if (main_value)
1623                 *reset_value |= 0x80808080U;
1624
1625         if (extra_value)
1626                 *reset_value |= 0x40404040U;
1627 }
1628
1629 void evergreen_do_fast_color_clear(struct r600_common_context *rctx,
1630                                    struct pipe_framebuffer_state *fb,
1631                                    struct r600_atom *fb_state,
1632                                    unsigned *buffers, unsigned *dirty_cbufs,
1633                                    const union pipe_color_union *color)
1634 {
1635         int i;
1636
1637         /* This function is broken in BE, so just disable this path for now */
1638 #ifdef PIPE_ARCH_BIG_ENDIAN
1639         return;
1640 #endif
1641
1642         if (rctx->render_cond)
1643                 return;
1644
1645         for (i = 0; i < fb->nr_cbufs; i++) {
1646                 struct r600_texture *tex;
1647                 unsigned clear_bit = PIPE_CLEAR_COLOR0 << i;
1648
1649                 if (!fb->cbufs[i])
1650                         continue;
1651
1652                 /* if this colorbuffer is not being cleared */
1653                 if (!(*buffers & clear_bit))
1654                         continue;
1655
1656                 tex = (struct r600_texture *)fb->cbufs[i]->texture;
1657
1658                 /* 128-bit formats are unusupported */
1659                 if (util_format_get_blocksizebits(fb->cbufs[i]->format) > 64) {
1660                         continue;
1661                 }
1662
1663                 /* the clear is allowed if all layers are bound */
1664                 if (fb->cbufs[i]->u.tex.first_layer != 0 ||
1665                     fb->cbufs[i]->u.tex.last_layer != util_max_layer(&tex->resource.b.b, 0)) {
1666                         continue;
1667                 }
1668
1669                 /* cannot clear mipmapped textures */
1670                 if (fb->cbufs[i]->texture->last_level != 0) {
1671                         continue;
1672                 }
1673
1674                 /* only supported on tiled surfaces */
1675                 if (tex->surface.level[0].mode < RADEON_SURF_MODE_1D) {
1676                         continue;
1677                 }
1678
1679                 /* shared textures can't use fast clear without an explicit flush,
1680                  * because there is no way to communicate the clear color among
1681                  * all clients
1682                  */
1683                 if (tex->resource.is_shared &&
1684                     !(tex->resource.external_usage & PIPE_HANDLE_USAGE_EXPLICIT_FLUSH))
1685                         continue;
1686
1687                 /* fast color clear with 1D tiling doesn't work on old kernels and CIK */
1688                 if (tex->surface.level[0].mode == RADEON_SURF_MODE_1D &&
1689                     rctx->chip_class >= CIK &&
1690                     rctx->screen->info.drm_major == 2 &&
1691                     rctx->screen->info.drm_minor < 38) {
1692                         continue;
1693                 }
1694
1695                 if (tex->dcc_offset) {
1696                         uint32_t reset_value;
1697                         bool clear_words_needed;
1698
1699                         if (rctx->screen->debug_flags & DBG_NO_DCC_CLEAR)
1700                                 continue;
1701
1702                         vi_get_fast_clear_parameters(fb->cbufs[i]->format, color, &reset_value, &clear_words_needed);
1703
1704                         rctx->clear_buffer(&rctx->b, &tex->resource.b.b,
1705                                            tex->dcc_offset, tex->surface.dcc_size,
1706                                            reset_value, R600_COHERENCY_CB_META);
1707
1708                         if (clear_words_needed)
1709                                 tex->dirty_level_mask |= 1 << fb->cbufs[i]->u.tex.level;
1710                 } else {
1711                         /* Stoney/RB+ doesn't work with CMASK fast clear. */
1712                         if (rctx->family == CHIP_STONEY)
1713                                 continue;
1714
1715                         /* ensure CMASK is enabled */
1716                         r600_texture_alloc_cmask_separate(rctx->screen, tex);
1717                         if (tex->cmask.size == 0) {
1718                                 continue;
1719                         }
1720
1721                         /* Do the fast clear. */
1722                         rctx->clear_buffer(&rctx->b, &tex->cmask_buffer->b.b,
1723                                            tex->cmask.offset, tex->cmask.size, 0,
1724                                            R600_COHERENCY_CB_META);
1725
1726                         tex->dirty_level_mask |= 1 << fb->cbufs[i]->u.tex.level;
1727                 }
1728
1729                 evergreen_set_clear_color(tex, fb->cbufs[i]->format, color);
1730
1731                 if (dirty_cbufs)
1732                         *dirty_cbufs |= 1 << i;
1733                 rctx->set_atom_dirty(rctx, fb_state, true);
1734                 *buffers &= ~clear_bit;
1735         }
1736 }
1737
1738 void r600_init_screen_texture_functions(struct r600_common_screen *rscreen)
1739 {
1740         rscreen->b.resource_from_handle = r600_texture_from_handle;
1741         rscreen->b.resource_get_handle = r600_texture_get_handle;
1742 }
1743
1744 void r600_init_context_texture_functions(struct r600_common_context *rctx)
1745 {
1746         rctx->b.create_surface = r600_create_surface;
1747         rctx->b.surface_destroy = r600_surface_destroy;
1748 }