src/gen9_hevc_encoder.c

   1 /*
   2  * Copyright © 2017 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the
   6  * "Software"), to deal in the Software without restriction, including
   7  * without limitation the rights to use, copy, modify, merge, publish,
   8  * distribute, sub license, and/or sell copies of the Software, and to
   9  * permit persons to whom the Software is furnished to do so, subject to
  10  * the following conditions:
  11  *
  12  * The above copyright notice and this permission notice (including the
  13  * next paragraph) shall be included in all copies or substantial portions
  14  * of the Software.
  15  *
  16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
  17  * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  18  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
  19  * IN NO EVENT SHALL PRECISION INSIGHT AND/OR ITS SUPPLIERS BE LIABLE FOR
  20  * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
  21  * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
  22  * SOFTWAR OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  23  *
  24  * Authors:
  25  *    Chen, Peng <chen.c.peng@intel.com>
  26  *
  27  */
  28
  29 #include <stdio.h>
  30 #include <stdlib.h>
  31 #include <stdbool.h>
  32 #include <string.h>
  33 #include <math.h>
  34 #include <assert.h>
  35 #include <va/va.h>
  36
  37 #include "intel_batchbuffer.h"
  38 #include "intel_driver.h"
  39
  40 #include "i965_defines.h"
  41 #include "i965_drv_video.h"
  42 #include "i965_encoder.h"
  43 #include "i965_encoder_common.h"
  44 #include "i965_encoder_utils.h"
  45 #include "i965_encoder_api.h"
  46 #include "gen9_hevc_enc_kernels.h"
  47 #include "gen9_hevc_enc_kernels_binary.h"
  48 #include "gen9_hevc_enc_utils.h"
  49 #include "gen9_hevc_encoder.h"
  50 #include "gen9_hevc_enc_const_def.h"
  51
  52 static void *hevc_enc_kernel_ptr = NULL;
  53 static int hevc_enc_kernel_size = 0;
  54
  55 static bool
  56 gen9_hevc_get_kernel_header_and_size(void *pvbinary,
  57                                      int binary_size,
  58                                      GEN9_ENC_OPERATION operation,
  59                                      int krnstate_idx,
  60                                      struct i965_kernel *ret_kernel)
  61 {
  62     typedef uint32_t BIN_PTR[4];
  63
  64     char *bin_start;
  65     gen9_hevc_enc_kernels_header_bxt *pkh_table = NULL;
  66     gen9_hevc_enc_kernel_header *pcurr_header = NULL, *pinvalid_entry, *pnext_header;
  67     int next_krnoffset;
  68
  69     if (!pvbinary || !ret_kernel)
  70         return false;
  71
  72     bin_start = (char *)pvbinary;
  73     pkh_table = (gen9_hevc_enc_kernels_header_bxt *)pvbinary;
  74     pinvalid_entry = (gen9_hevc_enc_kernel_header *)(pkh_table + 1);
  75     next_krnoffset = binary_size;
  76
  77     switch (operation) {
  78     case GEN9_ENC_SCALING2X:
  79         pcurr_header = &pkh_table->HEVC_ENC_I_2xDownSampling_Kernel;
  80         break;
  81     case GEN9_ENC_SCALING4X:
  82         pcurr_header = &pkh_table->HEVC_ENC_I_DS4HME;
  83         break;
  84     case GEN9_ENC_ME:
  85         if (krnstate_idx)
  86             pcurr_header = &pkh_table->HEVC_ENC_P_HME;
  87         else
  88             pcurr_header = &pkh_table->HEVC_ENC_B_HME;
  89         break;
  90     case GEN9_ENC_BRC:
  91         switch (krnstate_idx) {
  92         case GEN9_HEVC_ENC_BRC_COARSE_INTRA:
  93             pcurr_header = &pkh_table->HEVC_ENC_I_COARSE;
  94             break;
  95         case GEN9_HEVC_ENC_BRC_INIT:
  96             pcurr_header = &pkh_table->HEVC_ENC_BRC_Init;
  97             break;
  98         case GEN9_HEVC_ENC_BRC_RESET:
  99             pcurr_header = &pkh_table->HEVC_ENC_BRC_Reset;
 100             break;
 101         case GEN9_HEVC_ENC_BRC_FRAME_UPDATE:
 102             pcurr_header = &pkh_table->HEVC_ENC_BRC_Update;
 103             break;
 104         case GEN9_HEVC_ENC_BRC_LCU_UPDATE:
 105             pcurr_header = &pkh_table->HEVC_ENC_BRC_LCU_Update;
 106             break;
 107         default:
 108             break;
 109         }
 110         break;
 111     case GEN9_ENC_MBENC:
 112         switch (krnstate_idx) {
 113         case GEN9_HEVC_ENC_MBENC_2xSCALING:
 114         case GEN9_HEVC_ENC_MBENC_32x32MD:
 115         case GEN9_HEVC_ENC_MBENC_16x16SAD:
 116         case GEN9_HEVC_ENC_MBENC_16x16MD:
 117         case GEN9_HEVC_ENC_MBENC_8x8PU:
 118         case GEN9_HEVC_ENC_MBENC_8x8FMODE:
 119         case GEN9_HEVC_ENC_MBENC_32x32INTRACHECK:
 120         case GEN9_HEVC_ENC_MBENC_BENC:
 121             pcurr_header = &pkh_table->HEVC_ENC_I_2xDownSampling_Kernel;
 122             pcurr_header += krnstate_idx;
 123             break;
 124         case GEN9_HEVC_ENC_MBENC_BPAK:
 125             pcurr_header = &pkh_table->HEVC_ENC_PB_Pak;
 126             break;
 127         case GEN9_HEVC_ENC_MBENC_WIDI:
 128             pcurr_header = &pkh_table->HEVC_ENC_PB_Widi;
 129             break;
 130         case GEN9_HEVC_MBENC_PENC:
 131             pcurr_header = &pkh_table->HEVC_ENC_P_MB;
 132             break;
 133         case GEN9_HEVC_MBENC_P_WIDI:
 134             pcurr_header = &pkh_table->HEVC_ENC_P_Widi;
 135             break;
 136         case GEN9_HEVC_ENC_MBENC_DS_COMBINED:
 137             pcurr_header = &pkh_table->HEVC_ENC_DS_Combined;
 138             break;
 139         default:
 140             break;
 141         }
 142         break;
 143     default:
 144         break;
 145     }
 146
 147     if (!pcurr_header ||
 148         ((pcurr_header->kernel_start_pointer << 6) >= binary_size))
 149         return false;
 150
 151     ret_kernel->bin = (const BIN_PTR *)(bin_start + (pcurr_header->kernel_start_pointer << 6));
 152
 153     pnext_header = (pcurr_header + 1);
 154     if (pnext_header < pinvalid_entry)
 155         next_krnoffset = pnext_header->kernel_start_pointer << 6;
 156
 157     ret_kernel->size = next_krnoffset - (pcurr_header->kernel_start_pointer << 6);
 158
 159     return true;
 160 }
 161
 162 static void
 163 gen9_hevc_store_reg_mem(VADriverContextP ctx,
 164                         struct intel_batchbuffer *batch,
 165                         dri_bo *bo,
 166                         unsigned int offset,
 167                         unsigned int mmio_offset)
 168 {
 169     struct gpe_mi_store_register_mem_parameter mi_store_reg_mem_param;
 170
 171     memset((void *)&mi_store_reg_mem_param, 0,
 172            sizeof(mi_store_reg_mem_param));
 173     mi_store_reg_mem_param.bo = bo;
 174     mi_store_reg_mem_param.offset = offset;
 175     mi_store_reg_mem_param.mmio_offset = mmio_offset;
 176     gen8_gpe_mi_store_register_mem(ctx, batch, &mi_store_reg_mem_param);
 177 }
 178
 179 static void
 180 gen9_hevc_load_reg_mem(VADriverContextP ctx,
 181                        struct intel_batchbuffer *batch,
 182                        dri_bo *bo,
 183                        unsigned int offset,
 184                        unsigned int mmio_offset)
 185 {
 186     struct gpe_mi_load_register_mem_parameter mi_load_reg_mem_param;
 187
 188     memset((void *)&mi_load_reg_mem_param, 0,
 189            sizeof(mi_load_reg_mem_param));
 190     mi_load_reg_mem_param.bo = bo;
 191     mi_load_reg_mem_param.offset = offset;
 192     mi_load_reg_mem_param.mmio_offset = mmio_offset;
 193     gen8_gpe_mi_load_register_mem(ctx, batch, &mi_load_reg_mem_param);
 194 }
 195
 196 static void
 197 gen9_hevc_conditional_end(VADriverContextP ctx,
 198                           struct intel_batchbuffer *batch,
 199                           dri_bo *bo,
 200                           unsigned int offset,
 201                           unsigned int data)
 202 {
 203     struct gpe_mi_conditional_batch_buffer_end_parameter mi_cond_end;
 204
 205     memset(&mi_cond_end, 0, sizeof(mi_cond_end));
 206     mi_cond_end.offset = offset;
 207     mi_cond_end.bo = bo;
 208     mi_cond_end.compare_data = data;
 209     mi_cond_end.compare_mask_mode_disabled = 0;
 210     gen9_gpe_mi_conditional_batch_buffer_end(ctx, batch,
 211                                              &mi_cond_end);
 212 }
 213
 214 static VAStatus
 215 gen9_hevc_ensure_surface(VADriverContextP ctx,
 216                          struct gen9_hevc_encoder_state *priv_state,
 217                          struct object_surface *obj_surface,
 218                          int reallocate_flag)
 219 {
 220     VAStatus va_status = VA_STATUS_SUCCESS;
 221     int update = 0;
 222     unsigned int fourcc = VA_FOURCC_NV12;
 223
 224     if (!obj_surface) {
 225         va_status = VA_STATUS_ERROR_INVALID_PARAMETER;
 226
 227         goto EXIT;
 228     }
 229
 230     if ((priv_state->bit_depth_luma_minus8 > 0)
 231         || (priv_state->bit_depth_chroma_minus8 > 0)) {
 232         if (obj_surface->fourcc != VA_FOURCC_P010) {
 233             update = 1;
 234             fourcc = VA_FOURCC_P010;
 235         }
 236     } else if (obj_surface->fourcc != VA_FOURCC_NV12) {
 237         update = 1;
 238         fourcc = VA_FOURCC_NV12;
 239     }
 240
 241     /* (Re-)allocate the underlying surface buffer store, if necessary */
 242     if (!obj_surface->bo || update) {
 243         if (reallocate_flag) {
 244             struct i965_driver_data * const i965 = i965_driver_data(ctx);
 245
 246             i965_destroy_surface_storage(obj_surface);
 247
 248             va_status = i965_check_alloc_surface_bo(ctx,
 249                                                     obj_surface,
 250                                                     i965->codec_info->has_tiled_surface,
 251                                                     fourcc,
 252                                                     SUBSAMPLE_YUV420);
 253         } else
 254             va_status = VA_STATUS_ERROR_INVALID_PARAMETER;
 255     }
 256
 257 EXIT:
 258     return va_status;
 259 }
 260
 261 static void
 262 gen9_hevc_free_surface_private(void **data)
 263 {
 264     struct gen9_hevc_surface_priv *surface_priv = NULL;
 265     int i = 0;
 266
 267     surface_priv = (struct gen9_hevc_surface_priv *)(*data);
 268     if (!surface_priv)
 269         return;
 270
 271     for (i = 0; i < HEVC_SCALED_SURFS_NUM; i++) {
 272         if (surface_priv->scaled_surface_obj[i]) {
 273             i965_DestroySurfaces(surface_priv->ctx, &surface_priv->scaled_surface_id[i], 1);
 274             surface_priv->scaled_surface_id[i] = VA_INVALID_SURFACE;
 275             surface_priv->scaled_surface_obj[i] = NULL;
 276         }
 277     }
 278
 279     if (surface_priv->surface_obj_nv12) {
 280         i965_DestroySurfaces(surface_priv->ctx, &surface_priv->surface_id_nv12, 1);
 281         surface_priv->surface_id_nv12 = VA_INVALID_SURFACE;
 282         surface_priv->surface_obj_nv12 = NULL;
 283     }
 284
 285     if (surface_priv->motion_vector_temporal_bo)
 286         dri_bo_unreference(surface_priv->motion_vector_temporal_bo);
 287
 288     free(surface_priv);
 289 }
 290
 291 static VAStatus
 292 gen9_hevc_init_surface_private(VADriverContextP ctx,
 293                                struct generic_enc_codec_state *generic_state,
 294                                struct gen9_hevc_encoder_state *priv_state,
 295                                struct object_surface *obj_surface)
 296 {
 297     struct i965_driver_data *i965 = i965_driver_data(ctx);
 298     struct gen9_hevc_surface_priv *surface_priv = NULL;
 299     int size = 0;
 300
 301     if (!obj_surface || !obj_surface->bo)
 302         return VA_STATUS_ERROR_INVALID_SURFACE;
 303
 304     if (obj_surface->private_data &&
 305         obj_surface->free_private_data != gen9_hevc_free_surface_private) {
 306         obj_surface->free_private_data(&obj_surface->private_data);
 307         obj_surface->private_data = NULL;
 308     }
 309
 310     if (obj_surface->private_data) {
 311         surface_priv = (struct gen9_hevc_surface_priv *)obj_surface->private_data;
 312
 313         surface_priv->surface_nv12_valid = 0;
 314     } else {
 315         surface_priv = calloc(1, sizeof(*surface_priv));
 316         if (!surface_priv)
 317             return VA_STATUS_ERROR_ALLOCATION_FAILED;
 318
 319         surface_priv->ctx = ctx;
 320         surface_priv->surface_reff = obj_surface;
 321
 322         obj_surface->private_data = (void *)surface_priv;
 323         obj_surface->free_private_data = gen9_hevc_free_surface_private;
 324
 325         // alloc motion vector temporal buffer
 326         size = MAX(((priv_state->picture_width + 63) >> 6) *
 327                    ((priv_state->picture_height + 15) >> 4),
 328                    ((priv_state->picture_width + 31) >> 5) *
 329                    ((priv_state->picture_height + 31) >> 5));
 330         size = ALIGN(size, 2) * 64;
 331         surface_priv->motion_vector_temporal_bo =
 332             dri_bo_alloc(i965->intel.bufmgr,
 333                          "motion vector temporal buffer",
 334                          size,
 335                          0x1000);
 336         if (!surface_priv->motion_vector_temporal_bo)
 337             return VA_STATUS_ERROR_ALLOCATION_FAILED;
 338
 339         //alloc HME surfaces
 340         i965_CreateSurfaces(ctx,
 341                             priv_state->frame_width_4x,
 342                             priv_state->frame_height_4x,
 343                             VA_RT_FORMAT_YUV420,
 344                             1,
 345                             &surface_priv->scaled_surface_id[HEVC_SCALED_SURF_4X_ID]);
 346
 347         surface_priv->scaled_surface_obj[HEVC_SCALED_SURF_4X_ID] =
 348             SURFACE(surface_priv->scaled_surface_id[HEVC_SCALED_SURF_4X_ID]);
 349
 350         if (!surface_priv->scaled_surface_obj[HEVC_SCALED_SURF_4X_ID])
 351             return VA_STATUS_ERROR_ALLOCATION_FAILED;
 352
 353         i965_check_alloc_surface_bo(ctx, surface_priv->scaled_surface_obj[HEVC_SCALED_SURF_4X_ID], 1,
 354                                     VA_FOURCC('N', 'V', '1', '2'), SUBSAMPLE_YUV420);
 355
 356         if (generic_state->b16xme_supported) {
 357             i965_CreateSurfaces(ctx,
 358                                 priv_state->frame_width_16x,
 359                                 priv_state->frame_height_16x,
 360                                 VA_RT_FORMAT_YUV420,
 361                                 1,
 362                                 &surface_priv->scaled_surface_id[HEVC_SCALED_SURF_16X_ID]);
 363             surface_priv->scaled_surface_obj[HEVC_SCALED_SURF_16X_ID] =
 364                 SURFACE(surface_priv->scaled_surface_id[HEVC_SCALED_SURF_16X_ID]);
 365
 366             if (!surface_priv->scaled_surface_obj[HEVC_SCALED_SURF_16X_ID])
 367                 return VA_STATUS_ERROR_ALLOCATION_FAILED;
 368
 369             i965_check_alloc_surface_bo(ctx, surface_priv->scaled_surface_obj[HEVC_SCALED_SURF_16X_ID], 1,
 370                                         VA_FOURCC('N', 'V', '1', '2'), SUBSAMPLE_YUV420);
 371         }
 372
 373         if (generic_state->b32xme_supported) {
 374             i965_CreateSurfaces(ctx,
 375                                 priv_state->frame_width_32x,
 376                                 priv_state->frame_height_32x,
 377                                 VA_RT_FORMAT_YUV420,
 378                                 1,
 379                                 &surface_priv->scaled_surface_id[HEVC_SCALED_SURF_32X_ID]);
 380             surface_priv->scaled_surface_obj[HEVC_SCALED_SURF_32X_ID] =
 381                 SURFACE(surface_priv->scaled_surface_id[HEVC_SCALED_SURF_32X_ID]);
 382
 383             if (!surface_priv->scaled_surface_obj[HEVC_SCALED_SURF_32X_ID])
 384                 return VA_STATUS_ERROR_ALLOCATION_FAILED;
 385
 386             i965_check_alloc_surface_bo(ctx, surface_priv->scaled_surface_obj[HEVC_SCALED_SURF_32X_ID], 1,
 387                                         VA_FOURCC('N', 'V', '1', '2'), SUBSAMPLE_YUV420);
 388         }
 389
 390         if (obj_surface->fourcc == VA_FOURCC_P010) {
 391             i965_CreateSurfaces(ctx,
 392                                 priv_state->frame_width_in_max_lcu,
 393                                 priv_state->frame_height_in_max_lcu,
 394                                 VA_RT_FORMAT_YUV420,
 395                                 1,
 396                                 &surface_priv->surface_id_nv12);
 397             surface_priv->surface_obj_nv12 = SURFACE(surface_priv->surface_id_nv12);
 398
 399             if (!surface_priv->surface_obj_nv12)
 400                 return VA_STATUS_ERROR_ALLOCATION_FAILED;
 401
 402             i965_check_alloc_surface_bo(ctx, surface_priv->surface_obj_nv12, 1,
 403                                         VA_FOURCC('N', 'V', '1', '2'), SUBSAMPLE_YUV420);
 404         }
 405     }
 406
 407     return VA_STATUS_SUCCESS;
 408 }
 409
 410 static void
 411 gen9_hevc_enc_free_resources(struct encoder_vme_mfc_context *vme_context)
 412 {
 413     struct gen9_hevc_encoder_context *priv_ctx = NULL;
 414
 415     priv_ctx = (struct gen9_hevc_encoder_context *)vme_context->private_enc_ctx;
 416
 417     if (!priv_ctx)
 418         return;
 419
 420     i965_free_gpe_resource(&priv_ctx->res_brc_pic_states_write_buffer);
 421     i965_free_gpe_resource(&priv_ctx->res_brc_history_buffer);
 422     i965_free_gpe_resource(&priv_ctx->res_brc_intra_dist_buffer);
 423     i965_free_gpe_resource(&priv_ctx->res_brc_pak_statistic_buffer);
 424     i965_free_gpe_resource(&priv_ctx->res_brc_input_buffer_for_enc_kernels);
 425     i965_free_gpe_resource(&priv_ctx->res_brc_constant_data_buffer);
 426
 427     i965_free_gpe_resource(&priv_ctx->res_mb_code_surface);
 428
 429     // free VME buffers
 430     i965_free_gpe_resource(&priv_ctx->res_flatness_check_surface);
 431     i965_free_gpe_resource(&priv_ctx->res_brc_me_dist_buffer);
 432     i965_free_gpe_resource(&priv_ctx->s4x_memv_distortion_buffer);
 433     i965_free_gpe_resource(&priv_ctx->s4x_memv_data_buffer);
 434     i965_free_gpe_resource(&priv_ctx->s16x_memv_data_buffer);
 435     i965_free_gpe_resource(&priv_ctx->s32x_memv_data_buffer);
 436     i965_free_gpe_resource(&priv_ctx->res_32x32_pu_output_buffer);
 437     i965_free_gpe_resource(&priv_ctx->res_simplest_intra_buffer);
 438     i965_free_gpe_resource(&priv_ctx->res_kernel_debug);
 439     i965_free_gpe_resource(&priv_ctx->res_sad_16x16_pu_buffer);
 440     i965_free_gpe_resource(&priv_ctx->res_vme_8x8_mode_buffer);
 441     i965_free_gpe_resource(&priv_ctx->res_intra_mode_buffer);
 442     i965_free_gpe_resource(&priv_ctx->res_intra_distortion_buffer);
 443     i965_free_gpe_resource(&priv_ctx->res_vme_uni_sic_buffer);
 444     i965_free_gpe_resource(&priv_ctx->res_con_corrent_thread_buffer);
 445     i965_free_gpe_resource(&priv_ctx->res_mv_index_buffer);
 446     i965_free_gpe_resource(&priv_ctx->res_mvp_index_buffer);
 447     i965_free_gpe_resource(&priv_ctx->res_roi_buffer);
 448     i965_free_gpe_resource(&priv_ctx->res_mb_statistics_buffer);
 449
 450     if (priv_ctx->scaled_2x_surface_obj) {
 451         i965_DestroySurfaces(priv_ctx->ctx, &priv_ctx->scaled_2x_surface_id, 1);
 452         priv_ctx->scaled_2x_surface_obj = NULL;
 453     }
 454
 455     // free PAK buffers
 456     i965_free_gpe_resource(&priv_ctx->deblocking_filter_line_buffer);
 457     i965_free_gpe_resource(&priv_ctx->deblocking_filter_tile_line_buffer);
 458     i965_free_gpe_resource(&priv_ctx->deblocking_filter_tile_column_buffer);
 459     i965_free_gpe_resource(&priv_ctx->metadata_line_buffer);
 460     i965_free_gpe_resource(&priv_ctx->metadata_tile_line_buffer);
 461     i965_free_gpe_resource(&priv_ctx->metadata_tile_column_buffer);
 462     i965_free_gpe_resource(&priv_ctx->sao_line_buffer);
 463     i965_free_gpe_resource(&priv_ctx->sao_tile_line_buffer);
 464     i965_free_gpe_resource(&priv_ctx->sao_tile_column_buffer);
 465
 466     priv_ctx->res_inited = 0;
 467 }
 468
 469 #define ALLOC_GPE_RESOURCE(RES, NAME, SIZE)                 \
 470     do{                                                     \
 471         i965_free_gpe_resource(&priv_ctx->RES);             \
 472         if (!i965_allocate_gpe_resource(i965->intel.bufmgr, \
 473                                  &priv_ctx->RES,            \
 474                                  SIZE,                      \
 475                                  NAME))                     \
 476             goto FAIL;                                      \
 477     } while(0);
 478
 479 #define ALLOC_GPE_2D_RESOURCE(RES, NAME, W, H, P)               \
 480     do{                                                         \
 481         i965_free_gpe_resource(&priv_ctx->RES);                 \
 482         if (!i965_gpe_allocate_2d_resource(i965->intel.bufmgr,  \
 483                                  &priv_ctx->RES,                \
 484                                  (ALIGN(W, 64)), H,             \
 485                                  (ALIGN(P, 64)),                \
 486                                  NAME))                         \
 487             goto FAIL;                                          \
 488     } while(0);
 489
 490 static VAStatus
 491 gen9_hevc_enc_alloc_resources(VADriverContextP ctx,
 492                               struct encode_state *encode_state,
 493                               struct intel_encoder_context *encoder_context)
 494 {
 495     struct i965_driver_data *i965 = i965_driver_data(ctx);
 496     struct encoder_vme_mfc_context *vme_context = NULL;
 497     struct gen9_hevc_encoder_context *priv_ctx = NULL;
 498     struct generic_enc_codec_state *generic_state = NULL;
 499     struct gen9_hevc_encoder_state *priv_state = NULL;
 500     int res_size = 0, size_shift = 0;
 501     int width = 0, height = 0;
 502
 503     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
 504     priv_ctx = (struct gen9_hevc_encoder_context *)vme_context->private_enc_ctx;
 505     generic_state = (struct generic_enc_codec_state *)vme_context->generic_enc_state;
 506     priv_state = (struct gen9_hevc_encoder_state *)vme_context->private_enc_state;
 507
 508     if (priv_ctx->res_inited)
 509         return VA_STATUS_SUCCESS;
 510
 511     res_size = priv_state->mb_code_size;
 512     ALLOC_GPE_RESOURCE(res_mb_code_surface,
 513                        "Mb code surface", res_size);
 514
 515     res_size = priv_state->pic_state_size * generic_state->num_pak_passes;
 516     ALLOC_GPE_RESOURCE(res_brc_pic_states_write_buffer,
 517                        "Brc pic status write buffer",
 518                        res_size);
 519
 520     res_size = priv_state->pic_state_size * generic_state->num_pak_passes;
 521     ALLOC_GPE_RESOURCE(res_brc_pic_states_read_buffer,
 522                        "Brc pic status read buffer",
 523                        res_size);
 524
 525     res_size = GEN9_HEVC_ENC_BRC_HISTORY_BUFFER_SIZE;
 526     ALLOC_GPE_RESOURCE(res_brc_history_buffer,
 527                        "Brc history buffer",
 528                        res_size);
 529
 530     res_size = GEN9_HEVC_ENC_BRC_PAK_STATISTCS_SIZE;
 531     ALLOC_GPE_RESOURCE(res_brc_pak_statistic_buffer,
 532                        "Brc pak statistic buffer",
 533                        res_size);
 534
 535     res_size = 1024;
 536     ALLOC_GPE_RESOURCE(res_brc_input_buffer_for_enc_kernels,
 537                        "Brc input buffer for enc kernels buffer",
 538                        res_size);
 539
 540     width = ALIGN(priv_state->downscaled_width_4x_in_mb * 8, 64);
 541     height = ALIGN(priv_state->downscaled_height_4x_in_mb * 4, 8) * 2;
 542     ALLOC_GPE_2D_RESOURCE(res_brc_intra_dist_buffer,
 543                           "Brc intra distortion buffer",
 544                           width, height, width);
 545
 546     width = ALIGN(GEN9_HEVC_ENC_BRC_CONSTANT_SURFACE_WIDTH, 64);
 547     height = GEN9_HEVC_ENC_BRC_CONSTANT_SURFACE_HEIGHT;
 548     ALLOC_GPE_2D_RESOURCE(res_brc_constant_data_buffer,
 549                           "Brc constant data buffer",
 550                           width, height, width);
 551
 552     width = ALIGN((priv_state->frame_width_4x * 4 + 31) >> 4, 64);
 553     height = ALIGN((priv_state->frame_height_4x * 4 + 31) >> 5, 4);
 554     ALLOC_GPE_2D_RESOURCE(res_brc_mb_qp_buffer,
 555                           "Brc mb qp buffer",
 556                           width, height, width);
 557
 558     //HME scaling buffer allocation
 559     width = ALIGN(priv_state->downscaled_width_4x_in_mb * 8, 64);
 560     height = ALIGN(priv_state->downscaled_height_4x_in_mb * 4, 8);
 561     ALLOC_GPE_2D_RESOURCE(res_brc_me_dist_buffer,
 562                           "Brc me distortion buffer",
 563                           width, height, width);
 564
 565     if (generic_state->hme_supported) {
 566         width = ALIGN(priv_state->downscaled_width_4x_in_mb * 8, 64);
 567         height = ALIGN(priv_state->downscaled_height_4x_in_mb * 4 * 10, 8) * 2;
 568         ALLOC_GPE_2D_RESOURCE(s4x_memv_distortion_buffer,
 569                               "4x MEMV distortion buffer",
 570                               width, height, width);
 571
 572         width = ALIGN(priv_state->downscaled_width_4x_in_mb * 32, 64);
 573         height = priv_state->downscaled_height_4x_in_mb * 4 * 10;
 574         ALLOC_GPE_2D_RESOURCE(s4x_memv_data_buffer,
 575                               "4x MEMV data buffer",
 576                               width, height, width);
 577
 578         if (generic_state->b16xme_supported) {
 579             width = ALIGN(priv_state->downscaled_width_16x_in_mb * 32, 64);
 580             height = priv_state->downscaled_height_16x_in_mb * 2 * 4 * 10;
 581             ALLOC_GPE_2D_RESOURCE(s16x_memv_data_buffer,
 582                                   "16x MEMV data buffer",
 583                                   width, height, width);
 584
 585             if (generic_state->b32xme_supported) {
 586                 width = ALIGN(priv_state->downscaled_width_32x_in_mb * 32, 64);
 587                 height = priv_state->downscaled_height_32x_in_mb * 2 * 4 * 10;
 588                 ALLOC_GPE_2D_RESOURCE(s32x_memv_data_buffer,
 589                                       "32x MEMV data buffer",
 590                                       width, height, width);
 591             }
 592         }
 593     }
 594
 595     if (priv_state->flatness_check_supported) {
 596         width = ALIGN(priv_state->width_in_mb * 4, 64);
 597         height = priv_state->downscaled_height_4x_in_mb * 4;
 598         ALLOC_GPE_2D_RESOURCE(res_flatness_check_surface,
 599                               "Flatness check buffer",
 600                               width, height, width);
 601     }
 602
 603     if (priv_ctx->scaled_2x_surface_obj)
 604         i965_DestroySurfaces(priv_ctx->ctx, &priv_ctx->scaled_2x_surface_id, 1);
 605
 606     width = priv_state->frame_width_in_max_lcu >> 1;
 607     height = priv_state->frame_height_in_max_lcu >> 1;
 608     if (priv_state->bit_depth_luma_minus8) {
 609         width = ALIGN(width, 32);
 610         height = ALIGN(height, 32);
 611     }
 612
 613     i965_CreateSurfaces(ctx,
 614                         width,
 615                         height,
 616                         VA_RT_FORMAT_YUV420,
 617                         1,
 618                         &priv_ctx->scaled_2x_surface_id);
 619     priv_ctx->scaled_2x_surface_obj =
 620         SURFACE(priv_ctx->scaled_2x_surface_id);
 621
 622     if (!priv_ctx->scaled_2x_surface_obj)
 623         goto FAIL;
 624
 625     i965_check_alloc_surface_bo(ctx, priv_ctx->scaled_2x_surface_obj, 1,
 626                                 VA_FOURCC('N', 'V', '1', '2'), SUBSAMPLE_YUV420);
 627
 628     res_size = (priv_state->frame_width_in_max_lcu >> 5) *
 629                (priv_state->frame_height_in_max_lcu >> 5) *
 630                32;
 631     ALLOC_GPE_RESOURCE(res_32x32_pu_output_buffer,
 632                        "32x32 pu output buffer",
 633                        res_size);
 634
 635     width = priv_state->frame_width_in_max_lcu >> 3;
 636     height = priv_state->frame_height_in_max_lcu >> 5;
 637     ALLOC_GPE_2D_RESOURCE(res_slice_map_buffer,
 638                           "Slice map buffer",
 639                           width, height, width);
 640
 641     res_size = 8192 * 1024;
 642     ALLOC_GPE_RESOURCE(res_kernel_debug,
 643                        "kernel debug",
 644                        res_size);
 645
 646     width = priv_state->frame_width_in_max_lcu >> 3;
 647     height = priv_state->frame_height_in_max_lcu >> 5;
 648     ALLOC_GPE_2D_RESOURCE(res_simplest_intra_buffer,
 649                           "Simplest intra buffer",
 650                           width, height, width);
 651
 652     res_size = (priv_state->frame_width_in_max_lcu >> 4) *
 653                (priv_state->frame_height_in_max_lcu >> 4) * 8 * 4;
 654     ALLOC_GPE_RESOURCE(res_sad_16x16_pu_buffer,
 655                        "Sad 16x16 pu",
 656                        res_size);
 657
 658     res_size = (priv_state->frame_width_in_max_lcu >> 4) *
 659                (priv_state->frame_height_in_max_lcu >> 4) * 32;
 660     ALLOC_GPE_RESOURCE(res_vme_8x8_mode_buffer,
 661                        "Vme 8x8 mode",
 662                        res_size);
 663
 664     res_size = (priv_state->frame_width_in_max_lcu >> 3) *
 665                (priv_state->frame_height_in_max_lcu >> 3) * 32;
 666     ALLOC_GPE_RESOURCE(res_intra_mode_buffer,
 667                        "Intra mode",
 668                        res_size);
 669
 670     res_size = (priv_state->frame_width_in_max_lcu >> 4) *
 671                (priv_state->frame_height_in_max_lcu >> 4) * 16;
 672     ALLOC_GPE_RESOURCE(res_intra_distortion_buffer,
 673                        "Intra distortion",
 674                        res_size);
 675
 676     width = priv_state->frame_width_in_max_lcu >> 1;
 677     height = priv_state->frame_height_in_max_lcu >> 4;
 678     ALLOC_GPE_2D_RESOURCE(res_min_distortion_buffer,
 679                           "Min distortion buffer",
 680                           width, height, width);
 681
 682     res_size = priv_state->frame_width_in_max_lcu *
 683                priv_state->frame_height_in_max_lcu;
 684     ALLOC_GPE_RESOURCE(res_vme_uni_sic_buffer,
 685                        "Vme uni sic buffer",
 686                        res_size);
 687
 688     width = sizeof(gen9_hevc_mbenc_control_region);
 689     height = GEN9_HEVC_ENC_CONCURRENT_SURFACE_HEIGHT;
 690     ALLOC_GPE_2D_RESOURCE(res_con_corrent_thread_buffer,
 691                           "Con corrent thread buffer",
 692                           width, height, width);
 693
 694     res_size = priv_state->frame_width_in_max_lcu *
 695                priv_state->frame_height_in_max_lcu / 4;
 696     ALLOC_GPE_RESOURCE(res_mv_index_buffer,
 697                        "Mv index buffer",
 698                        res_size);
 699
 700     res_size = priv_state->frame_width_in_max_lcu *
 701                priv_state->frame_height_in_max_lcu / 2;
 702     ALLOC_GPE_RESOURCE(res_mvp_index_buffer,
 703                        "Mvp index buffer",
 704                        res_size);
 705
 706     width = ALIGN(priv_state->width_in_mb * 4, 64);
 707     height = ALIGN(priv_state->height_in_mb, 8);
 708     ALLOC_GPE_2D_RESOURCE(res_roi_buffer,
 709                           "ROI buffer",
 710                           width, height, width);
 711
 712     res_size = priv_state->width_in_mb *
 713                priv_state->height_in_mb * 52;
 714     ALLOC_GPE_RESOURCE(res_mb_statistics_buffer,
 715                        "MB statistics buffer",
 716                        res_size);
 717
 718     // PAK pipe buffer allocation
 719     size_shift = (priv_state->bit_depth_luma_minus8 ||
 720                   priv_state->bit_depth_chroma_minus8) ? 2 : 3;
 721
 722     res_size = ALIGN(priv_state->picture_width, 32) >> size_shift;
 723     ALLOC_GPE_RESOURCE(deblocking_filter_line_buffer,
 724                        "Deblocking filter line buffer",
 725                        res_size << 6);
 726     ALLOC_GPE_RESOURCE(deblocking_filter_tile_line_buffer,
 727                        "Deblocking filter tile line buffer",
 728                        res_size << 6);
 729
 730     res_size = ALIGN(priv_state->picture_height +
 731                      priv_state->height_in_lcu * 6, 32) >> size_shift;
 732     ALLOC_GPE_RESOURCE(deblocking_filter_tile_column_buffer,
 733                        "Deblocking filter tile column buffer",
 734                        res_size << 6);
 735
 736     res_size = (((priv_state->picture_width + 15) >> 4) * 188 +
 737                 priv_state->width_in_lcu * 9 + 1023) >> 9;
 738     ALLOC_GPE_RESOURCE(metadata_line_buffer,
 739                        "metadata line buffer",
 740                        res_size << 6);
 741
 742     res_size = (((priv_state->picture_width + 15) >> 4) * 172 +
 743                 priv_state->width_in_lcu * 9 + 1023) >> 9;
 744     ALLOC_GPE_RESOURCE(metadata_tile_line_buffer,
 745                        "metadata tile line buffer",
 746                        res_size << 6);
 747
 748     res_size = (((priv_state->picture_height + 15) >> 4) * 176 +
 749                 priv_state->height_in_lcu * 89 + 1023) >> 9;
 750     ALLOC_GPE_RESOURCE(metadata_tile_column_buffer,
 751                        "metadata tile column buffer",
 752                        res_size << 6);
 753
 754     res_size = ALIGN(((priv_state->picture_width >> 1) +
 755                       priv_state->width_in_lcu * 3), 16) >> size_shift;
 756     ALLOC_GPE_RESOURCE(sao_line_buffer,
 757                        "sao line buffer",
 758                        res_size << 6);
 759
 760     res_size = ALIGN(((priv_state->picture_width >> 1) +
 761                       priv_state->width_in_lcu * 6), 16) >> size_shift;
 762     ALLOC_GPE_RESOURCE(sao_tile_line_buffer,
 763                        "sao tile line buffer",
 764                        res_size << 6);
 765
 766     res_size = ALIGN(((priv_state->picture_height >> 1) +
 767                       priv_state->height_in_lcu * 6), 16) >> size_shift;
 768     ALLOC_GPE_RESOURCE(sao_tile_column_buffer,
 769                        "sao tile column buffer",
 770                        res_size << 6);
 771
 772     priv_ctx->res_inited = 1;
 773     return VA_STATUS_SUCCESS;
 774
 775 FAIL:
 776     gen9_hevc_enc_free_resources(vme_context);
 777     return VA_STATUS_ERROR_ALLOCATION_FAILED;
 778 }
 779
 780 #define VME_IMPLEMENTATION_START
 781
 782 static void
 783 gen9_hevc_set_gpe_1d_surface(VADriverContextP ctx,
 784                              struct gen9_hevc_encoder_context *priv_ctx,
 785                              struct i965_gpe_context *gpe_context,
 786                              enum GEN9_HEVC_ENC_SURFACE_TYPE surface_type,
 787                              int bti_idx,
 788                              int is_raw_buffer,
 789                              int size,
 790                              unsigned int offset,
 791                              struct i965_gpe_resource *gpe_buffer,
 792                              dri_bo *bo)
 793 {
 794     if (!gpe_buffer && !bo) {
 795         gpe_buffer = priv_ctx->gpe_surfaces[surface_type].gpe_resource;
 796         bo = priv_ctx->gpe_surfaces[surface_type].bo;
 797     }
 798
 799     if (gpe_buffer)
 800         i965_add_buffer_gpe_surface(ctx, gpe_context,
 801                                     gpe_buffer, is_raw_buffer,
 802                                     size == 0 ? gpe_buffer->size - offset : size,
 803                                     offset, bti_idx);
 804     else if (bo)
 805         gen9_add_dri_buffer_gpe_surface(ctx, gpe_context,
 806                                         bo, is_raw_buffer,
 807                                         size == 0 ? bo->size - offset : size,
 808                                         offset, bti_idx);
 809 }
 810
 811 static void
 812 gen9_hevc_set_gpe_2d_surface(VADriverContextP ctx,
 813                              struct gen9_hevc_encoder_context *priv_ctx,
 814                              struct i965_gpe_context *gpe_context,
 815                              enum GEN9_HEVC_ENC_SURFACE_TYPE surface_type,
 816                              int bti_idx,
 817                              int has_uv_surface,
 818                              int is_media_block_rw,
 819                              unsigned int format,
 820                              struct i965_gpe_resource *gpe_buffer,
 821                              struct object_surface *surface_object)
 822 {
 823     if (!gpe_buffer && !surface_object) {
 824         gpe_buffer = priv_ctx->gpe_surfaces[surface_type].gpe_resource;
 825         surface_object = priv_ctx->gpe_surfaces[surface_type].surface_object;
 826     }
 827
 828     if (gpe_buffer) {
 829         i965_add_buffer_2d_gpe_surface(ctx,
 830                                        gpe_context,
 831                                        gpe_buffer,
 832                                        is_media_block_rw,
 833                                        format,
 834                                        bti_idx);
 835     } else if (surface_object) {
 836         i965_add_2d_gpe_surface(ctx, gpe_context,
 837                                 surface_object,
 838                                 0, is_media_block_rw, format,
 839                                 bti_idx);
 840
 841         if (has_uv_surface)
 842             i965_add_2d_gpe_surface(ctx, gpe_context,
 843                                     surface_object,
 844                                     1, is_media_block_rw, format,
 845                                     bti_idx + 1);
 846     }
 847 }
 848
 849 static void
 850 gen9_hevc_set_gpe_adv_surface(VADriverContextP ctx,
 851                               struct gen9_hevc_encoder_context *priv_ctx,
 852                               struct i965_gpe_context *gpe_context,
 853                               enum GEN9_HEVC_ENC_SURFACE_TYPE surface_type,
 854                               int bti_idx,
 855                               struct object_surface *surface_object)
 856 {
 857     if (!surface_object)
 858         surface_object = priv_ctx->gpe_surfaces[surface_type].surface_object;
 859
 860     if (surface_object)
 861         i965_add_adv_gpe_surface(ctx, gpe_context,
 862                                  surface_object, bti_idx);
 863 }
 864
 865 static void
 866 gen9_hevc_add_gpe_surface(struct gen9_hevc_encoder_context *priv_ctx,
 867                           enum GEN9_HEVC_ENC_SURFACE_TYPE surface_type,
 868                           struct i965_gpe_resource *gpe_buffer,
 869                           struct object_surface *surface_object)
 870 {
 871     if (gpe_buffer && gpe_buffer->bo)
 872         priv_ctx->gpe_surfaces[surface_type].gpe_resource = gpe_buffer;
 873     else if (surface_object)
 874         priv_ctx->gpe_surfaces[surface_type].surface_object = surface_object;
 875 }
 876
 877 static void
 878 gen9_hevc_init_gpe_surfaces_table(VADriverContextP ctx,
 879                                   struct encode_state *encode_state,
 880                                   struct intel_encoder_context *encoder_context)
 881 {
 882     struct encoder_vme_mfc_context *vme_context = NULL;
 883     struct gen9_hevc_encoder_context *priv_ctx = NULL;
 884     struct gen9_hevc_surface_priv *surface_priv = NULL;
 885
 886     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
 887     priv_ctx = (struct gen9_hevc_encoder_context *)vme_context->private_enc_ctx;
 888
 889     if (encode_state->reconstructed_object->fourcc == VA_FOURCC_P010) {
 890         surface_priv = (struct gen9_hevc_surface_priv *)encode_state->reconstructed_object->private_data;
 891
 892         gen9_hevc_add_gpe_surface(priv_ctx, HEVC_ENC_SURFACE_RAW_Y, NULL,
 893                                   surface_priv->surface_obj_nv12);
 894         gen9_hevc_add_gpe_surface(priv_ctx, HEVC_ENC_SURFACE_RAW_Y_UV, NULL,
 895                                   surface_priv->surface_obj_nv12);
 896         gen9_hevc_add_gpe_surface(priv_ctx, HEVC_ENC_SURFACE_RAW_VME, NULL,
 897                                   surface_priv->surface_obj_nv12);
 898     } else {
 899         gen9_hevc_add_gpe_surface(priv_ctx, HEVC_ENC_SURFACE_RAW_Y, NULL,
 900                                   encode_state->input_yuv_object);
 901         gen9_hevc_add_gpe_surface(priv_ctx, HEVC_ENC_SURFACE_RAW_Y_UV, NULL,
 902                                   encode_state->input_yuv_object);
 903         gen9_hevc_add_gpe_surface(priv_ctx, HEVC_ENC_SURFACE_RAW_VME, NULL,
 904                                   encode_state->input_yuv_object);
 905     }
 906
 907     if (priv_ctx->scaled_2x_surface_obj) {
 908         gen9_hevc_add_gpe_surface(priv_ctx, HEVC_ENC_SURFACE_Y_2X, NULL,
 909                                   priv_ctx->scaled_2x_surface_obj);
 910         gen9_hevc_add_gpe_surface(priv_ctx, HEVC_ENC_SURFACE_Y_2X_VME, NULL,
 911                                   priv_ctx->scaled_2x_surface_obj);
 912     }
 913
 914     gen9_hevc_add_gpe_surface(priv_ctx, HEVC_ENC_SURFACE_BRC_HISTORY,
 915                               &priv_ctx->res_brc_history_buffer,
 916                               NULL);
 917     gen9_hevc_add_gpe_surface(priv_ctx, HEVC_ENC_SURFACE_BRC_PAST_PAK_INFO,
 918                               &priv_ctx->res_brc_pak_statistic_buffer,
 919                               NULL);
 920     gen9_hevc_add_gpe_surface(priv_ctx, HEVC_ENC_SURFACE_HCP_PAK,
 921                               &priv_ctx->res_mb_code_surface,
 922                               NULL);
 923     gen9_hevc_add_gpe_surface(priv_ctx, HEVC_ENC_SURFACE_CU_RECORD,
 924                               &priv_ctx->res_mb_code_surface,
 925                               NULL);
 926     gen9_hevc_add_gpe_surface(priv_ctx, HEVC_ENC_SURFACE_32x32_PU_OUTPUT,
 927                               &priv_ctx->res_32x32_pu_output_buffer,
 928                               NULL);
 929     gen9_hevc_add_gpe_surface(priv_ctx, HEVC_ENC_SURFACE_SLICE_MAP,
 930                               &priv_ctx->res_slice_map_buffer,
 931                               NULL);
 932     gen9_hevc_add_gpe_surface(priv_ctx, HEVC_ENC_SURFACE_BRC_INPUT,
 933                               &priv_ctx->res_brc_input_buffer_for_enc_kernels,
 934                               NULL);
 935     gen9_hevc_add_gpe_surface(priv_ctx, HEVC_ENC_SURFACE_LCU_QP,
 936                               &priv_ctx->res_brc_mb_qp_buffer,
 937                               NULL);
 938     gen9_hevc_add_gpe_surface(priv_ctx, HEVC_ENC_SURFACE_ROI,
 939                               &priv_ctx->res_roi_buffer,
 940                               NULL);
 941     gen9_hevc_add_gpe_surface(priv_ctx, HEVC_ENC_SURFACE_RAW_MBSTAT,
 942                               &priv_ctx->res_mb_statistics_buffer,
 943                               NULL);
 944     gen9_hevc_add_gpe_surface(priv_ctx, HEVC_ENC_SURFACE_BRC_DATA,
 945                               &priv_ctx->res_brc_constant_data_buffer,
 946                               NULL);
 947     gen9_hevc_add_gpe_surface(priv_ctx, HEVC_ENC_SURFACE_KERNEL_DEBUG,
 948                               &priv_ctx->res_kernel_debug,
 949                               NULL);
 950     gen9_hevc_add_gpe_surface(priv_ctx, HEVC_ENC_SURFACE_SIMPLIFIED_INTRA,
 951                               &priv_ctx->res_simplest_intra_buffer,
 952                               NULL);
 953     gen9_hevc_add_gpe_surface(priv_ctx, HEVC_ENC_SURFACE_HME_MVP,
 954                               &priv_ctx->s4x_memv_data_buffer,
 955                               NULL);
 956     gen9_hevc_add_gpe_surface(priv_ctx, HEVC_ENC_SURFACE_HME_DIST,
 957                               &priv_ctx->s4x_memv_distortion_buffer,
 958                               NULL);
 959     gen9_hevc_add_gpe_surface(priv_ctx, HEVC_ENC_SURFACE_16x16PU_SAD,
 960                               &priv_ctx->res_sad_16x16_pu_buffer,
 961                               NULL);
 962     gen9_hevc_add_gpe_surface(priv_ctx, HEVC_ENC_SURFACE_VME_8x8,
 963                               &priv_ctx->res_vme_8x8_mode_buffer,
 964                               NULL);
 965     gen9_hevc_add_gpe_surface(priv_ctx, HEVC_ENC_SURFACE_INTRA_MODE,
 966                               &priv_ctx->res_intra_mode_buffer,
 967                               NULL);
 968     gen9_hevc_add_gpe_surface(priv_ctx, HEVC_ENC_SURFACE_INTRA_DIST,
 969                               &priv_ctx->res_intra_distortion_buffer,
 970                               NULL);
 971     gen9_hevc_add_gpe_surface(priv_ctx, HEVC_ENC_SURFACE_MIN_DIST,
 972                               &priv_ctx->res_min_distortion_buffer,
 973                               NULL);
 974     gen9_hevc_add_gpe_surface(priv_ctx, HEVC_ENC_SURFACE_VME_UNI_SIC_DATA,
 975                               &priv_ctx->res_vme_uni_sic_buffer,
 976                               NULL);
 977     gen9_hevc_add_gpe_surface(priv_ctx, HEVC_ENC_SURFACE_CONCURRENT_THREAD,
 978                               &priv_ctx->res_con_corrent_thread_buffer,
 979                               NULL);
 980     gen9_hevc_add_gpe_surface(priv_ctx, HEVC_ENC_SURFACE_MB_MV_INDEX,
 981                               &priv_ctx->res_mv_index_buffer,
 982                               NULL);
 983     gen9_hevc_add_gpe_surface(priv_ctx, HEVC_ENC_SURFACE_MVP_INDEX,
 984                               &priv_ctx->res_mvp_index_buffer,
 985                               NULL);
 986 }
 987
 988 static VAStatus
 989 gen9_hevc_enc_check_parameters(VADriverContextP ctx,
 990                                struct encode_state *encode_state,
 991                                struct intel_encoder_context *encoder_context)
 992 {
 993     VAEncSequenceParameterBufferHEVC *seq_param = NULL;
 994     VAEncPictureParameterBufferHEVC *pic_param = NULL;
 995     VAEncSliceParameterBufferHEVC *slice_param = NULL;
 996     int i = 0, j = 0;
 997
 998     seq_param = (VAEncSequenceParameterBufferHEVC *)encode_state->seq_param_ext->buffer;
 999     pic_param = (VAEncPictureParameterBufferHEVC *)encode_state->pic_param_ext->buffer;
1000
1001     for (i = 0; i < encode_state->num_slice_params_ext; i++) {
1002         slice_param = (VAEncSliceParameterBufferHEVC *)encode_state->slice_params_ext[i]->buffer;
1003
1004         if (slice_param->slice_fields.bits.slice_temporal_mvp_enabled_flag &&
1005             slice_param->slice_fields.bits.collocated_from_l0_flag &&
1006             (pic_param->collocated_ref_pic_index == 0xff ||
1007              pic_param->collocated_ref_pic_index > GEN9_MAX_REF_SURFACES))
1008             slice_param->slice_fields.bits.slice_temporal_mvp_enabled_flag = 0;
1009
1010         if (slice_param->num_ref_idx_l0_active_minus1 > GEN9_HEVC_NUM_MAX_REF_L0 - 1 ||
1011             slice_param->num_ref_idx_l1_active_minus1 > GEN9_HEVC_NUM_MAX_REF_L1 - 1)
1012             return VA_STATUS_ERROR_ATTR_NOT_SUPPORTED;
1013     }
1014
1015     i = 1 << (seq_param->log2_diff_max_min_luma_coding_block_size +
1016               seq_param->log2_min_luma_coding_block_size_minus3 + 3);
1017     if (i < GEN9_HEVC_ENC_MIN_LCU_SIZE ||
1018         i > GEN9_HEVC_ENC_MAX_LCU_SIZE)
1019         return VA_STATUS_ERROR_ATTR_NOT_SUPPORTED;
1020
1021     //The TU max size in SPS must be the same as the CU max size in SPS
1022     i = seq_param->log2_min_transform_block_size_minus2 +
1023         seq_param->log2_diff_max_min_transform_block_size + 2;
1024     j = seq_param->log2_min_luma_coding_block_size_minus3 +
1025         seq_param->log2_diff_max_min_luma_coding_block_size + 3;
1026
1027     if (i != j)
1028         return VA_STATUS_ERROR_ATTR_NOT_SUPPORTED;
1029
1030     slice_param = (VAEncSliceParameterBufferHEVC *)encode_state->slice_params_ext[0]->buffer;
1031     i = pic_param->pic_init_qp + slice_param->slice_qp_delta;
1032     j = -seq_param->seq_fields.bits.bit_depth_luma_minus8 * 6;
1033     if (i < j || i > 51)
1034         return VA_STATUS_ERROR_INVALID_PARAMETER;
1035
1036     return VA_STATUS_SUCCESS;
1037 }
1038
1039 static void
1040 gen9_hevc_enc_init_seq_parameters(struct gen9_hevc_encoder_context *priv_ctx,
1041                                   struct generic_enc_codec_state *generic_state,
1042                                   struct gen9_hevc_encoder_state *priv_state,
1043                                   VAEncSequenceParameterBufferHEVC *seq_param)
1044 {
1045     int new = 0, m = 0, n = 0;
1046
1047     if (priv_state->picture_width != seq_param->pic_width_in_luma_samples ||
1048         priv_state->picture_height != seq_param->pic_height_in_luma_samples ||
1049         priv_state->bit_depth_luma_minus8 != seq_param->seq_fields.bits.bit_depth_luma_minus8 ||
1050         priv_state->bit_depth_chroma_minus8 != seq_param->seq_fields.bits.bit_depth_chroma_minus8)
1051         new = 1;
1052
1053     if (!new)
1054         return;
1055
1056     priv_state->bit_depth_luma_minus8 = seq_param->seq_fields.bits.bit_depth_luma_minus8;
1057     priv_state->bit_depth_chroma_minus8 = seq_param->seq_fields.bits.bit_depth_chroma_minus8;
1058     priv_state->cu_size = 1 << (seq_param->log2_min_luma_coding_block_size_minus3 + 3);
1059     priv_state->lcu_size = 1 << (seq_param->log2_diff_max_min_luma_coding_block_size +
1060                                  seq_param->log2_min_luma_coding_block_size_minus3 + 3);
1061     priv_state->picture_width = (seq_param->pic_width_in_luma_samples / priv_state->cu_size) * priv_state->cu_size;
1062     priv_state->picture_height = (seq_param->pic_height_in_luma_samples / priv_state->cu_size) * priv_state->cu_size;
1063     priv_state->width_in_lcu  = ALIGN(priv_state->picture_width, priv_state->lcu_size) / priv_state->lcu_size;
1064     priv_state->height_in_lcu = ALIGN(priv_state->picture_height, priv_state->lcu_size) / priv_state->lcu_size;
1065     priv_state->width_in_cu  = ALIGN(priv_state->picture_width, priv_state->cu_size) / priv_state->cu_size;
1066     priv_state->height_in_cu = ALIGN(priv_state->picture_height, priv_state->cu_size) / priv_state->cu_size;
1067     priv_state->width_in_mb  = ALIGN(priv_state->picture_width, 16) / 16;
1068     priv_state->height_in_mb = ALIGN(priv_state->picture_height, 16) / 16;
1069
1070     m = (priv_state->picture_width + GEN9_HEVC_ENC_MIN_LCU_SIZE - 1) / GEN9_HEVC_ENC_MIN_LCU_SIZE;
1071     n = (priv_state->picture_height + GEN9_HEVC_ENC_MIN_LCU_SIZE - 1) / GEN9_HEVC_ENC_MIN_LCU_SIZE;
1072     priv_state->mb_data_offset = ALIGN(m * n * priv_state->pak_obj_size, 0x1000);
1073
1074     m = ALIGN(priv_state->picture_width, GEN9_HEVC_ENC_MAX_LCU_SIZE) / 8;
1075     n = ALIGN(priv_state->picture_height, GEN9_HEVC_ENC_MAX_LCU_SIZE) / 8;
1076     priv_state->mb_code_size = priv_state->mb_data_offset +
1077                                ALIGN(m * n * priv_state->cu_record_size, 0x1000);
1078
1079     priv_state->frame_width_in_max_lcu = ALIGN(priv_state->picture_width, 32);
1080     priv_state->frame_height_in_max_lcu = ALIGN(priv_state->picture_height, 32);
1081     priv_state->frame_width_4x = ALIGN(priv_state->picture_width / 4, 16);
1082     priv_state->frame_height_4x = ALIGN(priv_state->picture_height / 4, 16);
1083     priv_state->frame_width_16x = ALIGN(priv_state->picture_width / 16, 16);
1084     priv_state->frame_height_16x = ALIGN(priv_state->picture_height / 16, 16);
1085     priv_state->frame_width_32x = ALIGN(priv_state->picture_width / 32, 16);
1086     priv_state->frame_height_32x = ALIGN(priv_state->picture_height / 32, 16);
1087
1088     priv_state->downscaled_width_4x_in_mb = priv_state->frame_width_4x / 16;
1089     if (priv_state->bit_depth_luma_minus8) {
1090         priv_state->downscaled_width_4x_in_mb = ALIGN(priv_state->downscaled_width_4x_in_mb * 16, 32) /
1091                                                 16;
1092         priv_state->frame_width_4x = priv_state->downscaled_width_4x_in_mb * 16;
1093     }
1094
1095     priv_state->downscaled_height_4x_in_mb = priv_state->frame_height_4x / 16;
1096     priv_state->downscaled_width_16x_in_mb = priv_state->frame_width_16x / 16;
1097     priv_state->downscaled_height_16x_in_mb = priv_state->frame_height_16x / 16;
1098     priv_state->downscaled_width_32x_in_mb = priv_state->frame_width_32x / 16;
1099     priv_state->downscaled_height_32x_in_mb = priv_state->frame_height_32x / 16;
1100     priv_state->flatness_check_enable = priv_state->flatness_check_supported;
1101     priv_state->widi_first_intra_refresh = 1;
1102
1103     generic_state->hme_supported = GEN9_HEVC_HME_SUPPORTED;
1104     generic_state->b16xme_supported = GEN9_HEVC_16XME_SUPPORTED;
1105     generic_state->b32xme_supported = GEN9_HEVC_32XME_SUPPORTED;
1106     if (generic_state->hme_supported &&
1107         (priv_state->frame_width_4x < GEN9_HEVC_VME_MIN_ALLOWED_SIZE ||
1108          priv_state->frame_height_4x < GEN9_HEVC_VME_MIN_ALLOWED_SIZE)) {
1109         generic_state->b16xme_supported = 0;
1110         generic_state->b32xme_supported = 0;
1111
1112         if (priv_state->frame_width_4x < GEN9_HEVC_VME_MIN_ALLOWED_SIZE) {
1113             priv_state->frame_width_4x = GEN9_HEVC_VME_MIN_ALLOWED_SIZE;
1114             priv_state->downscaled_width_4x_in_mb = priv_state->frame_width_4x / 16;
1115         }
1116
1117         if (priv_state->frame_height_4x < GEN9_HEVC_VME_MIN_ALLOWED_SIZE) {
1118             priv_state->frame_height_4x = GEN9_HEVC_VME_MIN_ALLOWED_SIZE;
1119             priv_state->downscaled_height_4x_in_mb = priv_state->frame_height_4x / 16;
1120         }
1121     } else if (generic_state->b16xme_supported &&
1122                (priv_state->frame_width_16x < GEN9_HEVC_VME_MIN_ALLOWED_SIZE ||
1123                 priv_state->frame_height_16x < GEN9_HEVC_VME_MIN_ALLOWED_SIZE)) {
1124         generic_state->b32xme_supported = 0;
1125
1126         if (priv_state->frame_width_16x < GEN9_HEVC_VME_MIN_ALLOWED_SIZE) {
1127             priv_state->frame_width_16x = GEN9_HEVC_VME_MIN_ALLOWED_SIZE;
1128             priv_state->downscaled_width_16x_in_mb = priv_state->frame_width_16x / 16;
1129         }
1130
1131         if (priv_state->frame_height_16x < GEN9_HEVC_VME_MIN_ALLOWED_SIZE) {
1132             priv_state->frame_height_16x = GEN9_HEVC_VME_MIN_ALLOWED_SIZE;
1133             priv_state->downscaled_height_16x_in_mb = priv_state->frame_height_16x / 16;
1134         }
1135     } else if (generic_state->b32xme_supported &&
1136                (priv_state->frame_width_32x < GEN9_HEVC_VME_MIN_ALLOWED_SIZE ||
1137                 priv_state->frame_height_32x < GEN9_HEVC_VME_MIN_ALLOWED_SIZE)) {
1138         if (priv_state->frame_width_32x < GEN9_HEVC_VME_MIN_ALLOWED_SIZE) {
1139             priv_state->frame_width_32x = GEN9_HEVC_VME_MIN_ALLOWED_SIZE;
1140             priv_state->downscaled_width_32x_in_mb = priv_state->frame_width_32x / 16;
1141         }
1142
1143         if (priv_state->frame_height_32x < GEN9_HEVC_VME_MIN_ALLOWED_SIZE) {
1144             priv_state->frame_height_32x = GEN9_HEVC_VME_MIN_ALLOWED_SIZE;
1145             priv_state->downscaled_height_32x_in_mb = priv_state->frame_height_32x / 16;
1146         }
1147     }
1148
1149     priv_state->gop_ref_dist = seq_param->ip_period;
1150     priv_state->gop_size = seq_param->intra_period;
1151     priv_state->frame_number = 0;
1152
1153     priv_ctx->res_inited = 0;
1154 }
1155
1156 static void
1157 gen9_hevc_enc_init_pic_parameters(struct generic_enc_codec_state *generic_state,
1158                                   struct gen9_hevc_encoder_state *priv_state,
1159                                   VAEncSequenceParameterBufferHEVC *seq_param,
1160                                   VAEncPictureParameterBufferHEVC *pic_param,
1161                                   VAEncSliceParameterBufferHEVC *slice_param)
1162 {
1163     unsigned int log2_max_coding_block_size = 0, raw_ctu_bits = 0;
1164
1165     priv_state->picture_coding_type = slice_param->slice_type;
1166
1167     priv_state->ctu_max_bitsize_allowed = pic_param->ctu_max_bitsize_allowed;
1168     log2_max_coding_block_size  = seq_param->log2_min_luma_coding_block_size_minus3 + 3 +
1169                                   seq_param->log2_diff_max_min_luma_coding_block_size;
1170     raw_ctu_bits = (1 << (2 * log2_max_coding_block_size + 3)) +
1171                    (1 << (2 * log2_max_coding_block_size + 2));
1172     raw_ctu_bits = (5 * raw_ctu_bits / 3);
1173
1174     if (priv_state->ctu_max_bitsize_allowed == 0 ||
1175         priv_state->ctu_max_bitsize_allowed > raw_ctu_bits)
1176         priv_state->ctu_max_bitsize_allowed = raw_ctu_bits;
1177 }
1178
1179 static void
1180 gen9_hevc_enc_init_slice_parameters(VADriverContextP ctx,
1181                                     struct encode_state *encode_state,
1182                                     struct intel_encoder_context *encoder_context)
1183 {
1184     struct encoder_vme_mfc_context *vme_context = NULL;
1185     struct gen9_hevc_encoder_state *priv_state = NULL;
1186     VAEncPictureParameterBufferHEVC *pic_param = NULL;
1187     VAEncSliceParameterBufferHEVC *slice_param = NULL;
1188     int i = 0, j = 0;
1189
1190     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
1191     priv_state = (struct gen9_hevc_encoder_state *)vme_context->private_enc_state;
1192     pic_param = (VAEncPictureParameterBufferHEVC *)encode_state->pic_param_ext->buffer;
1193
1194     priv_state->low_delay = 1;
1195     priv_state->arbitrary_num_mb_in_slice = 0;
1196
1197     for (i = 0; i < encode_state->num_slice_params_ext; i++) {
1198         slice_param = (VAEncSliceParameterBufferHEVC *)encode_state->slice_params_ext[i]->buffer;
1199
1200         if (slice_param->slice_type == HEVC_SLICE_B && priv_state->low_delay) {
1201             for (j = 0; j <= slice_param->num_ref_idx_l0_active_minus1; j++) {
1202                 if (pic_param->decoded_curr_pic.pic_order_cnt <
1203                     slice_param->ref_pic_list0[j].pic_order_cnt)
1204                     priv_state->low_delay = 0;
1205             }
1206
1207             for (j = 0; j <= slice_param->num_ref_idx_l1_active_minus1; j++) {
1208                 if (pic_param->decoded_curr_pic.pic_order_cnt <
1209                     slice_param->ref_pic_list1[j].pic_order_cnt)
1210                     priv_state->low_delay = 0;
1211             }
1212         }
1213
1214         if (!priv_state->arbitrary_num_mb_in_slice &&
1215             (slice_param->num_ctu_in_slice % priv_state->width_in_lcu))
1216             priv_state->arbitrary_num_mb_in_slice = 1;
1217     }
1218 }
1219
1220 static VAStatus
1221 gen9_hevc_enc_init_parameters(VADriverContextP ctx,
1222                               struct encode_state *encode_state,
1223                               struct intel_encoder_context *encoder_context)
1224 {
1225     struct i965_driver_data *i965 = i965_driver_data(ctx);
1226     struct encoder_vme_mfc_context *vme_context = NULL;
1227     struct generic_enc_codec_state *generic_state = NULL;
1228     struct gen9_hevc_encoder_context *priv_ctx = NULL;
1229     struct gen9_hevc_encoder_state *priv_state = NULL;
1230     VAEncPictureParameterBufferHEVC *pic_param = NULL;
1231     VAEncSequenceParameterBufferHEVC *seq_param = NULL;
1232     VAEncSliceParameterBufferHEVC *slice_param = NULL;
1233     struct object_buffer *obj_buffer = NULL;
1234     VAStatus va_status = VA_STATUS_SUCCESS;
1235
1236     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
1237     generic_state = (struct generic_enc_codec_state *)vme_context->generic_enc_state;
1238     priv_ctx = (struct gen9_hevc_encoder_context *)vme_context->private_enc_ctx;
1239     priv_state = (struct gen9_hevc_encoder_state *)vme_context->private_enc_state;
1240     pic_param = (VAEncPictureParameterBufferHEVC *)encode_state->pic_param_ext->buffer;
1241     seq_param = (VAEncSequenceParameterBufferHEVC *)encode_state->seq_param_ext->buffer;
1242     slice_param = (VAEncSliceParameterBufferHEVC *)encode_state->slice_params_ext[0]->buffer;
1243
1244     if (!pic_param || !seq_param || !slice_param)
1245         return VA_STATUS_ERROR_INVALID_PARAMETER;
1246
1247     va_status = gen9_hevc_enc_check_parameters(ctx, encode_state, encoder_context);
1248     if (va_status |= VA_STATUS_SUCCESS)
1249         return va_status;
1250
1251     gen9_hevc_enc_init_seq_parameters(priv_ctx, generic_state, priv_state, seq_param);
1252     gen9_hevc_enc_init_pic_parameters(generic_state, priv_state, seq_param, pic_param, slice_param);
1253     gen9_hevc_enc_init_slice_parameters(ctx, encode_state, encoder_context);
1254
1255     if (priv_state->picture_coding_type == HEVC_SLICE_I) {
1256         generic_state->hme_enabled = 0;
1257         generic_state->b16xme_enabled = 0;
1258         generic_state->b32xme_enabled = 0;
1259     } else {
1260         generic_state->hme_enabled = generic_state->hme_supported;
1261         generic_state->b16xme_enabled = generic_state->b16xme_supported;
1262         generic_state->b32xme_enabled = generic_state->b32xme_supported;
1263     }
1264
1265     obj_buffer = BUFFER(pic_param->coded_buf);
1266     if (!obj_buffer ||
1267         !obj_buffer->buffer_store ||
1268         !obj_buffer->buffer_store->bo) {
1269         va_status = VA_STATUS_ERROR_INVALID_PARAMETER;
1270         goto EXIT;
1271     }
1272     encode_state->coded_buf_object = obj_buffer;
1273     priv_state->status_buffer.bo = obj_buffer->buffer_store->bo;
1274
1275     va_status = gen9_hevc_ensure_surface(ctx, priv_state,
1276                                          encode_state->input_yuv_object, 0);
1277     if (va_status != VA_STATUS_SUCCESS)
1278         goto EXIT;
1279
1280     va_status = gen9_hevc_ensure_surface(ctx, priv_state,
1281                                          encode_state->reconstructed_object, 1);
1282     if (va_status != VA_STATUS_SUCCESS)
1283         goto EXIT;
1284
1285 #if 0
1286     if (encode_state->input_yuv_object->orig_width > priv_state->picture_width)
1287         encode_state->input_yuv_object->orig_width = priv_state->picture_width;
1288
1289     if (encode_state->input_yuv_object->orig_height > priv_state->picture_height)
1290         encode_state->input_yuv_object->orig_height = priv_state->picture_height;
1291
1292
1293     if (encode_state->reconstructed_object->orig_width > priv_state->picture_width)
1294         encode_state->reconstructed_object->orig_width = priv_state->picture_width;
1295
1296     if (encode_state->reconstructed_object->orig_height > priv_state->picture_height)
1297         encode_state->reconstructed_object->orig_height = priv_state->picture_height;
1298 #endif
1299
1300     va_status = gen9_hevc_init_surface_private(ctx, generic_state, priv_state,
1301                                                encode_state->reconstructed_object);
1302     if (va_status != VA_STATUS_SUCCESS)
1303         goto EXIT;
1304
1305     {
1306         struct gen9_hevc_surface_priv *surface_priv = NULL;
1307
1308         surface_priv = (struct gen9_hevc_surface_priv *)encode_state->reconstructed_object->private_data;
1309
1310         surface_priv->qp_value = pic_param->pic_init_qp + slice_param->slice_qp_delta;
1311     }
1312
1313     va_status = gen9_hevc_enc_alloc_resources(ctx, encode_state,
1314                                               encoder_context);
1315     if (va_status != VA_STATUS_SUCCESS)
1316         goto EXIT;
1317
1318     gen9_hevc_init_gpe_surfaces_table(ctx, encode_state,
1319                                       encoder_context);
1320
1321 EXIT:
1322     return va_status;
1323 }
1324
1325 // VME&BRC implementation
1326
1327 static void
1328 gen9_hevc_vme_init_gpe_context(VADriverContextP ctx,
1329                                struct i965_gpe_context *gpe_context,
1330                                unsigned int curbe_size,
1331                                unsigned int inline_data_size)
1332 {
1333     struct i965_driver_data *i965 = i965_driver_data(ctx);
1334
1335     gpe_context->curbe.length = curbe_size;
1336
1337     gpe_context->sampler.entry_size = 0;
1338     gpe_context->sampler.max_entries = 0;
1339
1340     gpe_context->idrt.entry_size = ALIGN(sizeof(struct gen8_interface_descriptor_data), 64);
1341     gpe_context->idrt.max_entries = 1;
1342
1343     gpe_context->surface_state_binding_table.max_entries = MAX_HEVC_KERNELS_ENCODER_SURFACES;
1344     gpe_context->surface_state_binding_table.binding_table_offset = 0;
1345     gpe_context->surface_state_binding_table.surface_state_offset = ALIGN(MAX_HEVC_KERNELS_ENCODER_SURFACES * 4, 64);
1346     gpe_context->surface_state_binding_table.length = ALIGN(MAX_HEVC_KERNELS_ENCODER_SURFACES * 4, 64) + ALIGN(MAX_HEVC_KERNELS_ENCODER_SURFACES * SURFACE_STATE_PADDED_SIZE_GEN9, 64);
1347
1348     if (i965->intel.eu_total > 0)
1349         gpe_context->vfe_state.max_num_threads = 6 * i965->intel.eu_total;
1350     else
1351         gpe_context->vfe_state.max_num_threads = 112;
1352
1353     gpe_context->vfe_state.curbe_allocation_size = MAX(1, ALIGN(gpe_context->curbe.length, 32) >> 5);
1354     gpe_context->vfe_state.urb_entry_size = MAX(1, ALIGN(inline_data_size, 32) >> 5);
1355     gpe_context->vfe_state.num_urb_entries = (MAX_HEVC_KERNELS_URB_SIZE -
1356                                               gpe_context->vfe_state.curbe_allocation_size -
1357                                               ((gpe_context->idrt.entry_size >> 5) *
1358                                                gpe_context->idrt.max_entries)) /
1359                                              gpe_context->vfe_state.urb_entry_size;
1360     gpe_context->vfe_state.num_urb_entries = CLAMP(gpe_context->vfe_state.num_urb_entries, 1, 64);
1361     gpe_context->vfe_state.gpgpu_mode = 0;
1362 }
1363
1364 static void
1365 gen9_hevc_vme_init_scoreboard(struct i965_gpe_context *gpe_context,
1366                               unsigned int mask,
1367                               unsigned int enable,
1368                               unsigned int type)
1369 {
1370     gpe_context->vfe_desc5.scoreboard0.mask = mask;
1371     gpe_context->vfe_desc5.scoreboard0.type = type;
1372     gpe_context->vfe_desc5.scoreboard0.enable = enable;
1373
1374     gpe_context->vfe_desc6.scoreboard1.delta_x0 = 0xF;
1375     gpe_context->vfe_desc6.scoreboard1.delta_y0 = 0x0;
1376
1377     gpe_context->vfe_desc6.scoreboard1.delta_x1 = 0x0;
1378     gpe_context->vfe_desc6.scoreboard1.delta_y1 = 0xF;
1379
1380     gpe_context->vfe_desc6.scoreboard1.delta_x2 = 0x1;
1381     gpe_context->vfe_desc6.scoreboard1.delta_y2 = 0xF;
1382
1383     gpe_context->vfe_desc6.scoreboard1.delta_x3 = 0xF;
1384     gpe_context->vfe_desc6.scoreboard1.delta_y3 = 0xF;
1385
1386     gpe_context->vfe_desc7.scoreboard2.delta_x4 = 0xF;
1387     gpe_context->vfe_desc7.scoreboard2.delta_y4 = 0x1;
1388
1389     gpe_context->vfe_desc7.scoreboard2.delta_x5 = 0x0;
1390     gpe_context->vfe_desc7.scoreboard2.delta_y5 = 0xE;
1391
1392     gpe_context->vfe_desc7.scoreboard2.delta_x6 = 0x1;
1393     gpe_context->vfe_desc7.scoreboard2.delta_y6 = 0xE;
1394
1395     gpe_context->vfe_desc7.scoreboard2.delta_x7 = 0xF;
1396     gpe_context->vfe_desc7.scoreboard2.delta_y7 = 0xE;
1397 }
1398
1399 static void
1400 gen9_hevc_vme_set_scoreboard_26z(struct i965_gpe_context *gpe_context,
1401                                  unsigned int mask,
1402                                  unsigned int enable,
1403                                  unsigned int type)
1404 {
1405     gpe_context->vfe_desc5.scoreboard0.mask = mask;
1406     gpe_context->vfe_desc5.scoreboard0.type = type;
1407     gpe_context->vfe_desc5.scoreboard0.enable = enable;
1408
1409     gpe_context->vfe_desc6.scoreboard1.delta_x0 = -1;
1410     gpe_context->vfe_desc6.scoreboard1.delta_y0 = 3;
1411
1412     gpe_context->vfe_desc6.scoreboard1.delta_x1 = -1;
1413     gpe_context->vfe_desc6.scoreboard1.delta_y1 = 1;
1414
1415     gpe_context->vfe_desc6.scoreboard1.delta_x2 = -1;
1416     gpe_context->vfe_desc6.scoreboard1.delta_y2 = -1;
1417
1418     gpe_context->vfe_desc6.scoreboard1.delta_x3 = 0;
1419     gpe_context->vfe_desc6.scoreboard1.delta_y3 = -1;
1420
1421     gpe_context->vfe_desc7.scoreboard2.delta_x4 = 0;
1422     gpe_context->vfe_desc7.scoreboard2.delta_y4 = -2;
1423
1424     gpe_context->vfe_desc7.scoreboard2.delta_x5 = 0;
1425     gpe_context->vfe_desc7.scoreboard2.delta_y5 = -3;
1426
1427     gpe_context->vfe_desc7.scoreboard2.delta_x6 = 1;
1428     gpe_context->vfe_desc7.scoreboard2.delta_y6 = -2;
1429
1430     gpe_context->vfe_desc7.scoreboard2.delta_x7 = 1;
1431     gpe_context->vfe_desc7.scoreboard2.delta_y7 = -3;
1432 }
1433
1434 static void
1435 gen9_hevc_vme_set_scoreboard_26(struct i965_gpe_context *gpe_context,
1436                                 unsigned int mask,
1437                                 unsigned int enable,
1438                                 unsigned int type)
1439 {
1440     gpe_context->vfe_desc5.scoreboard0.mask = mask;
1441     gpe_context->vfe_desc5.scoreboard0.type = type;
1442     gpe_context->vfe_desc5.scoreboard0.enable = enable;
1443
1444     gpe_context->vfe_desc6.scoreboard1.delta_x0 = -1;
1445     gpe_context->vfe_desc6.scoreboard1.delta_y0 = 0;
1446
1447     gpe_context->vfe_desc6.scoreboard1.delta_x1 = -1;
1448     gpe_context->vfe_desc6.scoreboard1.delta_y1 = -1;
1449
1450     gpe_context->vfe_desc6.scoreboard1.delta_x2 = 0;
1451     gpe_context->vfe_desc6.scoreboard1.delta_y2 = -1;
1452
1453     gpe_context->vfe_desc6.scoreboard1.delta_x3 = 1;
1454     gpe_context->vfe_desc6.scoreboard1.delta_y3 = -1;
1455 }
1456
1457 static void
1458 gen9_hevc_init_object_walker(struct hevc_enc_kernel_walker_parameter *hevc_walker_param,
1459                              struct gpe_media_object_walker_parameter *gpe_param)
1460 {
1461     memset(gpe_param, 0, sizeof(*gpe_param));
1462
1463     gpe_param->use_scoreboard = hevc_walker_param->use_scoreboard;
1464     gpe_param->block_resolution.x = hevc_walker_param->resolution_x;
1465     gpe_param->block_resolution.y = hevc_walker_param->resolution_y;
1466     gpe_param->global_resolution.x = hevc_walker_param->resolution_x;
1467     gpe_param->global_resolution.y = hevc_walker_param->resolution_y;
1468     gpe_param->global_outer_loop_stride.x = hevc_walker_param->resolution_x;
1469     gpe_param->global_outer_loop_stride.y = 0;
1470     gpe_param->global_inner_loop_unit.x = 0;
1471     gpe_param->global_inner_loop_unit.y = hevc_walker_param->resolution_y;
1472     gpe_param->local_loop_exec_count = 0xFFFF;
1473     gpe_param->global_loop_exec_count = 0xFFFF;
1474
1475     if (hevc_walker_param->no_dependency) {
1476         gpe_param->scoreboard_mask = 0;
1477         gpe_param->use_scoreboard = 0;
1478         gpe_param->local_outer_loop_stride.x = 0;
1479         gpe_param->local_outer_loop_stride.y = 1;
1480         gpe_param->local_inner_loop_unit.x = 1;
1481         gpe_param->local_inner_loop_unit.y = 0;
1482         gpe_param->local_end.x = hevc_walker_param->resolution_x - 1;
1483         gpe_param->local_end.y = 0;
1484     }
1485 }
1486
1487 static void
1488 gen9_hevc_init_object_walker_26z(struct gen9_hevc_encoder_state *priv_state,
1489                                  struct i965_gpe_context *gpe_context,
1490                                  struct gpe_media_object_walker_parameter *gpe_param,
1491                                  int split_count,
1492                                  unsigned int max_slight_height,
1493                                  int use_hw_scoreboard,
1494                                  int scoreboard_type)
1495 {
1496     int width = priv_state->width_in_mb;
1497     int height = max_slight_height * 2;
1498     int ts_width = ((width + 3) & 0xfffc) >> 1;
1499     int lcu_width = (width + 1) >> 1;
1500     int lcu_height = (height + 1) >> 1;
1501     int tmp1 = ((lcu_width + 1) >> 1) + ((lcu_width + ((lcu_height - 1) << 1)) + (2 * split_count - 1)) /
1502                (2 * split_count);
1503
1504     gpe_param->use_scoreboard  = use_hw_scoreboard;
1505     gpe_param->scoreboard_mask = 0xFF;
1506     gpe_param->global_resolution.x = ts_width;
1507     gpe_param->global_resolution.y = 4 * tmp1;
1508     gpe_param->global_start.x = 0;
1509     gpe_param->global_start.y = 0;
1510     gpe_param->global_outer_loop_stride.x = ts_width;
1511     gpe_param->global_outer_loop_stride.y = 0;
1512     gpe_param->global_inner_loop_unit.x = 0;
1513     gpe_param->global_inner_loop_unit.y = 4 * tmp1;
1514     gpe_param->block_resolution.x = ts_width;
1515     gpe_param->block_resolution.y = 4 * tmp1;
1516     gpe_param->local_start.x = ts_width;
1517     gpe_param->local_start.y = 0;
1518     gpe_param->local_end.x = 0;
1519     gpe_param->local_end.y = 0;
1520     gpe_param->local_outer_loop_stride.x = 1;
1521     gpe_param->local_outer_loop_stride.y = 0;
1522     gpe_param->local_inner_loop_unit.x = -2;
1523     gpe_param->local_inner_loop_unit.y = 4;
1524     gpe_param->middle_loop_extra_steps = 3;
1525     gpe_param->mid_loop_unit_x = 0;
1526     gpe_param->mid_loop_unit_y = 1;
1527     gpe_param->global_loop_exec_count = 0;
1528     gpe_param->local_loop_exec_count = ((lcu_width + (lcu_height - 1) * 2 + 2 * split_count - 1) /
1529                                         (2 * split_count)) * 2 -
1530                                        1;
1531
1532     gen9_hevc_vme_set_scoreboard_26z(gpe_context, 0xff, use_hw_scoreboard, scoreboard_type);
1533 }
1534
1535 static void
1536 gen9_hevc_init_object_walker_26(struct gen9_hevc_encoder_state *priv_state,
1537                                 struct i965_gpe_context *gpe_context,
1538                                 struct gpe_media_object_walker_parameter *gpe_param,
1539                                 int split_count,
1540                                 unsigned int max_slight_height,
1541                                 int use_hw_scoreboard,
1542                                 int scoreboard_type)
1543 {
1544     int width = priv_state->width_in_mb;
1545     int height = max_slight_height;
1546     int ts_width = (width + 1) & 0xfffe;
1547     int ts_height = (height + 1) & 0xfffe;
1548     int tmp1 = ((ts_width + 1) >> 1) +
1549                ((ts_width + ((ts_height - 1) << 1)) +
1550                 (2 * split_count - 1)) / (2 * split_count);
1551
1552     gpe_param->use_scoreboard = use_hw_scoreboard;
1553     gpe_param->scoreboard_mask = 0x0f;
1554     gpe_param->global_resolution.x = ts_width;
1555     gpe_param->global_resolution.y = tmp1;
1556     gpe_param->global_start.x = 0;
1557     gpe_param->global_start.y = 0;
1558     gpe_param->global_outer_loop_stride.x = ts_width;
1559     gpe_param->global_outer_loop_stride.y = 0;
1560     gpe_param->global_inner_loop_unit.x = 0;
1561     gpe_param->global_inner_loop_unit.y = tmp1;
1562     gpe_param->block_resolution.x = ts_width;
1563     gpe_param->block_resolution.y = tmp1;
1564     gpe_param->local_start.x = ts_width;
1565     gpe_param->local_start.y = 0;
1566     gpe_param->local_end.x = 0;
1567     gpe_param->local_end.y = 0;
1568     gpe_param->local_outer_loop_stride.x = 1;
1569     gpe_param->local_outer_loop_stride.y = 0;
1570     gpe_param->local_inner_loop_unit.x = -2;
1571     gpe_param->local_inner_loop_unit.y = 1;
1572     gpe_param->middle_loop_extra_steps = 0;
1573     gpe_param->mid_loop_unit_x = 0;
1574     gpe_param->mid_loop_unit_y = 0;
1575     gpe_param->global_loop_exec_count = 0;
1576     gpe_param->local_loop_exec_count = (width + (height - 1) * 2 + split_count - 1) /
1577                                        split_count;
1578
1579     gen9_hevc_vme_set_scoreboard_26(gpe_context, 0xff, use_hw_scoreboard, scoreboard_type);
1580 }
1581
1582 static void
1583 gen9_hevc_run_object_walker(VADriverContextP ctx,
1584                             struct intel_encoder_context *encoder_context,
1585                             struct i965_gpe_context *gpe_context,
1586                             struct gpe_media_object_walker_parameter *param,
1587                             int media_state)
1588 {
1589     struct intel_batchbuffer *batch = encoder_context->base.batch;
1590
1591     intel_batchbuffer_start_atomic(batch, 0x1000);
1592
1593     intel_batchbuffer_emit_mi_flush(batch);
1594
1595     gen9_gpe_pipeline_setup(ctx, gpe_context, batch);
1596     gen8_gpe_media_object_walker(ctx, gpe_context, batch, param);
1597     gen8_gpe_media_state_flush(ctx, gpe_context, batch);
1598     gen9_gpe_pipeline_end(ctx, gpe_context, batch);
1599
1600     intel_batchbuffer_end_atomic(batch);
1601
1602     intel_batchbuffer_flush(batch);
1603 }
1604
1605 static void
1606 gen9_hevc_run_object(VADriverContextP ctx,
1607                      struct intel_encoder_context *encoder_context,
1608                      struct i965_gpe_context *gpe_context,
1609                      struct gpe_media_object_parameter *param,
1610                      int media_state)
1611 {
1612     struct intel_batchbuffer *batch = encoder_context->base.batch;
1613
1614     intel_batchbuffer_start_atomic(batch, 0x1000);
1615
1616     intel_batchbuffer_emit_mi_flush(batch);
1617
1618     gen9_gpe_pipeline_setup(ctx, gpe_context, batch);
1619     gen8_gpe_media_object(ctx, gpe_context, batch, param);
1620     gen8_gpe_media_state_flush(ctx, gpe_context, batch);
1621
1622     gen9_gpe_pipeline_end(ctx, gpe_context, batch);
1623
1624     intel_batchbuffer_end_atomic(batch);
1625
1626     intel_batchbuffer_flush(batch);
1627 }
1628
1629 static void
1630 gen9_hevc_get_b_mbenc_default_curbe(enum HEVC_TU_MODE tu_mode,
1631                                     int slice_type,
1632                                     void **curbe_ptr,
1633                                     int *curbe_size)
1634 {
1635     if (tu_mode == HEVC_TU_BEST_SPEED) {
1636         if (slice_type == HEVC_SLICE_I) {
1637             *curbe_size = sizeof(HEVC_ENC_ENCB_TU7_I_CURBE_DATA);
1638             *curbe_ptr = (void *)HEVC_ENC_ENCB_TU7_I_CURBE_DATA;
1639         } else if (slice_type == HEVC_SLICE_P) {
1640             *curbe_size = sizeof(HEVC_ENC_ENCB_TU7_P_CURBE_DATA);
1641             *curbe_ptr = (void *)HEVC_ENC_ENCB_TU7_P_CURBE_DATA;
1642         } else {
1643             *curbe_size = sizeof(HEVC_ENC_ENCB_TU7_B_CURBE_DATA);
1644             *curbe_ptr = (void *)HEVC_ENC_ENCB_TU7_B_CURBE_DATA;
1645         }
1646     } else if (tu_mode == HEVC_TU_RT_SPEED) {
1647         if (slice_type == HEVC_SLICE_P) {
1648             *curbe_size = sizeof(HEVC_ENC_ENCB_TU4_P_CURBE_DATA);
1649             *curbe_ptr = (void *)HEVC_ENC_ENCB_TU4_P_CURBE_DATA;
1650         } else {
1651             *curbe_size = sizeof(HEVC_ENC_ENCB_TU4_B_CURBE_DATA);
1652             *curbe_ptr = (void *)HEVC_ENC_ENCB_TU4_B_CURBE_DATA;
1653         }
1654     } else {
1655         if (slice_type == HEVC_SLICE_P) {
1656             *curbe_size = sizeof(HEVC_ENC_ENCB_TU1_P_CURBE_DATA);
1657             *curbe_ptr = (void *)HEVC_ENC_ENCB_TU1_P_CURBE_DATA;
1658         } else {
1659             *curbe_size = sizeof(HEVC_ENC_ENCB_TU1_B_CURBE_DATA);
1660             *curbe_ptr = (void *)HEVC_ENC_ENCB_TU1_B_CURBE_DATA;
1661         }
1662     }
1663 }
1664
1665 // BRC start
1666
1667 static void
1668 gen9_hevc_configure_roi(struct encode_state *encode_state,
1669                         struct intel_encoder_context *encoder_context)
1670 {
1671     struct encoder_vme_mfc_context *vme_context = NULL;
1672     struct gen9_hevc_encoder_state *priv_state = NULL;
1673     int i = 0;
1674
1675     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
1676     priv_state = (struct gen9_hevc_encoder_state *)vme_context->private_enc_state;
1677
1678     priv_state->num_roi = MIN(encoder_context->brc.num_roi, I965_MAX_NUM_ROI_REGIONS);
1679     priv_state->roi_value_is_qp_delta = encoder_context->brc.roi_value_is_qp_delta;
1680
1681     for (i = 0; i < priv_state->num_roi; i++) {
1682         priv_state->roi[i].left = encoder_context->brc.roi[i].left >> 4;
1683         priv_state->roi[i].right = encoder_context->brc.roi[i].right >> 4;
1684         priv_state->roi[i].top = encoder_context->brc.roi[i].top >> 4;
1685         priv_state->roi[i].bottom = encoder_context->brc.roi[i].bottom >> 4;
1686         priv_state->roi[i].value = encoder_context->brc.roi[i].value;
1687     }
1688 }
1689
1690 static void
1691 gen9_hevc_brc_prepare(struct encode_state *encode_state,
1692                       struct intel_encoder_context *encoder_context)
1693 {
1694     struct encoder_vme_mfc_context *vme_context = NULL;
1695     struct generic_enc_codec_state *generic_state = NULL;
1696     struct gen9_hevc_encoder_state *priv_state = NULL;
1697     enum HEVC_BRC_METHOD brc_method = HEVC_BRC_CQP;
1698     int internal_tu_mode = encoder_context->quality_level;
1699     int brc_reset = 0;
1700
1701     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
1702     generic_state = (struct generic_enc_codec_state *)vme_context->generic_enc_state;
1703     priv_state = (struct gen9_hevc_encoder_state *)vme_context->private_enc_state;
1704
1705     if (encoder_context->rate_control_mode & VA_RC_CBR)
1706         brc_method = HEVC_BRC_CBR;
1707     else if (encoder_context->rate_control_mode & VA_RC_VBR)
1708         brc_method = HEVC_BRC_VBR;
1709     else if (encoder_context->rate_control_mode & VA_RC_VCM)
1710         brc_method = HEVC_BRC_VCM;
1711
1712     if (internal_tu_mode >= HEVC_TU_RT_SPEED ||
1713         internal_tu_mode == 0)
1714         internal_tu_mode = internal_tu_mode >= HEVC_TU_BEST_SPEED ?
1715                            HEVC_TU_BEST_SPEED : HEVC_TU_RT_SPEED;
1716     else
1717         internal_tu_mode = HEVC_TU_BEST_QUALITY;
1718
1719     brc_reset = priv_state->brc_method != brc_method ||
1720                 priv_state->tu_mode != internal_tu_mode;
1721
1722     if (!generic_state->brc_inited ||
1723         encoder_context->brc.need_reset ||
1724         brc_reset) {
1725         priv_state->tu_mode = internal_tu_mode;
1726         if (priv_state->tu_mode == HEVC_TU_BEST_QUALITY)
1727             priv_state->num_regions_in_slice = 1;
1728         else
1729             priv_state->num_regions_in_slice = 4;
1730
1731         if (internal_tu_mode == HEVC_TU_BEST_SPEED)
1732             priv_state->walking_pattern_26 = 1;
1733         else
1734             priv_state->walking_pattern_26 = 0;
1735
1736         if (brc_method == HEVC_BRC_CQP) {
1737             generic_state->brc_enabled = 0;
1738             generic_state->num_pak_passes = 1;
1739             priv_state->lcu_brc_enabled = 0;
1740         } else {
1741             generic_state->brc_enabled = 1;
1742             generic_state->num_pak_passes = 4;
1743
1744             if (brc_method == HEVC_BRC_VCM ||
1745                 encoder_context->brc.mb_rate_control[0] == 0)
1746                 priv_state->lcu_brc_enabled = (priv_state->tu_mode == HEVC_TU_BEST_QUALITY);
1747             else if (brc_method == HEVC_BRC_ICQ ||
1748                      encoder_context->brc.mb_rate_control[0] == 1)
1749                 priv_state->lcu_brc_enabled = 1;
1750             else
1751                 priv_state->lcu_brc_enabled = 0;
1752
1753             if (brc_method == HEVC_BRC_CBR) {
1754                 priv_state->target_bit_rate_in_kbs =
1755                     ALIGN(encoder_context->brc.bits_per_second[0], HEVC_BRC_KBPS) /
1756                     HEVC_BRC_KBPS;
1757                 priv_state->max_bit_rate_in_kbs = priv_state->target_bit_rate_in_kbs;
1758                 priv_state->min_bit_rate_in_kbs = priv_state->target_bit_rate_in_kbs;
1759             } else {
1760                 if (encoder_context->brc.target_percentage[0] > HEVC_BRC_MIN_TARGET_PERCENTAGE) {
1761                     priv_state->target_bit_rate_in_kbs =
1762                         ALIGN(encoder_context->brc.bits_per_second[0], HEVC_BRC_KBPS) /
1763                         HEVC_BRC_KBPS;
1764                     priv_state->max_bit_rate_in_kbs = priv_state->target_bit_rate_in_kbs;
1765                     priv_state->min_bit_rate_in_kbs = priv_state->target_bit_rate_in_kbs *
1766                                                       (2 * encoder_context->brc.target_percentage[0] - 100) /
1767                                                       100;
1768                     priv_state->target_bit_rate_in_kbs = priv_state->max_bit_rate_in_kbs *
1769                                                          encoder_context->brc.target_percentage[0] / 100;
1770
1771                     brc_reset = 1;
1772                 }
1773             }
1774
1775             if (encoder_context->brc.framerate[0].den)
1776                 priv_state->frames_per_100s = encoder_context->brc.framerate[0].num * 100 /
1777                                               encoder_context->brc.framerate[0].den;
1778
1779             priv_state->init_vbv_buffer_fullness_in_bit =
1780                 encoder_context->brc.hrd_initial_buffer_fullness;
1781             priv_state->vbv_buffer_size_in_bit =
1782                 encoder_context->brc.hrd_buffer_size;
1783         }
1784
1785         priv_state->brc_method = brc_method;
1786         generic_state->brc_need_reset = brc_reset;
1787         encoder_context->brc.need_reset = 0;
1788     }
1789
1790     gen9_hevc_configure_roi(encode_state, encoder_context);
1791 }
1792
1793 static void
1794 gen9_hevc_brc_init_rest_set_curbe(VADriverContextP ctx,
1795                                   struct encode_state *encode_state,
1796                                   struct intel_encoder_context *encoder_context,
1797                                   struct i965_gpe_context *gpe_context,
1798                                   int reset)
1799 {
1800     struct encoder_vme_mfc_context *vme_context = NULL;
1801     struct gen9_hevc_encoder_state *priv_state = NULL;
1802     gen9_hevc_brc_initreset_curbe_data *cmd = NULL;
1803     VAEncSequenceParameterBufferHEVC *seq_param = NULL;
1804     double input_bits_per_frame = 0;
1805     double bps_ratio = 0;
1806
1807     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
1808     priv_state = (struct gen9_hevc_encoder_state *)vme_context->private_enc_state;
1809     seq_param = (VAEncSequenceParameterBufferHEVC *)encode_state->seq_param_ext->buffer;
1810
1811     cmd = i965_gpe_context_map_curbe(gpe_context);
1812     if (!cmd)
1813         return;
1814
1815     memcpy((void *)cmd, GEN9_HEVC_BRCINIT_CURBE_DATA, sizeof(GEN9_HEVC_BRCINIT_CURBE_DATA));
1816
1817     cmd->dw0.profile_level_max_frame = gen9_hevc_get_profile_level_max_frame(seq_param,
1818                                                                              priv_state->user_max_frame_size,
1819                                                                              priv_state->frames_per_100s);
1820     cmd->dw1.init_buf_full = priv_state->init_vbv_buffer_fullness_in_bit;
1821     cmd->dw2.buf_size = priv_state->vbv_buffer_size_in_bit;
1822     cmd->dw3.targe_bit_rate = priv_state->target_bit_rate_in_kbs * HEVC_BRC_KBPS;
1823     cmd->dw4.maximum_bit_rate = priv_state->max_bit_rate_in_kbs * HEVC_BRC_KBPS;
1824
1825     cmd->dw9.frame_width = priv_state->picture_width;
1826     cmd->dw10.frame_height = priv_state->picture_height;
1827     cmd->dw12.number_slice = encode_state->num_slice_params_ext;
1828     cmd->dw6.frame_rate_m = priv_state->frames_per_100s;
1829     cmd->dw7.frame_rate_d = 100;
1830     cmd->dw8.brc_flag = 0;
1831     cmd->dw8.brc_flag |= (priv_state->lcu_brc_enabled) ? 0 : HEVC_BRCINIT_DISABLE_MBBRC;
1832     cmd->dw25.acqp_buffer = 1;
1833
1834     if (priv_state->brc_method == HEVC_BRC_CBR) {
1835         cmd->dw4.maximum_bit_rate = cmd->dw3.targe_bit_rate;
1836         cmd->dw8.brc_flag |= HEVC_BRCINIT_ISCBR;
1837     } else if (priv_state->brc_method == HEVC_BRC_VBR) {
1838         if (cmd->dw4.maximum_bit_rate < cmd->dw3.targe_bit_rate)
1839             cmd->dw4.maximum_bit_rate = cmd->dw3.targe_bit_rate * 2;
1840         cmd->dw8.brc_flag |= HEVC_BRCINIT_ISVBR;
1841     } else if (priv_state->brc_method == HEVC_BRC_AVBR) {
1842         cmd->dw4.maximum_bit_rate = priv_state->target_bit_rate_in_kbs * HEVC_BRC_KBPS;
1843         cmd->dw8.brc_flag |= HEVC_BRCINIT_ISAVBR;
1844     } else if (priv_state->brc_method == HEVC_BRC_ICQ) {
1845         cmd->dw25.acqp_buffer = priv_state->crf_quality_factor;
1846         cmd->dw8.brc_flag |= HEVC_BRCINIT_ISICQ;
1847     } else if (priv_state->brc_method == HEVC_BRC_VCM) {
1848         cmd->dw4.maximum_bit_rate = priv_state->target_bit_rate_in_kbs * HEVC_BRC_KBPS;
1849         cmd->dw8.brc_flag |= HEVC_BRCINIT_ISVCM;
1850     }
1851
1852     if (priv_state->num_b_in_gop[1] ||
1853         priv_state->num_b_in_gop[2]) {
1854         cmd->dw8.brc_param_a = priv_state->gop_size / priv_state->gop_ref_dist;
1855         cmd->dw9.brc_param_b = cmd->dw8.brc_param_a;
1856         cmd->dw13.brc_param_c = cmd->dw8.brc_param_a * 2;
1857         cmd->dw14.brc_param_d = priv_state->gop_size - cmd->dw8.brc_param_a -
1858                                 cmd->dw9.brc_param_b - cmd->dw13.brc_param_c;
1859
1860         if (!priv_state->num_b_in_gop[2])
1861             cmd->dw14.max_brc_level = 3;
1862         else
1863             cmd->dw14.max_brc_level = 4;
1864     } else {
1865         cmd->dw14.max_brc_level = 1;
1866         cmd->dw8.brc_param_a = priv_state->gop_ref_dist ? (priv_state->gop_size - 1) / priv_state->gop_ref_dist : 0;
1867         cmd->dw9.brc_param_b = priv_state->gop_size - 1 - cmd->dw8.brc_param_a;
1868     }
1869
1870     cmd->dw10.avbr_accuracy = GEN9_HEVC_AVBR_ACCURACY;
1871     cmd->dw11.avbr_convergence = GEN9_HEVC_AVBR_CONVERGENCE;
1872
1873     input_bits_per_frame = (double)(cmd->dw4.maximum_bit_rate) *
1874                            (double)(cmd->dw7.frame_rate_d) /
1875                            (double)(cmd->dw6.frame_rate_m);
1876     if (cmd->dw2.buf_size < (unsigned int)input_bits_per_frame * 4)
1877         cmd->dw2.buf_size = (unsigned int)input_bits_per_frame * 4;
1878
1879     if (cmd->dw1.init_buf_full == 0)
1880         cmd->dw1.init_buf_full = cmd->dw2.buf_size * 7 / 8;
1881
1882     if (cmd->dw1.init_buf_full < (unsigned int)(input_bits_per_frame * 2))
1883         cmd->dw1.init_buf_full = (unsigned int)(input_bits_per_frame * 2);
1884
1885     if (cmd->dw1.init_buf_full > cmd->dw2.buf_size)
1886         cmd->dw1.init_buf_full = cmd->dw2.buf_size;
1887
1888     if (priv_state->brc_method == HEVC_BRC_AVBR) {
1889         cmd->dw2.buf_size = priv_state->target_bit_rate_in_kbs * 2 * HEVC_BRC_KBPS;
1890         cmd->dw1.init_buf_full = (unsigned int)(cmd->dw2.buf_size * 3 / 4);
1891     }
1892
1893     bps_ratio = input_bits_per_frame / (cmd->dw2.buf_size / 30);
1894     bps_ratio = (bps_ratio < 0.1) ? 0.1 : (bps_ratio > 3.5) ? 3.5 : bps_ratio;
1895
1896     cmd->dw19.deviation_threshold0_pbframe = (unsigned int)(-50 * pow(0.90, bps_ratio));
1897     cmd->dw19.deviation_threshold1_pbframe = (unsigned int)(-50 * pow(0.66, bps_ratio));
1898     cmd->dw19.deviation_threshold2_pbframe = (unsigned int)(-50 * pow(0.46, bps_ratio));
1899     cmd->dw19.deviation_threshold3_pbframe = (unsigned int)(-50 * pow(0.3, bps_ratio));
1900
1901     cmd->dw20.deviation_threshold4_pbframe = (unsigned int)(50 *  pow(0.3, bps_ratio));
1902     cmd->dw20.deviation_threshold5_pbframe = (unsigned int)(50 * pow(0.46, bps_ratio));
1903     cmd->dw20.deviation_threshold6_pbframe = (unsigned int)(50 * pow(0.7,  bps_ratio));
1904     cmd->dw20.deviation_threshold7_pbframe = (unsigned int)(50 * pow(0.9,  bps_ratio));
1905
1906     cmd->dw21.deviation_threshold0_vbr_control = (unsigned int)(-50 * pow(0.9, bps_ratio));
1907     cmd->dw21.deviation_threshold1_vbr_control = (unsigned int)(-50 * pow(0.7, bps_ratio));
1908     cmd->dw21.deviation_threshold2_vbr_control = (unsigned int)(-50 * pow(0.5, bps_ratio));
1909     cmd->dw21.deviation_threshold3_vbr_control = (unsigned int)(-50 * pow(0.3, bps_ratio));
1910
1911     cmd->dw22.deviation_threshold4_vbr_control = (unsigned int)(100 * pow(0.4, bps_ratio));
1912     cmd->dw22.deviation_threshold5_vbr_control = (unsigned int)(100 * pow(0.5, bps_ratio));
1913     cmd->dw22.deviation_threshold6_vbr_control = (unsigned int)(100 * pow(0.75, bps_ratio));
1914     cmd->dw22.deviation_threshold7_vbr_control = (unsigned int)(100 * pow(0.9, bps_ratio));
1915
1916     cmd->dw23.deviation_threshold0_iframe = (unsigned int)(-50 * pow(0.8, bps_ratio));
1917     cmd->dw23.deviation_threshold1_iframe = (unsigned int)(-50 * pow(0.6, bps_ratio));
1918     cmd->dw23.deviation_threshold2_iframe = (unsigned int)(-50 * pow(0.34, bps_ratio));
1919     cmd->dw23.deviation_threshold3_iframe = (unsigned int)(-50 * pow(0.2, bps_ratio));
1920
1921     cmd->dw24.deviation_threshold4_iframe = (unsigned int)(50 * pow(0.2,  bps_ratio));
1922     cmd->dw24.deviation_threshold5_iframe = (unsigned int)(50 * pow(0.4,  bps_ratio));
1923     cmd->dw24.deviation_threshold6_iframe = (unsigned int)(50 * pow(0.66, bps_ratio));
1924     cmd->dw24.deviation_threshold7_iframe = (unsigned int)(50 * pow(0.9,  bps_ratio));
1925
1926     if (!reset)
1927         priv_state->brc_init_current_target_buf_full_in_bits = cmd->dw1.init_buf_full;
1928
1929     priv_state->brc_init_reset_buf_size_in_bits = (double)cmd->dw2.buf_size;
1930     priv_state->brc_init_reset_input_bits_per_frame = input_bits_per_frame;
1931
1932     i965_gpe_context_unmap_curbe(gpe_context);
1933 }
1934
1935 static void
1936 gen9_hevc_brc_init_rest_set_surfaces(VADriverContextP ctx,
1937                                      struct intel_encoder_context *encoder_context,
1938                                      struct i965_gpe_context *gpe_context)
1939 {
1940     struct encoder_vme_mfc_context *vme_context = NULL;
1941     struct gen9_hevc_encoder_context *priv_ctx = NULL;
1942     int bti_idx = 0;
1943
1944     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
1945     priv_ctx = (struct gen9_hevc_encoder_context *)vme_context->private_enc_ctx;
1946
1947     gen9_hevc_set_gpe_1d_surface(ctx, priv_ctx, gpe_context,
1948                                  HEVC_ENC_SURFACE_BRC_HISTORY, bti_idx++,
1949                                  0, 0, 0, NULL, NULL);
1950
1951     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
1952                                  HEVC_ENC_SURFACE_BRC_ME_DIST, bti_idx++,
1953                                  0, 1, I965_SURFACEFORMAT_R8_UNORM,
1954                                  &priv_ctx->res_brc_me_dist_buffer, NULL);
1955 }
1956
1957 static void
1958 gen9_hevc_brc_init_reset(VADriverContextP ctx,
1959                          struct encode_state *encode_state,
1960                          struct intel_encoder_context *encoder_context,
1961                          int reset)
1962 {
1963     struct encoder_vme_mfc_context *vme_context = NULL;
1964     struct gen9_hevc_encoder_context *priv_ctx = NULL;
1965
1966     struct i965_gpe_context *gpe_context = NULL;
1967     struct gpe_media_object_parameter param;
1968     int media_state = HEVC_ENC_MEDIA_STATE_BRC_INIT_RESET;
1969     int gpe_idx = reset ? HEVC_BRC_RESET_IDX : HEVC_BRC_INIT_IDX;
1970
1971     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
1972     priv_ctx = (struct gen9_hevc_encoder_context *)vme_context->private_enc_ctx;
1973     gpe_context = &priv_ctx->brc_context.gpe_contexts[gpe_idx];
1974
1975     gen8_gpe_context_init(ctx, gpe_context);
1976     gen9_gpe_reset_binding_table(ctx, gpe_context);
1977     gen9_hevc_brc_init_rest_set_curbe(ctx, encode_state, encoder_context, gpe_context,
1978                                       reset);
1979     gen9_hevc_brc_init_rest_set_surfaces(ctx, encoder_context, gpe_context);
1980     gen8_gpe_setup_interface_data(ctx, gpe_context);
1981
1982     memset(&param, 0, sizeof(param));
1983     gen9_hevc_run_object(ctx, encoder_context, gpe_context, &param,
1984                          media_state);
1985 }
1986
1987 static void
1988 gen9_hevc_brc_intra_dist_set_curbe(VADriverContextP ctx,
1989                                    struct encode_state *encode_state,
1990                                    struct intel_encoder_context *encoder_context,
1991                                    struct i965_gpe_context *gpe_context)
1992 {
1993     struct encoder_vme_mfc_context *vme_context = NULL;
1994     struct gen9_hevc_encoder_state *priv_state = NULL;
1995     gen9_hevc_brc_coarse_intra_curbe_data *cmd = NULL;
1996     int bti_idx = 0;
1997
1998     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
1999     priv_state = (struct gen9_hevc_encoder_state *)vme_context->private_enc_state;
2000
2001     cmd = i965_gpe_context_map_curbe(gpe_context);
2002     if (!cmd)
2003         return;
2004
2005     memset((void *)cmd, 0, sizeof(*cmd));
2006
2007     cmd->dw0.picture_width_in_luma_samples = priv_state->frame_width_4x;
2008     cmd->dw0.picture_height_in_luma_samples = priv_state->frame_height_4x;
2009
2010     cmd->dw1.inter_sad = 2;
2011     cmd->dw1.intra_sad = 2;
2012
2013     cmd->dw8.bti_src_y4 = bti_idx++;
2014     cmd->dw9.bti_intra_dist = bti_idx++;
2015     cmd->dw10.bti_vme_intra = bti_idx++;
2016
2017     i965_gpe_context_unmap_curbe(gpe_context);
2018 }
2019
2020 static void
2021 gen9_hevc_brc_intra_dist_set_surfaces(VADriverContextP ctx,
2022                                       struct encode_state *encode_state,
2023                                       struct intel_encoder_context *encoder_context,
2024                                       struct i965_gpe_context *gpe_context)
2025 {
2026     struct encoder_vme_mfc_context *vme_context = NULL;
2027     struct gen9_hevc_encoder_context *priv_ctx = NULL;
2028     struct object_surface *obj_surface;
2029     struct gen9_hevc_surface_priv *surface_priv;
2030     int bti_idx = 0;
2031
2032     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
2033     priv_ctx = (struct gen9_hevc_encoder_context *)vme_context->private_enc_ctx;
2034
2035     obj_surface = encode_state->reconstructed_object;
2036     surface_priv = (struct gen9_hevc_surface_priv *)(obj_surface->private_data);
2037     obj_surface = surface_priv->scaled_surface_obj[HEVC_SCALED_SURF_4X_ID];
2038
2039     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
2040                                  HEVC_ENC_SURFACE_Y_4X, bti_idx++,
2041                                  0, 1, I965_SURFACEFORMAT_R8_UNORM,
2042                                  NULL, obj_surface);
2043
2044     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
2045                                  HEVC_ENC_SURFACE_BRC_ME_DIST, bti_idx++,
2046                                  0, 1, I965_SURFACEFORMAT_R8_UNORM,
2047                                  &priv_ctx->res_brc_intra_dist_buffer, NULL);
2048
2049     gen9_hevc_set_gpe_adv_surface(ctx, priv_ctx, gpe_context,
2050                                   HEVC_ENC_SURFACE_Y_4X_VME, bti_idx++,
2051                                   obj_surface);
2052 }
2053
2054 static void
2055 gen9_hevc_brc_intra_dist(VADriverContextP ctx,
2056                          struct encode_state *encode_state,
2057                          struct intel_encoder_context *encoder_context)
2058 {
2059     struct encoder_vme_mfc_context *vme_context = NULL;
2060     struct gen9_hevc_encoder_context *priv_ctx = NULL;
2061     struct gen9_hevc_encoder_state *priv_state = NULL;
2062     struct i965_gpe_context *gpe_context = NULL;
2063     struct gpe_media_object_walker_parameter param;
2064     struct hevc_enc_kernel_walker_parameter hevc_walker_param;
2065     int media_state = HEVC_ENC_MEDIA_STATE_ENC_I_FRAME_DIST;
2066     int gpe_idx = HEVC_BRC_COARSE_INTRA_IDX;
2067
2068     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
2069     priv_ctx = (struct gen9_hevc_encoder_context *)vme_context->private_enc_ctx;
2070     priv_state = (struct gen9_hevc_encoder_state *)vme_context->private_enc_state;
2071     gpe_context = &priv_ctx->brc_context.gpe_contexts[gpe_idx];
2072
2073     gen8_gpe_context_init(ctx, gpe_context);
2074     gen9_gpe_reset_binding_table(ctx, gpe_context);
2075     gen9_hevc_brc_intra_dist_set_curbe(ctx, encode_state, encoder_context, gpe_context);
2076     gen9_hevc_brc_intra_dist_set_surfaces(ctx, encode_state, encoder_context, gpe_context);
2077     gen8_gpe_setup_interface_data(ctx, gpe_context);
2078
2079     memset((void *)&hevc_walker_param, 0, sizeof(hevc_walker_param));
2080     hevc_walker_param.resolution_x = priv_state->downscaled_width_4x_in_mb;
2081     hevc_walker_param.resolution_y = priv_state->downscaled_height_4x_in_mb;
2082     hevc_walker_param.no_dependency = 1;
2083     gen9_hevc_init_object_walker(&hevc_walker_param, &param);
2084     gen9_hevc_run_object_walker(ctx, encoder_context, gpe_context, &param,
2085                                 media_state);
2086 }
2087
2088 static GEN9_HEVC_BRC_UPDATE_FRAME_TYPE gen9_hevc_get_brc_frame_type(unsigned int pic_type,
2089                                                                     int low_delay)
2090 {
2091     if (pic_type == HEVC_SLICE_I)
2092         return HEVC_BRC_FTYPE_I;
2093     else if (pic_type == HEVC_SLICE_P)
2094         return HEVC_BRC_FTYPE_P_OR_LB;
2095     else
2096         return low_delay ? HEVC_BRC_FTYPE_P_OR_LB : HEVC_BRC_FTYPE_B;
2097 }
2098
2099 static void
2100 gen9_hevc_brc_update_set_roi_curbe(VADriverContextP ctx,
2101                                    struct encode_state *encode_state,
2102                                    struct intel_encoder_context *encoder_context,
2103                                    gen9_hevc_brc_udpate_curbe_data *cmd)
2104 {
2105     struct encoder_vme_mfc_context *vme_context = NULL;
2106     struct generic_enc_codec_state *generic_state = NULL;
2107     struct gen9_hevc_encoder_state *priv_state = NULL;
2108     struct intel_roi *roi_par = NULL;
2109     unsigned int roi_size = 0, roi_ratio = 0;
2110     int i = 0;
2111
2112     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
2113     generic_state = (struct generic_enc_codec_state *)vme_context->generic_enc_state;
2114     priv_state = (struct gen9_hevc_encoder_state *)vme_context->private_enc_state;
2115
2116     cmd->dw6.cqp_value = 0;
2117     cmd->dw6.roi_flag = 1 | (generic_state->brc_enabled << 1) |
2118                         (priv_state->video_surveillance_flag << 2);
2119
2120     for (i = 0; i < priv_state->num_roi; i++) {
2121         roi_par = &priv_state->roi[i];
2122         roi_size += abs(roi_par->right - roi_par->left) *
2123                     abs(roi_par->bottom - roi_par->top) * 256;
2124     }
2125
2126     if (roi_size)
2127         roi_ratio = MIN(2 * (priv_state->width_in_mb * priv_state->height_in_mb * 256 / roi_size - 1),
2128                         51);
2129
2130     cmd->dw6.roi_ratio = roi_ratio;
2131     cmd->dw7.frame_width_in_lcu = priv_state->frame_width_in_max_lcu;
2132
2133     if (!generic_state->brc_enabled) {
2134         VAEncPictureParameterBufferHEVC *pic_param = NULL;
2135         VAEncSliceParameterBufferHEVC *slice_param = NULL;
2136
2137         pic_param = (VAEncPictureParameterBufferHEVC *)encode_state->pic_param_ext->buffer;
2138         slice_param = (VAEncSliceParameterBufferHEVC *)encode_state->slice_params_ext[0]->buffer;
2139
2140         cmd->dw1.frame_number = priv_state->frame_number;
2141         cmd->dw6.cqp_value = pic_param->pic_init_qp + slice_param->slice_qp_delta;
2142         cmd->dw5.curr_frame_type = gen9_hevc_get_brc_frame_type(priv_state->picture_coding_type,
2143                                                                 priv_state->low_delay);
2144     }
2145 }
2146
2147 static void
2148 gen9_hevc_brc_update_lcu_based_set_roi_parameters(VADriverContextP ctx,
2149                                                   struct encode_state *encode_state,
2150                                                   struct intel_encoder_context *encoder_context,
2151                                                   struct i965_gpe_context *gpe_context)
2152 {
2153     struct encoder_vme_mfc_context *vme_context = NULL;
2154     struct gen9_hevc_encoder_context *priv_ctx = NULL;
2155     struct generic_enc_codec_state *generic_state = NULL;
2156     struct gen9_hevc_encoder_state *priv_state = NULL;
2157     struct intel_roi *roi_par = NULL;
2158     unsigned int width_in_mb_aligned = 0;
2159     unsigned int roi_level, qp_delta;
2160     unsigned int mb_num = 0;
2161     unsigned int *pdata = NULL;
2162     unsigned int out_data = 0;
2163     int i = 0;
2164
2165     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
2166     priv_ctx = (struct gen9_hevc_encoder_context *)vme_context->private_enc_ctx;
2167     generic_state = (struct generic_enc_codec_state *)vme_context->generic_enc_state;
2168     priv_state = (struct gen9_hevc_encoder_state *)vme_context->private_enc_state;
2169
2170     width_in_mb_aligned = ALIGN(priv_state->width_in_mb * 4, 64);
2171     mb_num = priv_state->width_in_mb * priv_state->height_in_mb;
2172
2173     pdata = i965_map_gpe_resource(&priv_ctx->res_roi_buffer);
2174     if (!pdata)
2175         return;
2176
2177     for (i = 0 ; i < mb_num; i++) {
2178         int cur_mb_y = i / priv_state->width_in_mb;
2179         int cur_mb_x = i - cur_mb_y * priv_state->width_in_mb;
2180         int roi_idx = 0;
2181
2182         out_data = 0;
2183
2184         for (roi_idx = (priv_state->num_roi - 1); roi_idx >= 0; roi_idx--) {
2185             roi_par = &priv_state->roi[roi_idx];
2186
2187             roi_level = qp_delta = 0;
2188             if (generic_state->brc_enabled && !priv_state->roi_value_is_qp_delta)
2189                 roi_level = roi_par->value * 5;
2190             else
2191                 qp_delta = roi_par->value;
2192
2193             if (roi_level == 0 && qp_delta == 0)
2194                 continue;
2195
2196             if ((cur_mb_x >= roi_par->left) &&
2197                 (cur_mb_x < roi_par->right) &&
2198                 (cur_mb_y >= roi_par->top) &&
2199                 (cur_mb_y < roi_par->bottom))
2200                 out_data = 15 | (((roi_level) & 0xFF) << 8) | ((qp_delta & 0xFF) << 16);
2201             else if ((cur_mb_x >= roi_par->left - 1) &&
2202                      (cur_mb_x < roi_par->right + 1) &&
2203                      (cur_mb_y >= roi_par->top - 1) &&
2204                      (cur_mb_y < roi_par->bottom + 1))
2205                 out_data = 14 | (((roi_level) & 0xFF) << 8) | ((qp_delta & 0xFF) << 16);
2206             else if ((cur_mb_x >= roi_par->left - 2) &&
2207                      (cur_mb_x < roi_par->right + 2) &&
2208                      (cur_mb_y >= roi_par->top - 2) &&
2209                      (cur_mb_y < roi_par->bottom + 2))
2210                 out_data = 13 | (((roi_level) & 0xFF) << 8) | ((qp_delta & 0xFF) << 16);
2211             else if ((cur_mb_x >= roi_par->left - 3) &&
2212                      (cur_mb_x < roi_par->right + 3) &&
2213                      (cur_mb_y >= roi_par->top - 3) &&
2214                      (cur_mb_y < roi_par->bottom + 3)) {
2215                 out_data = 12 | (((roi_level) & 0xFF) << 8) | ((qp_delta & 0xFF) << 16);
2216             }
2217         }
2218
2219         pdata[(cur_mb_y * (width_in_mb_aligned >> 2)) + cur_mb_x] = out_data;
2220     }
2221
2222     i965_unmap_gpe_resource(&priv_ctx->res_roi_buffer);
2223 }
2224
2225 static void
2226 gen9_hevc_brc_update_lcu_based_set_curbe(VADriverContextP ctx,
2227                                          struct encode_state *encode_state,
2228                                          struct intel_encoder_context *encoder_context,
2229                                          struct i965_gpe_context *gpe_context)
2230 {
2231     struct encoder_vme_mfc_context *vme_context = NULL;
2232     struct gen9_hevc_encoder_context *priv_ctx = NULL;
2233     struct generic_enc_codec_state *generic_state = NULL;
2234     struct gen9_hevc_encoder_state *priv_state = NULL;
2235     gen9_hevc_brc_udpate_curbe_data *cmd = NULL, *frame_cmd = NULL;
2236
2237     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
2238     priv_ctx = (struct gen9_hevc_encoder_context *)vme_context->private_enc_ctx;
2239     generic_state = (struct generic_enc_codec_state *)vme_context->generic_enc_state;
2240     priv_state = (struct gen9_hevc_encoder_state *)vme_context->private_enc_state;
2241
2242     cmd = i965_gpe_context_map_curbe(gpe_context);
2243     if (!cmd)
2244         return;
2245
2246     if (generic_state->brc_enabled) {
2247         frame_cmd = i965_gpe_context_map_curbe(&priv_ctx->brc_context.gpe_contexts[HEVC_BRC_FRAME_UPDATE_IDX]);
2248
2249         if (!frame_cmd)
2250             return;
2251
2252         memcpy((void *)cmd, (void *)frame_cmd, sizeof(*cmd));
2253
2254         i965_gpe_context_unmap_curbe(&priv_ctx->brc_context.gpe_contexts[HEVC_BRC_FRAME_UPDATE_IDX]);
2255     } else {
2256         memcpy((void *)cmd, GEN9_HEVC_BRCUPDATE_CURBE_DATA,
2257                sizeof(GEN9_HEVC_BRCUPDATE_CURBE_DATA));
2258     }
2259
2260     if (priv_state->num_roi)
2261         gen9_hevc_brc_update_set_roi_curbe(ctx, encode_state, encoder_context, cmd);
2262
2263     i965_gpe_context_unmap_curbe(gpe_context);
2264 }
2265
2266 static void
2267 gen9_hevc_brc_update_lcu_based_set_surfaces(VADriverContextP ctx,
2268                                             struct encode_state *encode_state,
2269                                             struct intel_encoder_context *encoder_context,
2270                                             struct i965_gpe_context *gpe_context)
2271 {
2272     struct encoder_vme_mfc_context *vme_context = NULL;
2273     struct gen9_hevc_encoder_context *priv_ctx = NULL;
2274     struct gen9_hevc_encoder_state *priv_state = NULL;
2275     int bti_idx = 0;
2276
2277     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
2278     priv_ctx = (struct gen9_hevc_encoder_context *)vme_context->private_enc_ctx;
2279     priv_state = (struct gen9_hevc_encoder_state *)vme_context->private_enc_state;
2280
2281     gen9_hevc_set_gpe_1d_surface(ctx, priv_ctx, gpe_context,
2282                                  HEVC_ENC_SURFACE_BRC_HISTORY, bti_idx++,
2283                                  0, 0, 0, NULL, NULL);
2284
2285     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
2286                                  HEVC_ENC_SURFACE_BRC_ME_DIST, bti_idx++,
2287                                  0, 1, I965_SURFACEFORMAT_R8_UNORM,
2288                                  priv_state->picture_coding_type == HEVC_SLICE_I ?
2289                                  &priv_ctx->res_brc_intra_dist_buffer :
2290                                  &priv_ctx->res_brc_me_dist_buffer,
2291                                  NULL);
2292
2293     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
2294                                  HEVC_ENC_SURFACE_BRC_ME_DIST, bti_idx++,
2295                                  0, 1, I965_SURFACEFORMAT_R8_UNORM,
2296                                  &priv_ctx->res_brc_intra_dist_buffer, NULL);
2297
2298     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
2299                                  HEVC_ENC_SURFACE_HME_MVP, bti_idx++,
2300                                  0, 1, I965_SURFACEFORMAT_R8_UNORM,
2301                                  NULL, NULL);
2302
2303     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
2304                                  HEVC_ENC_SURFACE_LCU_QP, bti_idx++,
2305                                  0, 1, I965_SURFACEFORMAT_R8_UNORM,
2306                                  NULL, NULL);
2307
2308     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
2309                                  HEVC_ENC_SURFACE_ROI, bti_idx++,
2310                                  0, 1, I965_SURFACEFORMAT_R8_UNORM,
2311                                  NULL, NULL);
2312 }
2313
2314 static void
2315 gen9_hevc_brc_update_lcu_based(VADriverContextP ctx,
2316                                struct encode_state *encode_state,
2317                                struct intel_encoder_context *encoder_context)
2318 {
2319     struct encoder_vme_mfc_context *vme_context = NULL;
2320     struct gen9_hevc_encoder_context *priv_ctx = NULL;
2321     struct gen9_hevc_encoder_state *priv_state = NULL;
2322     struct i965_gpe_context *gpe_context = NULL;
2323     struct gpe_media_object_walker_parameter param;
2324     struct hevc_enc_kernel_walker_parameter hevc_walker_param;
2325     int media_state = HEVC_ENC_MEDIA_STATE_HEVC_BRC_LCU_UPDATE;
2326     int gpe_idx = HEVC_BRC_LCU_UPDATE_IDX;
2327
2328     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
2329     priv_ctx = (struct gen9_hevc_encoder_context *)vme_context->private_enc_ctx;
2330     priv_state = (struct gen9_hevc_encoder_state *)vme_context->private_enc_state;
2331     gpe_context = &priv_ctx->brc_context.gpe_contexts[gpe_idx];
2332
2333     gen8_gpe_context_init(ctx, gpe_context);
2334     gen9_gpe_reset_binding_table(ctx, gpe_context);
2335
2336     if (priv_state->num_roi)
2337         gen9_hevc_brc_update_lcu_based_set_roi_parameters(ctx, encode_state, encoder_context,
2338                                                           gpe_context);
2339
2340     gen9_hevc_brc_update_lcu_based_set_curbe(ctx, encode_state, encoder_context, gpe_context);
2341     gen9_hevc_brc_update_lcu_based_set_surfaces(ctx, encode_state, encoder_context, gpe_context);
2342     gen8_gpe_setup_interface_data(ctx, gpe_context);
2343
2344     memset((void *)&hevc_walker_param, 0, sizeof(hevc_walker_param));
2345     hevc_walker_param.resolution_x = ALIGN(priv_state->picture_width, 128) >> 7;
2346     hevc_walker_param.resolution_y = ALIGN(priv_state->picture_height, 128) >> 7;
2347     hevc_walker_param.no_dependency = 1;
2348     gen9_hevc_init_object_walker(&hevc_walker_param, &param);
2349     gen9_hevc_run_object_walker(ctx, encoder_context, gpe_context, &param,
2350                                 media_state);
2351 }
2352
2353 static void
2354 gen9_hevc_add_pic_state(VADriverContextP ctx,
2355                         struct encode_state *encode_state,
2356                         struct intel_encoder_context *encoder_context,
2357                         struct i965_gpe_resource *pic_state_ptr,
2358                         int pic_state_offset,
2359                         int brc_update)
2360 {
2361     struct i965_driver_data *i965 = i965_driver_data(ctx);
2362     struct encoder_vme_mfc_context *pak_context = NULL;
2363     struct gen9_hevc_encoder_state *priv_state = NULL;
2364     VAEncPictureParameterBufferHEVC *pic_param = NULL;
2365     VAEncSequenceParameterBufferHEVC *seq_param = NULL;
2366     unsigned int tmp_data[31], *cmd_ptr = NULL;
2367     int cmd_size = 0;
2368
2369     pak_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
2370     priv_state = (struct gen9_hevc_encoder_state *)pak_context->private_enc_state;
2371
2372     pic_param = (VAEncPictureParameterBufferHEVC *)encode_state->pic_param_ext->buffer;
2373     seq_param = (VAEncSequenceParameterBufferHEVC *)encode_state->seq_param_ext->buffer;
2374
2375     cmd_ptr = tmp_data;
2376     cmd_size = (IS_KBL(i965->intel.device_info) || IS_GLK(i965->intel.device_info)) ? 31 : 19;
2377     memset((void *)tmp_data, 0, 4 * cmd_size);
2378
2379     if (IS_KBL(i965->intel.device_info) || IS_GLK(i965->intel.device_info))
2380         *cmd_ptr++ = HCP_PIC_STATE | (31 - 2);
2381     else
2382         *cmd_ptr++ = HCP_PIC_STATE | (19 - 2);
2383
2384     *cmd_ptr++ = (priv_state->height_in_cu - 1) << 16 |
2385                  0 << 14 |
2386                  (priv_state->width_in_cu - 1);
2387     *cmd_ptr++ = (seq_param->log2_min_transform_block_size_minus2 +
2388                   seq_param->log2_diff_max_min_transform_block_size) << 6 |
2389                  seq_param->log2_min_transform_block_size_minus2 << 4 |
2390                  (seq_param->log2_min_luma_coding_block_size_minus3 +
2391                   seq_param->log2_diff_max_min_luma_coding_block_size) << 2 |
2392                  seq_param->log2_min_luma_coding_block_size_minus3;
2393     *cmd_ptr++ = 0;
2394     *cmd_ptr++ = ((IS_KBL(i965->intel.device_info) || IS_GLK(i965->intel.device_info)) ? 1 : 0) << 27 |
2395                  seq_param->seq_fields.bits.strong_intra_smoothing_enabled_flag << 26 |
2396                  pic_param->pic_fields.bits.transquant_bypass_enabled_flag << 25 |
2397                  ((IS_KBL(i965->intel.device_info) || IS_GLK(i965->intel.device_info)) ? 0 : priv_state->ctu_max_bitsize_allowed > 0) << 24 |
2398                  seq_param->seq_fields.bits.amp_enabled_flag << 23 |
2399                  pic_param->pic_fields.bits.transform_skip_enabled_flag << 22 |
2400                  0 << 21 |
2401                  0 << 20 |
2402                  pic_param->pic_fields.bits.weighted_pred_flag << 19 |
2403                  pic_param->pic_fields.bits.weighted_bipred_flag << 18 |
2404                  0 << 17 |
2405                  pic_param->pic_fields.bits.entropy_coding_sync_enabled_flag << 16 |
2406                  0 << 15 |
2407                  pic_param->pic_fields.bits.sign_data_hiding_enabled_flag << 13 |
2408                  pic_param->log2_parallel_merge_level_minus2 << 10 |
2409                  pic_param->pic_fields.bits.constrained_intra_pred_flag << 9 |
2410                  seq_param->seq_fields.bits.pcm_loop_filter_disabled_flag << 8 |
2411                  (pic_param->diff_cu_qp_delta_depth & 0x03) << 6 |
2412                  pic_param->pic_fields.bits.cu_qp_delta_enabled_flag << 5 |
2413                  0 << 4 |
2414                  seq_param->seq_fields.bits.sample_adaptive_offset_enabled_flag << 3 |
2415                  0;
2416     *cmd_ptr++ = seq_param->seq_fields.bits.bit_depth_luma_minus8 << 27 |
2417                  seq_param->seq_fields.bits.bit_depth_chroma_minus8 << 24 |
2418                  ((IS_KBL(i965->intel.device_info) || IS_GLK(i965->intel.device_info)) ? 0 : 7) << 20 |
2419                  ((IS_KBL(i965->intel.device_info) || IS_GLK(i965->intel.device_info)) ? 0 : 7) << 16 |
2420                  seq_param->max_transform_hierarchy_depth_inter << 13 |
2421                  seq_param->max_transform_hierarchy_depth_intra << 10 |
2422                  (pic_param->pps_cr_qp_offset & 0x1f) << 5 |
2423                  (pic_param->pps_cb_qp_offset & 0x1f);
2424
2425     *cmd_ptr++ = 0 << 29 |
2426                  priv_state->ctu_max_bitsize_allowed;
2427     if (brc_update)
2428         *(cmd_ptr - 1) |= 0 << 31 |
2429                           1 << 26 |
2430                           1 << 25 |
2431                           0 << 24 |
2432                           (pic_state_offset ? 1 : 0) << 16;
2433
2434     *cmd_ptr++ = 0 << 31 |
2435                  0;
2436     *cmd_ptr++ = 0 << 31 |
2437                  0;
2438     *cmd_ptr++ = 0 << 16 |
2439                  0;
2440     *cmd_ptr++ = 0;
2441     *cmd_ptr++ = 0;
2442     *cmd_ptr++ = 0;
2443     *cmd_ptr++ = 0;
2444     *cmd_ptr++ = 0;
2445     *cmd_ptr++ = 0;
2446     *cmd_ptr++ = 0;
2447     *cmd_ptr++ = 0;
2448     *cmd_ptr++ = 0 << 30 |
2449                  0;
2450
2451     if (IS_KBL(i965->intel.device_info) || IS_GLK(i965->intel.device_info)) {
2452         int i = 0;
2453
2454         for (i = 0; i < 12; i++)
2455             *cmd_ptr++ = 0;
2456     }
2457
2458     if (pic_state_ptr) {
2459         char *pdata = i965_map_gpe_resource(pic_state_ptr);
2460
2461         if (!pdata)
2462             return;
2463
2464         memcpy(pdata + pic_state_offset, tmp_data, cmd_size * 4);
2465
2466         pdata += pic_state_offset + cmd_size * 4;
2467
2468         *(unsigned int *)pdata++ = MI_BATCH_BUFFER_END;
2469
2470         i965_unmap_gpe_resource(pic_state_ptr);
2471     } else {
2472         struct intel_batchbuffer *batch = encoder_context->base.batch;
2473
2474         BEGIN_BCS_BATCH(batch, cmd_size);
2475
2476         intel_batchbuffer_data(batch, tmp_data, cmd_size * 4);
2477
2478         ADVANCE_BCS_BATCH(batch);
2479     }
2480 }
2481
2482 static void
2483 gen9_hevc_brc_update_set_pic_states(VADriverContextP ctx,
2484                                     struct encode_state *encode_state,
2485                                     struct intel_encoder_context *encoder_context)
2486 {
2487     struct encoder_vme_mfc_context *vme_context = NULL;
2488     struct gen9_hevc_encoder_context *priv_ctx = NULL;
2489     struct generic_enc_codec_state *generic_state = NULL;
2490     struct gen9_hevc_encoder_state *priv_state = NULL;
2491     int i = 0, offset = 0;
2492
2493     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
2494     priv_ctx = (struct gen9_hevc_encoder_context *)vme_context->private_enc_ctx;
2495     generic_state = (struct generic_enc_codec_state *)vme_context->generic_enc_state;
2496     priv_state = (struct gen9_hevc_encoder_state *)vme_context->private_enc_state;
2497
2498     for (i = 0; i < generic_state->num_pak_passes; i++) {
2499         gen9_hevc_add_pic_state(ctx, encode_state, encoder_context,
2500                                 &priv_ctx->res_brc_pic_states_read_buffer,
2501                                 offset, 1);
2502
2503         offset += priv_state->pic_state_size;
2504     }
2505 }
2506
2507 static void
2508 gen9_hevc_brc_update_set_constant(VADriverContextP ctx,
2509                                   struct encode_state *encode_state,
2510                                   struct intel_encoder_context *encoder_context)
2511 {
2512     struct encoder_vme_mfc_context *vme_context = NULL;
2513     struct gen9_hevc_encoder_context *priv_ctx = NULL;
2514     struct gen9_hevc_encoder_state *priv_state = NULL;
2515     unsigned int width, height, size;
2516     unsigned char *pdata = NULL;
2517     int idx = 0;
2518
2519     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
2520     priv_ctx = (struct gen9_hevc_encoder_context *)vme_context->private_enc_ctx;
2521     priv_state = (struct gen9_hevc_encoder_state *)vme_context->private_enc_state;
2522
2523     pdata = i965_map_gpe_resource(&priv_ctx->res_brc_constant_data_buffer);
2524     if (!pdata)
2525         return;
2526
2527     width = ALIGN(GEN9_HEVC_ENC_BRC_CONSTANT_SURFACE_WIDTH, 64);
2528     height = GEN9_HEVC_ENC_BRC_CONSTANT_SURFACE_HEIGHT;
2529     size = width * height;
2530     memset((void *)pdata, 0, size);
2531
2532     memcpy((void *)pdata, GEN9_HEVC_BRCUPDATE_QP_ADJUST, GEN9_HEVC_BRCUPDATE_QP_ADJUST_SIZE);
2533     pdata += GEN9_HEVC_BRCUPDATE_QP_ADJUST_SIZE;
2534
2535     if (priv_state->picture_coding_type == HEVC_SLICE_I)
2536         memset((void *)pdata, 0, GEN9_HEVC_ENC_SKIP_VAL_SIZE);
2537     else {
2538         gen9_hevc_mbenc_b_mb_enc_curbe_data *curbe_cmd = NULL;
2539         int curbe_size = 0;
2540
2541         gen9_hevc_get_b_mbenc_default_curbe(priv_state->tu_mode,
2542                                             priv_state->picture_coding_type,
2543                                             (void **)&curbe_cmd,
2544                                             &curbe_size);
2545
2546         idx = curbe_cmd->dw3.block_based_skip_enable ? 1 : 0;
2547         memcpy((void *)pdata, GEN9_HEVC_ENC_SKIP_THREAD[idx], sizeof(GEN9_HEVC_ENC_SKIP_THREAD[idx]));
2548     }
2549     pdata += GEN9_HEVC_ENC_SKIP_VAL_SIZE;
2550
2551     memcpy((void *)pdata, GEN9_HEVC_ENC_BRC_LAMBDA_HAAR, sizeof(GEN9_HEVC_ENC_BRC_LAMBDA_HAAR));
2552     pdata += GEN9_HEVC_ENC_BRC_LAMBDA_TABLE_SIZE;
2553
2554     idx = (priv_state->picture_coding_type == HEVC_SLICE_I) ? 0 :
2555           (priv_state->picture_coding_type == HEVC_SLICE_P) ? 1 : 2;
2556     memcpy((void *)pdata, GEN9_HEVC_ENC_BRC_MVCOST_HAAR[idx], sizeof(GEN9_HEVC_ENC_BRC_MVCOST_HAAR[idx]));
2557
2558     i965_unmap_gpe_resource(&priv_ctx->res_brc_constant_data_buffer);
2559 }
2560
2561 static
2562 unsigned int gen9_hevc_get_start_code_offset(unsigned char *ptr,
2563                                              unsigned int size)
2564 {
2565     unsigned int count = 0;
2566
2567     while (count < size && *ptr != 0x01) {
2568         if (*ptr != 0)
2569             break;
2570
2571         count++;
2572         ptr++;
2573     }
2574
2575     return count + 1;
2576 }
2577
2578 static
2579 unsigned int gen9_hevc_get_emulation_num(unsigned char *ptr,
2580                                          unsigned int size)
2581 {
2582     unsigned int emulation_num = 0;
2583     unsigned int header_offset = 0;
2584     unsigned int zero_count = 0;
2585     int i = 0;
2586
2587     header_offset = gen9_hevc_get_start_code_offset(ptr, size);
2588     ptr += header_offset;
2589
2590     for (i = 0 ; i < (size - header_offset); i++, ptr++) {
2591         if (zero_count == 2 && !(*ptr & 0xFC)) {
2592             zero_count = 0;
2593             emulation_num++;
2594         }
2595
2596         if (*ptr == 0x00)
2597             zero_count++;
2598         else
2599             zero_count = 0;
2600     }
2601
2602     return emulation_num;
2603 }
2604
2605 #define HEVC_ENC_START_CODE_NAL_OFFSET                  (2)
2606
2607 static unsigned int
2608 gen9_hevc_get_pic_header_size(struct encode_state *encode_state)
2609 {
2610     VAEncPackedHeaderParameterBuffer *param = NULL;
2611     unsigned int header_begin = 0;
2612     unsigned int accum_size = 0;
2613     unsigned char *header_data = NULL;
2614     unsigned int length_in_bytes = 0;
2615     int packed_type = 0;
2616     int idx = 0, count = 0, idx_offset = 0;
2617     int i = 0, slice_idx = 0, start_index = 0;
2618
2619     for (i = 0; i < 4; i++) {
2620         idx_offset = 0;
2621         switch (i) {
2622         case 0:
2623             packed_type = VAEncPackedHeaderHEVC_VPS;
2624             break;
2625         case 1:
2626             packed_type = VAEncPackedHeaderHEVC_VPS;
2627             idx_offset = 1;
2628             break;
2629         case 2:
2630             packed_type = VAEncPackedHeaderHEVC_PPS;
2631             break;
2632         case 3:
2633             packed_type = VAEncPackedHeaderHEVC_SEI;
2634             break;
2635         default:
2636             break;
2637         }
2638
2639         idx = va_enc_packed_type_to_idx(packed_type) + idx_offset;
2640         if (encode_state->packed_header_data[idx]) {
2641             param = (VAEncPackedHeaderParameterBuffer *)encode_state->packed_header_param[idx]->buffer;
2642             header_data = (unsigned char *)encode_state->packed_header_data[idx]->buffer;
2643             length_in_bytes = (param->bit_length + 7) / 8;
2644
2645             header_begin = gen9_hevc_get_start_code_offset(header_data, length_in_bytes) +
2646                            HEVC_ENC_START_CODE_NAL_OFFSET;
2647
2648             accum_size += length_in_bytes;
2649             if (!param->has_emulation_bytes)
2650                 accum_size += gen9_hevc_get_emulation_num(header_data,
2651                                                           length_in_bytes);
2652         }
2653     }
2654
2655     for (slice_idx = 0; slice_idx < encode_state->num_slice_params_ext; slice_idx++) {
2656         count = encode_state->slice_rawdata_count[slice_idx];
2657         start_index = encode_state->slice_rawdata_index[slice_idx] &
2658                       SLICE_PACKED_DATA_INDEX_MASK;
2659
2660         for (i = 0; i < count; i++) {
2661             param = (VAEncPackedHeaderParameterBuffer *)
2662                     (encode_state->packed_header_params_ext[start_index + i]->buffer);
2663
2664             if (param->type == VAEncPackedHeaderSlice)
2665                 continue;
2666
2667             header_data = (unsigned char *)encode_state->packed_header_data[start_index]->buffer;
2668             length_in_bytes = (param->bit_length + 7) / 8;
2669
2670             accum_size += length_in_bytes;
2671             if (!param->has_emulation_bytes)
2672                 accum_size += gen9_hevc_get_emulation_num(header_data,
2673                                                           length_in_bytes);
2674         }
2675     }
2676
2677     header_begin = MIN(header_begin, accum_size);
2678
2679     return ((accum_size - header_begin) * 8);
2680 }
2681
2682 static void
2683 gen9_hevc_brc_update_set_curbe(VADriverContextP ctx,
2684                                struct encode_state *encode_state,
2685                                struct intel_encoder_context *encoder_context,
2686                                struct i965_gpe_context *gpe_context)
2687 {
2688     struct encoder_vme_mfc_context *vme_context = NULL;
2689     struct generic_enc_codec_state *generic_state = NULL;
2690     struct gen9_hevc_encoder_state *priv_state = NULL;
2691     gen9_hevc_brc_udpate_curbe_data *cmd = NULL;
2692
2693     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
2694     generic_state = (struct generic_enc_codec_state *)vme_context->generic_enc_state;
2695     priv_state = (struct gen9_hevc_encoder_state *)vme_context->private_enc_state;
2696
2697     cmd = i965_gpe_context_map_curbe(gpe_context);
2698     if (!cmd)
2699         return;
2700
2701     memcpy((void *)cmd, GEN9_HEVC_BRCUPDATE_CURBE_DATA,
2702            sizeof(GEN9_HEVC_BRCUPDATE_CURBE_DATA));
2703
2704     cmd->dw5.target_size_flag = 0;
2705     if (priv_state->brc_init_current_target_buf_full_in_bits >
2706         (double)priv_state->brc_init_reset_buf_size_in_bits) {
2707         priv_state->brc_init_current_target_buf_full_in_bits -=
2708             (double)priv_state->brc_init_reset_buf_size_in_bits;
2709         cmd->dw5.target_size_flag = 1;
2710     }
2711
2712     if (priv_state->num_skip_frames) {
2713         cmd->dw6.num_skipped_frames = priv_state->num_skip_frames;
2714         cmd->dw15.size_of_skipped_frames = priv_state->size_skip_frames;
2715
2716         priv_state->brc_init_current_target_buf_full_in_bits +=
2717             priv_state->brc_init_reset_input_bits_per_frame * priv_state->num_skip_frames;
2718     }
2719
2720     cmd->dw0.target_size = (unsigned int)priv_state->brc_init_current_target_buf_full_in_bits;
2721     cmd->dw1.frame_number = priv_state->frame_number;
2722     cmd->dw2.picture_header_size = gen9_hevc_get_pic_header_size(encode_state);
2723
2724     cmd->dw5.brc_flag = 0;
2725     cmd->dw5.curr_frame_type = gen9_hevc_get_brc_frame_type(priv_state->picture_coding_type,
2726                                                             priv_state->low_delay);
2727
2728     cmd->dw5.max_num_paks = generic_state->num_pak_passes;
2729     cmd->dw14.parallel_mode = priv_state->parallel_brc;
2730
2731     priv_state->brc_init_current_target_buf_full_in_bits +=
2732         priv_state->brc_init_reset_input_bits_per_frame;
2733
2734     if (priv_state->brc_method == HEVC_BRC_AVBR) {
2735         cmd->dw3.start_gadj_frame0 = (unsigned int)((10 * GEN9_HEVC_AVBR_CONVERGENCE) / (double)150);
2736         cmd->dw3.start_gadj_frame1 = (unsigned int)((50 * GEN9_HEVC_AVBR_CONVERGENCE) / (double)150);
2737         cmd->dw4.start_gadj_frame2 = (unsigned int)((100 * GEN9_HEVC_AVBR_CONVERGENCE) / (double)150);
2738         cmd->dw4.start_gadj_frame3 = (unsigned int)((150 * GEN9_HEVC_AVBR_CONVERGENCE) / (double)150);
2739         cmd->dw11.g_rate_ratio_threshold0 = (unsigned int)((100 - (GEN9_HEVC_AVBR_ACCURACY / (double)30) * (100 - 40)));
2740         cmd->dw11.g_rate_ratio_threshold1 = (unsigned int)((100 - (GEN9_HEVC_AVBR_ACCURACY / (double)30) * (100 - 75)));
2741         cmd->dw12.g_rate_ratio_threshold2 = (unsigned int)((100 - (GEN9_HEVC_AVBR_ACCURACY / (double)30) * (100 - 97)));
2742         cmd->dw12.g_rate_ratio_threshold3 = (unsigned int)((100 + (GEN9_HEVC_AVBR_ACCURACY / (double)30) * (103 - 100)));
2743         cmd->dw12.g_rate_ratio_threshold4 = (unsigned int)((100 + (GEN9_HEVC_AVBR_ACCURACY / (double)30) * (125 - 100)));
2744         cmd->dw12.g_rate_ratio_threshold5 = (unsigned int)((100 + (GEN9_HEVC_AVBR_ACCURACY / (double)30) * (160 - 100)));
2745     } else {
2746         cmd->dw3.start_gadj_frame0 = 10;
2747         cmd->dw3.start_gadj_frame1 = 50;
2748         cmd->dw4.start_gadj_frame2 = 100;
2749         cmd->dw4.start_gadj_frame3 = 150;
2750         cmd->dw11.g_rate_ratio_threshold0 = 40;
2751         cmd->dw11.g_rate_ratio_threshold1 = 75;
2752         cmd->dw12.g_rate_ratio_threshold2 = 97;
2753         cmd->dw12.g_rate_ratio_threshold3 = 103;
2754         cmd->dw12.g_rate_ratio_threshold4 = 125;
2755         cmd->dw12.g_rate_ratio_threshold5 = 160;
2756     }
2757
2758     i965_gpe_context_unmap_curbe(gpe_context);
2759 }
2760
2761 static void
2762 gen9_hevc_brc_update_set_surfaces(VADriverContextP ctx,
2763                                   struct intel_encoder_context *encoder_context,
2764                                   struct i965_gpe_context *gpe_context)
2765 {
2766     struct encoder_vme_mfc_context *vme_context = NULL;
2767     struct gen9_hevc_encoder_context *priv_ctx = NULL;
2768     struct gen9_hevc_encoder_state *priv_state = NULL;
2769     int bti_idx = 0;
2770
2771     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
2772     priv_ctx = (struct gen9_hevc_encoder_context *)vme_context->private_enc_ctx;
2773     priv_state = (struct gen9_hevc_encoder_state *)vme_context->private_enc_state;
2774
2775     gen9_hevc_set_gpe_1d_surface(ctx, priv_ctx, gpe_context,
2776                                  HEVC_ENC_SURFACE_BRC_HISTORY, bti_idx++,
2777                                  0, 0, 0, NULL, NULL);
2778
2779     gen9_hevc_set_gpe_1d_surface(ctx, priv_ctx, gpe_context,
2780                                  HEVC_ENC_SURFACE_BRC_PAST_PAK_INFO, bti_idx++,
2781                                  0, 0, 0, NULL, NULL);
2782
2783     gen9_hevc_set_gpe_1d_surface(ctx, priv_ctx, gpe_context,
2784                                  HEVC_ENC_SURFACE_BRC_HCP_PIC_STATE, bti_idx++,
2785                                  0, 0, 0, &priv_ctx->res_brc_pic_states_read_buffer,
2786                                  NULL);
2787
2788     gen9_hevc_set_gpe_1d_surface(ctx, priv_ctx, gpe_context,
2789                                  HEVC_ENC_SURFACE_BRC_HCP_PIC_STATE, bti_idx++,
2790                                  0, 0, 0, &priv_ctx->res_brc_pic_states_write_buffer,
2791                                  NULL);
2792
2793     gen9_hevc_set_gpe_1d_surface(ctx, priv_ctx, gpe_context,
2794                                  HEVC_ENC_SURFACE_BRC_INPUT, bti_idx++,
2795                                  0, 0, 0, NULL, NULL);
2796
2797     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
2798                                  HEVC_ENC_SURFACE_BRC_ME_DIST, bti_idx++,
2799                                  0, 1, I965_SURFACEFORMAT_R8_UNORM,
2800                                  priv_state->picture_coding_type == HEVC_SLICE_I ?
2801                                  &priv_ctx->res_brc_intra_dist_buffer :
2802                                  &priv_ctx->res_brc_me_dist_buffer,
2803                                  NULL);
2804
2805     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
2806                                  HEVC_ENC_SURFACE_BRC_DATA, bti_idx++,
2807                                  0, 1, I965_SURFACEFORMAT_R8_UNORM,
2808                                  NULL, NULL);
2809 }
2810
2811 static void
2812 gen9_hevc_brc_update(VADriverContextP ctx,
2813                      struct encode_state *encode_state,
2814                      struct intel_encoder_context *encoder_context)
2815 {
2816     struct encoder_vme_mfc_context *vme_context = NULL;
2817     struct gen9_hevc_encoder_context *priv_ctx = NULL;
2818     struct gen9_hevc_encoder_state *priv_state = NULL;
2819     struct i965_gpe_context *gpe_context = NULL;
2820     struct gpe_media_object_parameter param;
2821     int media_state = HEVC_ENC_MEDIA_STATE_BRC_UPDATE;
2822     int gpe_idx = HEVC_BRC_FRAME_UPDATE_IDX;
2823
2824     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
2825     priv_ctx = (struct gen9_hevc_encoder_context *)vme_context->private_enc_ctx;
2826     priv_state = (struct gen9_hevc_encoder_state *)vme_context->private_enc_state;
2827     gpe_context = &priv_ctx->brc_context.gpe_contexts[gpe_idx];
2828
2829     gen8_gpe_context_init(ctx, gpe_context);
2830     gen9_gpe_reset_binding_table(ctx, gpe_context);
2831
2832     gen9_hevc_brc_update_set_pic_states(ctx, encode_state, encoder_context);
2833     gen9_hevc_brc_update_set_constant(ctx, encode_state, encoder_context);
2834     gen9_hevc_brc_update_set_curbe(ctx, encode_state, encoder_context, gpe_context);
2835     gen9_hevc_brc_update_set_surfaces(ctx, encoder_context, gpe_context);
2836
2837     gen8_gpe_setup_interface_data(ctx, gpe_context);
2838
2839     memset(&param, 0, sizeof(param));
2840     gen9_hevc_run_object(ctx, encoder_context, gpe_context, &param,
2841                          media_state);
2842
2843     if (priv_state->lcu_brc_enabled ||
2844         priv_state->num_roi)
2845         gen9_hevc_brc_update_lcu_based(ctx, encode_state, encoder_context);
2846 }
2847
2848 // Depth converstion for 10bits
2849
2850 static void
2851 gen9_hevc_frame_depth_conversion_set_curbe(VADriverContextP ctx,
2852                                            struct encode_state *encode_state,
2853                                            struct intel_encoder_context *encoder_context,
2854                                            struct i965_gpe_context *gpe_context,
2855                                            GEN9_HEVC_DOWNSCALE_STAGE scale_stage)
2856 {
2857     struct encoder_vme_mfc_context *vme_context = NULL;
2858     struct gen9_hevc_encoder_state *priv_state = NULL;
2859     gen95_hevc_mbenc_ds_combined_curbe_data *cmd = NULL;
2860     int bti_idx = 0;
2861
2862     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
2863     priv_state = (struct gen9_hevc_encoder_state *)vme_context->private_enc_state;
2864
2865     cmd = i965_gpe_context_map_curbe(gpe_context);
2866     if (!cmd)
2867         return;
2868
2869     memset((void *)cmd, 0, sizeof(*cmd));
2870
2871     cmd->dw0.pak_bitdepth_chroma = 10;
2872     cmd->dw0.pak_bitdepth_luma = 10;
2873     cmd->dw0.enc_bitdepth_chroma = 8;
2874     cmd->dw0.enc_bitdepth_luma = 8;
2875     cmd->dw0.rounding_value = 1;
2876     cmd->dw1.pic_format = 0;
2877     cmd->dw1.pic_convert_flag = 1;
2878     cmd->dw1.pic_down_scale = scale_stage;
2879     cmd->dw1.pic_mb_stat_output_cntrl = 0;
2880     cmd->dw2.orig_pic_width = priv_state->picture_width;
2881     cmd->dw2.orig_pic_height = priv_state->picture_height;
2882
2883     cmd->dw3.bti_surface_p010 = bti_idx++;
2884     bti_idx++;
2885     cmd->dw4.bti_surface_nv12 = bti_idx++;
2886     bti_idx++;
2887     cmd->dw5.bti_src_y_4xdownscaled = bti_idx++;
2888     cmd->dw6.bti_surf_mbstate = bti_idx++;
2889     cmd->dw7.bit_src_y_2xdownscaled = bti_idx++;
2890
2891     i965_gpe_context_unmap_curbe(gpe_context);
2892 }
2893
2894 static void
2895 gen9_hevc_frame_depth_conversion_set_surfaces(VADriverContextP ctx,
2896                                               struct encode_state *encode_state,
2897                                               struct intel_encoder_context *encoder_context,
2898                                               struct i965_gpe_context *gpe_context,
2899                                               struct object_surface *src_surface,
2900                                               struct object_surface *dst_surface)
2901 {
2902     struct encoder_vme_mfc_context *vme_context = NULL;
2903     struct gen9_hevc_encoder_context *priv_ctx = NULL;
2904     struct gen9_hevc_surface_priv *surface_priv = NULL;
2905     int bti_idx = 0;
2906
2907     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
2908     priv_ctx = (struct gen9_hevc_encoder_context *)vme_context->private_enc_ctx;
2909     surface_priv = (struct gen9_hevc_surface_priv *)dst_surface->private_data;
2910
2911     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
2912                                  HEVC_ENC_SURFACE_RAW_10bit_Y_UV, bti_idx++,
2913                                  1, 1, I965_SURFACEFORMAT_R8_UNORM,
2914                                  NULL, src_surface);
2915     bti_idx++;
2916
2917     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
2918                                  HEVC_ENC_SURFACE_RAW_FC_8bit_Y_UV, bti_idx++,
2919                                  1, 1, I965_SURFACEFORMAT_R8_UNORM,
2920                                  NULL,
2921                                  surface_priv->surface_obj_nv12);
2922     bti_idx++;
2923
2924     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
2925                                  HEVC_ENC_SURFACE_Y_4X, bti_idx++,
2926                                  0, 1, I965_SURFACEFORMAT_R32_UNORM,
2927                                  NULL,
2928                                  surface_priv->scaled_surface_obj[HEVC_SCALED_SURF_4X_ID]);
2929
2930     gen9_hevc_set_gpe_1d_surface(ctx, priv_ctx, gpe_context,
2931                                  HEVC_ENC_SURFACE_RAW_MBSTAT, bti_idx++,
2932                                  0, 0, 0, NULL, NULL);
2933
2934     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
2935                                  HEVC_ENC_SURFACE_Y_2X, bti_idx++,
2936                                  0, 1, I965_SURFACEFORMAT_R32_UNORM,
2937                                  NULL, NULL);
2938 }
2939
2940 static void
2941 gen9_hevc_frame_depth_conversion(VADriverContextP ctx,
2942                                  struct encode_state *encode_state,
2943                                  struct intel_encoder_context *encoder_context,
2944                                  struct object_surface *src_surface,
2945                                  struct object_surface *dst_surface,
2946                                  GEN9_HEVC_DOWNSCALE_STAGE scale_stage)
2947 {
2948     struct encoder_vme_mfc_context *vme_context = NULL;
2949     struct gen9_hevc_encoder_context *priv_ctx = NULL;
2950     struct gen9_hevc_encoder_state *priv_state = NULL;
2951     struct gpe_media_object_walker_parameter param;
2952     struct hevc_enc_kernel_walker_parameter hevc_walker_param;
2953     struct i965_gpe_context *gpe_context = NULL;
2954     int media_state = HEVC_ENC_MEDIA_STATE_DS_COMBINED;
2955
2956     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
2957     priv_ctx = (struct gen9_hevc_encoder_context *)vme_context->private_enc_ctx;
2958     priv_state = (struct gen9_hevc_encoder_state *)vme_context->private_enc_state;
2959
2960     gpe_context = &priv_ctx->mbenc_context.gpe_contexts[HEVC_MBENC_DS_COMBINED_IDX];
2961
2962     gen8_gpe_context_init(ctx, gpe_context);
2963     gen9_gpe_reset_binding_table(ctx, gpe_context);
2964     gen9_hevc_frame_depth_conversion_set_curbe(ctx, encode_state, encoder_context, gpe_context,
2965                                                scale_stage);
2966     gen9_hevc_frame_depth_conversion_set_surfaces(ctx, encode_state, encoder_context, gpe_context,
2967                                                   src_surface, dst_surface);
2968     gen8_gpe_setup_interface_data(ctx, gpe_context);
2969
2970     memset((void *)&hevc_walker_param, 0, sizeof(hevc_walker_param));
2971     hevc_walker_param.resolution_x = ALIGN(priv_state->picture_width >> 2, 32) >> 3;
2972     hevc_walker_param.resolution_y = ALIGN(priv_state->picture_height >> 2, 32) >> 3;
2973     hevc_walker_param.no_dependency = 1;
2974     gen9_hevc_init_object_walker(&hevc_walker_param, &param);
2975     gen9_hevc_run_object_walker(ctx, encoder_context, gpe_context, &param,
2976                                 media_state);
2977 }
2978
2979 static void
2980 gen9_hevc_ref_frame_depth_conversion(VADriverContextP ctx,
2981                                      struct encode_state *encode_state,
2982                                      struct intel_encoder_context *encoder_context)
2983 {
2984     struct i965_driver_data *i965 = i965_driver_data(ctx);
2985     struct gen9_hevc_surface_priv *surface_priv = NULL;
2986     VAEncSliceParameterBufferHEVC *slice_param = NULL;
2987     struct object_surface *obj_surface = NULL;
2988     int i = 0;
2989
2990     slice_param = (VAEncSliceParameterBufferHEVC *)encode_state->slice_params_ext[0]->buffer;
2991
2992     for (i = 0; i < slice_param->num_ref_idx_l0_active_minus1 + 1; i++) {
2993         obj_surface = SURFACE(slice_param->ref_pic_list0[i].picture_id);
2994         if (obj_surface) {
2995             surface_priv = (struct gen9_hevc_surface_priv *)obj_surface->private_data;
2996
2997             if (!surface_priv->surface_nv12_valid) {
2998                 gen9_hevc_frame_depth_conversion(ctx, encode_state, encoder_context,
2999                                                  obj_surface, obj_surface,
3000                                                  HEVC_ENC_DS_DISABLED);
3001
3002                 surface_priv->surface_reff = surface_priv->surface_obj_nv12;
3003                 surface_priv->surface_nv12_valid = 1;
3004             }
3005         }
3006     }
3007
3008     for (i = 0; i < slice_param->num_ref_idx_l1_active_minus1 + 1; i++) {
3009         obj_surface = SURFACE(slice_param->ref_pic_list1[i].picture_id);
3010         if (obj_surface) {
3011             surface_priv = (struct gen9_hevc_surface_priv *)obj_surface->private_data;
3012
3013             if (!surface_priv->surface_nv12_valid) {
3014                 gen9_hevc_frame_depth_conversion(ctx, encode_state, encoder_context,
3015                                                  obj_surface, obj_surface,
3016                                                  HEVC_ENC_DS_DISABLED);
3017
3018                 surface_priv->surface_reff = surface_priv->surface_obj_nv12;
3019                 surface_priv->surface_nv12_valid = 1;
3020             }
3021         }
3022     }
3023 }
3024
3025 // Scaling implementation
3026
3027 static void
3028 gen9_hevc_scaling_set_curbe_2x(struct i965_gpe_context *gpe_context,
3029                                struct gen9_hevc_scaling_parameter *scaling_param)
3030 {
3031     gen9_hevc_scaling2x_curbe_data *cmd;
3032
3033     cmd = i965_gpe_context_map_curbe(gpe_context);
3034     if (!cmd)
3035         return;
3036
3037     memset((void *)cmd, 0, sizeof(*cmd));
3038
3039     cmd->dw0.input_picture_width  = scaling_param->input_frame_width;
3040     cmd->dw0.input_picture_height = scaling_param->input_frame_height;
3041
3042     cmd->dw8.input_y_bti = GEN9_HEVC_SCALING_FRAME_SRC_Y_INDEX;
3043     cmd->dw9.output_y_bti = GEN9_HEVC_SCALING_FRAME_DST_Y_INDEX;
3044
3045     i965_gpe_context_unmap_curbe(gpe_context);
3046 }
3047
3048 static void
3049 gen9_hevc_scaling_set_curbe_4x(struct i965_gpe_context *gpe_context,
3050                                struct gen9_hevc_scaling_parameter *scaling_param)
3051 {
3052     gen9_hevc_scaling4x_curbe_data *cmd;
3053
3054     cmd = i965_gpe_context_map_curbe(gpe_context);
3055     if (!cmd)
3056         return;
3057
3058     memset((void *)cmd, 0, sizeof(*cmd));
3059
3060     cmd->dw0.input_picture_width  = scaling_param->input_frame_width;
3061     cmd->dw0.input_picture_height = scaling_param->input_frame_height;
3062
3063     cmd->dw1.input_y_bti = GEN9_HEVC_SCALING_FRAME_SRC_Y_INDEX;
3064     cmd->dw2.output_y_bti = GEN9_HEVC_SCALING_FRAME_DST_Y_INDEX;
3065
3066     cmd->dw5.flatness_threshold = 0;
3067     cmd->dw6.enable_mb_flatness_check = scaling_param->enable_mb_flatness_check;
3068     cmd->dw7.enable_mb_variance_output = scaling_param->enable_mb_variance_output;
3069     cmd->dw8.enable_mb_pixel_average_output = scaling_param->enable_mb_pixel_average_output;
3070
3071     if (cmd->dw6.enable_mb_flatness_check ||
3072         cmd->dw7.enable_mb_variance_output ||
3073         cmd->dw8.enable_mb_pixel_average_output)
3074         cmd->dw10.mbv_proc_stat_bti = GEN9_HEVC_SCALING_FRAME_MBVPROCSTATS_DST_INDEX;
3075
3076     i965_gpe_context_unmap_curbe(gpe_context);
3077 }
3078
3079 static void
3080 gen9_hevc_scaling_set_curbe(struct gen9_hevc_scaling_parameter *scaling_param,
3081                             struct i965_gpe_context *gpe_context)
3082 {
3083     if (scaling_param->use_32x_scaling)
3084         gen9_hevc_scaling_set_curbe_2x(gpe_context, scaling_param);
3085     else
3086         gen9_hevc_scaling_set_curbe_4x(gpe_context, scaling_param);
3087 }
3088
3089 static void
3090 gen9_hevc_scaling_set_surfaces(VADriverContextP ctx,
3091                                struct encode_state *encode_state,
3092                                struct intel_encoder_context *encoder_context,
3093                                struct gen9_hevc_scaling_parameter *scaling_param,
3094                                struct i965_gpe_context *gpe_context)
3095 {
3096     unsigned int surface_format;
3097
3098     if (scaling_param->scaling_out_use_32unorm_surf_fmt)
3099         surface_format = I965_SURFACEFORMAT_R32_UNORM;
3100     else if (scaling_param->scaling_out_use_16unorm_surf_fmt)
3101         surface_format = I965_SURFACEFORMAT_R16_UNORM;
3102     else
3103         surface_format = I965_SURFACEFORMAT_R8_UNORM;
3104
3105     i965_add_2d_gpe_surface(ctx, gpe_context,
3106                             scaling_param->input_surface,
3107                             0, 1, surface_format,
3108                             GEN9_HEVC_SCALING_FRAME_SRC_Y_INDEX);
3109
3110     i965_add_2d_gpe_surface(ctx, gpe_context,
3111                             scaling_param->output_surface,
3112                             0, 1, surface_format,
3113                             GEN9_HEVC_SCALING_FRAME_DST_Y_INDEX);
3114
3115     if ((scaling_param->enable_mb_flatness_check ||
3116          scaling_param->enable_mb_variance_output ||
3117          scaling_param->enable_mb_pixel_average_output) &&
3118         scaling_param->use_4x_scaling) {
3119         i965_add_buffer_2d_gpe_surface(ctx, gpe_context,
3120                                        scaling_param->pres_mbv_proc_stat_buffer,
3121                                        1,
3122                                        I965_SURFACEFORMAT_R8_UNORM,
3123                                        GEN9_HEVC_SCALING_FRAME_MBVPROCSTATS_DST_INDEX);
3124     }
3125 }
3126
3127 static void
3128 gen9_hevc_kernel_scaling(VADriverContextP ctx,
3129                          struct encode_state *encode_state,
3130                          struct intel_encoder_context *encoder_context,
3131                          enum HEVC_HME_TYPE hme_type)
3132 {
3133     struct encoder_vme_mfc_context *vme_context = NULL;
3134     struct gen9_hevc_encoder_context *priv_ctx = NULL;
3135     struct gen9_hevc_encoder_state *priv_state = NULL;
3136     struct gen9_hevc_surface_priv *surface_priv = NULL;
3137     struct gen9_hevc_scaling_parameter scaling_param;
3138     struct gpe_media_object_walker_parameter param;
3139     unsigned int downscaled_width_in_mb, downscaled_height_in_mb;
3140     struct hevc_enc_kernel_walker_parameter hevc_walker_param;
3141     struct i965_gpe_context *gpe_context = NULL;
3142     int gpe_idx = 0, media_state = 0;;
3143
3144     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
3145     priv_ctx = (struct gen9_hevc_encoder_context *)vme_context->private_enc_ctx;
3146     priv_state = (struct gen9_hevc_encoder_state *)vme_context->private_enc_state;
3147
3148     surface_priv = (struct gen9_hevc_surface_priv *)encode_state->reconstructed_object->private_data;
3149
3150     memset((void *)&scaling_param, 0, sizeof(scaling_param));
3151     switch (hme_type) {
3152     case HEVC_HME_4X:
3153         media_state = HEVC_ENC_MEDIA_STATE_4X_SCALING;
3154         gpe_idx = HEVC_ENC_SCALING_4X;
3155         downscaled_width_in_mb = priv_state->downscaled_width_4x_in_mb;
3156         downscaled_height_in_mb = priv_state->downscaled_height_4x_in_mb;
3157
3158         scaling_param.input_surface = encode_state->input_yuv_object;
3159         scaling_param.input_frame_width = priv_state->picture_width;
3160         scaling_param.input_frame_height = priv_state->picture_height;
3161
3162         scaling_param.output_surface = surface_priv->scaled_surface_obj[HEVC_SCALED_SURF_4X_ID];
3163         scaling_param.output_frame_width = priv_state->frame_width_4x;
3164         scaling_param.output_frame_height = priv_state->frame_height_4x;
3165
3166         scaling_param.enable_mb_flatness_check = priv_state->flatness_check_enable;
3167         scaling_param.enable_mb_variance_output = 0;
3168         scaling_param.enable_mb_pixel_average_output = 0;
3169         scaling_param.pres_mbv_proc_stat_buffer = &(priv_ctx->res_flatness_check_surface);
3170
3171         scaling_param.blk8x8_stat_enabled = 0;
3172         scaling_param.use_4x_scaling  = 1;
3173         scaling_param.use_16x_scaling = 0;
3174         scaling_param.use_32x_scaling = 0;
3175         break;
3176     case HEVC_HME_16X:
3177         media_state = HEVC_ENC_MEDIA_STATE_16X_SCALING;
3178         gpe_idx = HEVC_ENC_SCALING_16X;
3179         downscaled_width_in_mb = priv_state->downscaled_width_16x_in_mb;
3180         downscaled_height_in_mb = priv_state->downscaled_height_16x_in_mb;
3181
3182         scaling_param.input_surface = surface_priv->scaled_surface_obj[HEVC_SCALED_SURF_4X_ID];
3183         scaling_param.input_frame_width = priv_state->frame_width_4x;
3184         scaling_param.input_frame_height = priv_state->frame_height_4x;
3185
3186         scaling_param.output_surface = surface_priv->scaled_surface_obj[HEVC_SCALED_SURF_16X_ID];
3187         scaling_param.output_frame_width = priv_state->frame_width_16x;
3188         scaling_param.output_frame_height = priv_state->frame_height_16x;
3189
3190         scaling_param.enable_mb_flatness_check = 0;
3191         scaling_param.enable_mb_variance_output = 0;
3192         scaling_param.enable_mb_pixel_average_output = 0;
3193
3194         scaling_param.blk8x8_stat_enabled = 0;
3195         scaling_param.use_4x_scaling  = 0;
3196         scaling_param.use_16x_scaling = 1;
3197         scaling_param.use_32x_scaling = 0;
3198         break;
3199     case HEVC_HME_32X:
3200         media_state = HEVC_ENC_MEDIA_STATE_32X_SCALING;
3201         gpe_idx = HEVC_ENC_SCALING_32X;
3202         downscaled_width_in_mb = priv_state->downscaled_width_32x_in_mb;
3203         downscaled_height_in_mb = priv_state->downscaled_height_32x_in_mb;
3204
3205         scaling_param.input_surface = surface_priv->scaled_surface_obj[HEVC_SCALED_SURF_16X_ID];
3206         scaling_param.input_frame_width = priv_state->frame_width_16x;
3207         scaling_param.input_frame_height = priv_state->frame_height_16x;
3208
3209         scaling_param.output_surface = surface_priv->scaled_surface_obj[HEVC_SCALED_SURF_32X_ID];
3210         scaling_param.output_frame_width = priv_state->frame_width_32x;
3211         scaling_param.output_frame_height = priv_state->frame_height_32x;
3212
3213         scaling_param.enable_mb_flatness_check = 0;
3214         scaling_param.enable_mb_variance_output = 0;
3215         scaling_param.enable_mb_pixel_average_output = 0;
3216
3217         scaling_param.blk8x8_stat_enabled = 0;
3218         scaling_param.use_4x_scaling  = 0;
3219         scaling_param.use_16x_scaling = 0;
3220         scaling_param.use_32x_scaling = 1;
3221         break;
3222     default:
3223         return;
3224     }
3225
3226     gpe_context = &priv_ctx->scaling_context.gpe_contexts[gpe_idx];
3227     gen8_gpe_context_init(ctx, gpe_context);
3228     gen9_gpe_reset_binding_table(ctx, gpe_context);
3229     gen9_hevc_scaling_set_curbe(&scaling_param, gpe_context);
3230
3231     if (hme_type == HEVC_HME_32X) {
3232         scaling_param.scaling_out_use_16unorm_surf_fmt = 1;
3233         scaling_param.scaling_out_use_32unorm_surf_fmt = 0;
3234     } else {
3235         scaling_param.scaling_out_use_16unorm_surf_fmt = 0;
3236         scaling_param.scaling_out_use_32unorm_surf_fmt = 1;
3237     }
3238
3239     gen9_hevc_scaling_set_surfaces(ctx, encode_state, encoder_context, &scaling_param,
3240                                    gpe_context);
3241     gen8_gpe_setup_interface_data(ctx, gpe_context);
3242
3243     memset((void *)&hevc_walker_param, 0, sizeof(hevc_walker_param));
3244     if (hme_type == HEVC_HME_32X) {
3245         hevc_walker_param.resolution_x = downscaled_width_in_mb;
3246         hevc_walker_param.resolution_y = downscaled_height_in_mb;
3247     } else {
3248         hevc_walker_param.resolution_x = downscaled_width_in_mb * 2;
3249         hevc_walker_param.resolution_y = downscaled_height_in_mb * 2;
3250     }
3251     hevc_walker_param.no_dependency = 1;
3252     gen9_hevc_init_object_walker(&hevc_walker_param, &param);
3253     gen9_hevc_run_object_walker(ctx, encoder_context, gpe_context, &param,
3254                                 media_state);
3255 }
3256
3257 static void
3258 gen9_hevc_hme_scaling(VADriverContextP ctx,
3259                       struct encode_state *encode_state,
3260                       struct intel_encoder_context *encoder_context)
3261 {
3262     struct encoder_vme_mfc_context *vme_context = NULL;
3263     struct generic_enc_codec_state *generic_state = NULL;
3264     struct gen9_hevc_encoder_state *priv_state = NULL;
3265
3266     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
3267     generic_state = (struct generic_enc_codec_state *)vme_context->generic_enc_state;
3268     priv_state = (struct gen9_hevc_encoder_state *)vme_context->private_enc_state;
3269
3270     if (priv_state->bit_depth_luma_minus8)
3271         gen9_hevc_frame_depth_conversion(ctx, encode_state, encoder_context,
3272                                          encode_state->input_yuv_object,
3273                                          encode_state->reconstructed_object,
3274                                          HEVC_ENC_2xDS_4xDS_STAGE);
3275     else
3276         gen9_hevc_kernel_scaling(ctx, encode_state, encoder_context, HEVC_HME_4X);
3277
3278     if (generic_state->b16xme_supported) {
3279         gen9_hevc_kernel_scaling(ctx, encode_state, encoder_context, HEVC_HME_16X);
3280
3281         if (generic_state->b32xme_supported)
3282             gen9_hevc_kernel_scaling(ctx, encode_state, encoder_context, HEVC_HME_32X);
3283     }
3284 }
3285
3286 // ME implementation
3287
3288 static void
3289 gen9_hevc_me_set_curbe(VADriverContextP ctx,
3290                        struct encode_state *encode_state,
3291                        struct intel_encoder_context *encoder_context,
3292                        enum HEVC_HME_TYPE hme_type,
3293                        struct i965_gpe_context *gpe_context)
3294 {
3295     struct encoder_vme_mfc_context *vme_context = NULL;
3296     struct generic_enc_codec_state *generic_state = NULL;
3297     struct gen9_hevc_encoder_state *priv_state = NULL;
3298     gen9_hevc_me_curbe_data *cmd = NULL;
3299     VAEncPictureParameterBufferHEVC *pic_param = NULL;
3300     VAEncSliceParameterBufferHEVC *slice_param = NULL;
3301     unsigned int use_mv_from_prev_step = 0;
3302     unsigned int write_distortions = 0;
3303     unsigned int slice_qp = 0;
3304     unsigned int me_method = 0;
3305     unsigned int mv_shift_factor = 0, prev_mv_read_pos_factor = 0;
3306     unsigned int downscaled_width_in_mb, downscaled_height_in_mb;
3307
3308     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
3309     generic_state = (struct generic_enc_codec_state *)vme_context->generic_enc_state;
3310     priv_state = (struct gen9_hevc_encoder_state *)vme_context->private_enc_state;
3311     pic_param = (VAEncPictureParameterBufferHEVC *)encode_state->pic_param_ext->buffer;
3312     slice_param = (VAEncSliceParameterBufferHEVC *)encode_state->slice_params_ext[0]->buffer;
3313
3314     switch (hme_type) {
3315     case HEVC_HME_4X :
3316         use_mv_from_prev_step = (generic_state->b16xme_enabled) ? 1 : 0;;
3317         write_distortions = 1;
3318         mv_shift_factor = 2;
3319         prev_mv_read_pos_factor = 0;
3320         downscaled_width_in_mb = ALIGN(priv_state->picture_width / 4, 16) / 16;
3321         downscaled_height_in_mb = ALIGN(priv_state->picture_height / 4, 16) / 16;
3322         break;
3323     case HEVC_HME_16X :
3324         use_mv_from_prev_step = (generic_state->b32xme_enabled) ? 1 : 0;
3325         write_distortions = 0;
3326         mv_shift_factor = 2;
3327         prev_mv_read_pos_factor = 1;
3328         downscaled_width_in_mb = ALIGN(priv_state->picture_width / 16, 16) / 16;
3329         downscaled_height_in_mb = ALIGN(priv_state->picture_height / 16, 16) / 16;
3330         break;
3331     case HEVC_HME_32X :
3332         use_mv_from_prev_step = 0;
3333         write_distortions = 0;
3334         mv_shift_factor = 1;
3335         prev_mv_read_pos_factor = 0;
3336         downscaled_width_in_mb = ALIGN(priv_state->picture_width / 32, 16) / 16;
3337         downscaled_height_in_mb = ALIGN(priv_state->picture_height / 32, 16) / 16;
3338         break;
3339     default:
3340         return;
3341     }
3342
3343     me_method = GEN9_HEVC_ME_METHOD[priv_state->tu_mode];
3344     slice_qp = pic_param->pic_init_qp + slice_param->slice_qp_delta;
3345
3346     cmd = i965_gpe_context_map_curbe(gpe_context);
3347     if (!cmd)
3348         return;
3349
3350     memcpy((void *)cmd, GEN9_HEVC_ME_INIT_CURBE_DATA, sizeof(gen9_hevc_me_curbe_data));
3351
3352     cmd->dw3.sub_pel_mode = 3;
3353     cmd->dw4.picture_height_minus1 = downscaled_height_in_mb - 1;
3354     cmd->dw4.picture_width = downscaled_width_in_mb;
3355     cmd->dw5.qp_prime_y = slice_qp;
3356     cmd->dw6.use_mv_from_prev_step = use_mv_from_prev_step;
3357     cmd->dw6.write_distortions = write_distortions;
3358     cmd->dw6.super_combine_dist = GEN9_HEVC_SUPER_COMBINE_DIST[priv_state->tu_mode];
3359     cmd->dw6.max_vmvr = 512;
3360
3361     if (priv_state->picture_coding_type != HEVC_SLICE_I) {
3362         cmd->dw13.num_ref_idx_l0_minus1 = slice_param->num_ref_idx_l0_active_minus1;
3363         if (priv_state->picture_coding_type == HEVC_SLICE_B) {
3364             cmd->dw1.bi_weight = 32;
3365             cmd->dw13.num_ref_idx_l1_minus1 = slice_param->num_ref_idx_l1_active_minus1;
3366         }
3367     }
3368
3369     cmd->dw15.prev_mv_read_pos_factor = prev_mv_read_pos_factor;
3370     cmd->dw15.mv_shift_factor = mv_shift_factor;
3371
3372     memcpy(&cmd->dw16, table_enc_search_path[GEN9_HEVC_ENC_MEMETHOD_TABLE][me_method], 14 * sizeof(unsigned int));
3373
3374     cmd->dw32._4x_memv_output_data_surf_index = GEN9_HEVC_ME_MV_DATA_SURFACE_INDEX;
3375     cmd->dw33._16x_32x_memv_input_data_surf_index = (hme_type == HEVC_HME_32X) ?
3376                                                     GEN9_HEVC_ME_32X_MV_DATA_SURFACE_INDEX : GEN9_HEVC_ME_16X_MV_DATA_SURFACE_INDEX;
3377     cmd->dw34._4x_me_output_dist_surf_index = GEN9_HEVC_ME_DISTORTION_SURFACE_INDEX;
3378     cmd->dw35._4x_me_output_brc_dist_surf_index = GEN9_HEVC_ME_BRC_DISTORTION_INDEX;
3379     cmd->dw36.vme_fwd_inter_pred_surf_index = GEN9_HEVC_ME_CURR_FOR_FWD_REF_INDEX;
3380     cmd->dw37.vme_bdw_inter_pred_surf_index = GEN9_HEVC_ME_CURR_FOR_BWD_REF_INDEX;
3381
3382     i965_gpe_context_unmap_curbe(gpe_context);
3383 }
3384
3385 static void
3386 gen9_hevc_me_set_surfaces(VADriverContextP ctx,
3387                           struct encode_state *encode_state,
3388                           struct intel_encoder_context *encoder_context,
3389                           enum HEVC_HME_TYPE hme_type,
3390                           struct i965_gpe_context *gpe_context)
3391 {
3392     struct i965_driver_data *i965 = i965_driver_data(ctx);
3393     struct encoder_vme_mfc_context *vme_context = NULL;
3394     struct gen9_hevc_encoder_context *priv_ctx = NULL;
3395     struct generic_enc_codec_state *generic_state = NULL;
3396     VAEncSliceParameterBufferHEVC *slice_param = NULL;
3397     struct gen9_hevc_surface_priv *surface_priv = NULL;
3398     struct object_surface *obj_surface = NULL;
3399     int scaled_surf_id = VA_INVALID_SURFACE, surface_id = VA_INVALID_SURFACE;
3400     int i = 0;
3401
3402     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
3403     priv_ctx = (struct gen9_hevc_encoder_context *)vme_context->private_enc_ctx;
3404     generic_state = (struct generic_enc_codec_state *)vme_context->generic_enc_state;
3405     slice_param = (VAEncSliceParameterBufferHEVC *)encode_state->slice_params_ext[0]->buffer;
3406
3407     switch (hme_type) {
3408     case HEVC_HME_4X:
3409         scaled_surf_id = HEVC_SCALED_SURF_4X_ID;
3410         i965_add_buffer_2d_gpe_surface(ctx, gpe_context,
3411                                        &priv_ctx->s4x_memv_data_buffer,
3412                                        1,
3413                                        I965_SURFACEFORMAT_R8_UNORM,
3414                                        GEN9_HEVC_ME_MV_DATA_SURFACE_INDEX);
3415
3416         if (generic_state->b16xme_enabled)
3417             i965_add_buffer_2d_gpe_surface(ctx, gpe_context,
3418                                            &priv_ctx->s16x_memv_data_buffer,
3419                                            1,
3420                                            I965_SURFACEFORMAT_R8_UNORM,
3421                                            GEN9_HEVC_ME_16X_MV_DATA_SURFACE_INDEX);
3422
3423         i965_add_buffer_2d_gpe_surface(ctx, gpe_context,
3424                                        &priv_ctx->res_brc_me_dist_buffer,
3425                                        1,
3426                                        I965_SURFACEFORMAT_R8_UNORM,
3427                                        GEN9_HEVC_ME_BRC_DISTORTION_INDEX);
3428
3429         i965_add_buffer_2d_gpe_surface(ctx, gpe_context,
3430                                        &priv_ctx->s4x_memv_distortion_buffer,
3431                                        1,
3432                                        I965_SURFACEFORMAT_R8_UNORM,
3433                                        GEN9_HEVC_ME_DISTORTION_SURFACE_INDEX);
3434         break;
3435     case HEVC_HME_16X:
3436         scaled_surf_id = HEVC_SCALED_SURF_16X_ID;
3437
3438         i965_add_buffer_2d_gpe_surface(ctx, gpe_context,
3439                                        &priv_ctx->s16x_memv_data_buffer,
3440                                        1,
3441                                        I965_SURFACEFORMAT_R8_UNORM,
3442                                        GEN9_HEVC_ME_MV_DATA_SURFACE_INDEX);
3443
3444         if (generic_state->b32xme_enabled)
3445             i965_add_buffer_2d_gpe_surface(ctx, gpe_context,
3446                                            &priv_ctx->s32x_memv_data_buffer,
3447                                            1,
3448                                            I965_SURFACEFORMAT_R8_UNORM,
3449                                            GEN9_HEVC_ME_32X_MV_DATA_SURFACE_INDEX);
3450         else
3451             i965_add_buffer_2d_gpe_surface(ctx, gpe_context,
3452                                            &priv_ctx->s16x_memv_data_buffer,
3453                                            1,
3454                                            I965_SURFACEFORMAT_R8_UNORM,
3455                                            GEN9_HEVC_ME_16X_MV_DATA_SURFACE_INDEX);
3456         break;
3457     case HEVC_HME_32X:
3458         scaled_surf_id = HEVC_SCALED_SURF_32X_ID;
3459         i965_add_buffer_2d_gpe_surface(ctx, gpe_context,
3460                                        &priv_ctx->s32x_memv_data_buffer,
3461                                        1,
3462                                        I965_SURFACEFORMAT_R8_UNORM,
3463                                        GEN9_HEVC_ME_MV_DATA_SURFACE_INDEX);
3464         break;
3465     default:
3466         return;
3467     }
3468
3469     obj_surface = encode_state->reconstructed_object;
3470     surface_priv = (struct gen9_hevc_surface_priv *)obj_surface->private_data;
3471     i965_add_adv_gpe_surface(ctx, gpe_context,
3472                              surface_priv->scaled_surface_obj[scaled_surf_id],
3473                              GEN9_HEVC_ME_CURR_FOR_FWD_REF_INDEX);
3474
3475     for (i = 0; i < slice_param->num_ref_idx_l0_active_minus1 + 1; i++) {
3476         surface_id = slice_param->ref_pic_list0[i].picture_id;
3477         obj_surface = SURFACE(surface_id);
3478         if (!obj_surface || !obj_surface->private_data)
3479             break;
3480
3481         surface_priv = (struct gen9_hevc_surface_priv *)obj_surface->private_data;
3482         i965_add_adv_gpe_surface(ctx, gpe_context,
3483                                  surface_priv->scaled_surface_obj[scaled_surf_id],
3484                                  GEN9_HEVC_ME_CURR_FOR_FWD_REF_INDEX + i * 2 + 1);
3485     }
3486
3487     obj_surface = encode_state->reconstructed_object;
3488     surface_priv = (struct gen9_hevc_surface_priv *)obj_surface->private_data;
3489     i965_add_adv_gpe_surface(ctx, gpe_context,
3490                              surface_priv->scaled_surface_obj[scaled_surf_id],
3491                              GEN9_HEVC_ME_CURR_FOR_BWD_REF_INDEX);
3492
3493     for (i = 0; i < slice_param->num_ref_idx_l1_active_minus1 + 1; i++) {
3494         surface_id = slice_param->ref_pic_list1[i].picture_id;
3495         obj_surface = SURFACE(surface_id);
3496         if (!obj_surface || !obj_surface->private_data)
3497             break;
3498
3499         surface_priv = (struct gen9_hevc_surface_priv *)obj_surface->private_data;
3500         i965_add_adv_gpe_surface(ctx, gpe_context,
3501                                  surface_priv->scaled_surface_obj[scaled_surf_id],
3502                                  GEN9_HEVC_ME_CURR_FOR_BWD_REF_INDEX + i * 2 + 1);
3503     }
3504 }
3505
3506 static void
3507 gen9_hevc_kernel_me(VADriverContextP ctx,
3508                     struct encode_state *encode_state,
3509                     struct intel_encoder_context *encoder_context,
3510                     enum HEVC_HME_TYPE hme_type)
3511 {
3512     struct encoder_vme_mfc_context *vme_context = NULL;
3513     struct gen9_hevc_encoder_context *priv_ctx = NULL;
3514     struct gen9_hevc_encoder_state *priv_state = NULL;
3515     struct gpe_media_object_walker_parameter param;
3516     struct hevc_enc_kernel_walker_parameter hevc_walker_param;
3517     unsigned int downscaled_width_in_mb, downscaled_height_in_mb;
3518     struct i965_gpe_context *gpe_context;
3519     int media_state = 0;
3520
3521     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
3522     priv_ctx = (struct gen9_hevc_encoder_context *)vme_context->private_enc_ctx;
3523     priv_state = (struct gen9_hevc_encoder_state *)vme_context->private_enc_state;
3524
3525     switch (hme_type) {
3526     case HEVC_HME_4X:
3527         media_state = HEVC_ENC_MEDIA_STATE_4X_ME;
3528         downscaled_width_in_mb = priv_state->downscaled_width_4x_in_mb;
3529         downscaled_height_in_mb = priv_state->downscaled_height_4x_in_mb;
3530         break;
3531     case HEVC_HME_16X:
3532         media_state = HEVC_ENC_MEDIA_STATE_16X_ME;
3533         downscaled_width_in_mb = priv_state->downscaled_width_16x_in_mb;
3534         downscaled_height_in_mb = priv_state->downscaled_height_16x_in_mb;
3535         break;
3536     case HEVC_HME_32X:
3537         media_state = HEVC_ENC_MEDIA_STATE_32X_ME;
3538         downscaled_width_in_mb = priv_state->downscaled_width_32x_in_mb;
3539         downscaled_height_in_mb = priv_state->downscaled_height_32x_in_mb;
3540         break;
3541     default:
3542         return;
3543     }
3544
3545     if (priv_state->picture_coding_type == HEVC_SLICE_P)
3546         gpe_context = &priv_ctx->me_context.gpe_context[hme_type][HEVC_ENC_ME_P];
3547     else
3548         gpe_context = &priv_ctx->me_context.gpe_context[hme_type][HEVC_ENC_ME_B];
3549
3550     gen8_gpe_context_init(ctx, gpe_context);
3551     gen9_gpe_reset_binding_table(ctx, gpe_context);
3552     gen9_hevc_me_set_curbe(ctx, encode_state, encoder_context, hme_type, gpe_context);
3553     gen9_hevc_me_set_surfaces(ctx, encode_state, encoder_context, hme_type, gpe_context);
3554     gen8_gpe_setup_interface_data(ctx, gpe_context);
3555
3556     memset((void *)&hevc_walker_param, 0, sizeof(hevc_walker_param));
3557     hevc_walker_param.resolution_x = downscaled_width_in_mb;
3558     hevc_walker_param.resolution_y = downscaled_height_in_mb;
3559     hevc_walker_param.no_dependency = 1;
3560     gen9_hevc_init_object_walker(&hevc_walker_param, &param);
3561
3562     gen9_hevc_run_object_walker(ctx, encoder_context, gpe_context, &param,
3563                                 media_state);
3564 }
3565
3566 static void
3567 gen9_hevc_hme_encode_me(VADriverContextP ctx,
3568                         struct encode_state *encode_state,
3569                         struct intel_encoder_context *encoder_context)
3570 {
3571     struct encoder_vme_mfc_context *vme_context = NULL;
3572     struct generic_enc_codec_state *generic_state = NULL;
3573
3574     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
3575     generic_state = (struct generic_enc_codec_state *)vme_context->generic_enc_state;
3576
3577     if (generic_state->b16xme_enabled) {
3578         if (generic_state->b32xme_enabled)
3579             gen9_hevc_kernel_me(ctx, encode_state, encoder_context, HEVC_HME_32X);
3580
3581         gen9_hevc_kernel_me(ctx, encode_state, encoder_context, HEVC_HME_16X);
3582     }
3583
3584     gen9_hevc_kernel_me(ctx, encode_state, encoder_context, HEVC_HME_4X);
3585 }
3586
3587 // MBENC kernels setting start
3588
3589 static unsigned char
3590 map_44_lut_value(unsigned int value,
3591                  unsigned char max)
3592 {
3593     unsigned int max_cost = 0;
3594     int data = 0;
3595     unsigned char ret = 0;
3596
3597     if (value == 0)
3598         return 0;
3599
3600     max_cost = ((max & 15) << (max >> 4));
3601     if (value >= max_cost)
3602         return max;
3603
3604     data = (int)(log((double)value) / log(2.)) - 3;
3605     if (data < 0)
3606         data = 0;
3607
3608     ret = (unsigned char)((data << 4) +
3609                           (int)((value + (data == 0 ? 0 : (1 << (data - 1)))) >> data));
3610     ret = (ret & 0xf) == 0 ? (ret | 8) : ret;
3611
3612     return ret;
3613 }
3614
3615 static void
3616 gen9_hevc_mbenc_set_costs(struct gen9_hevc_encoder_context *priv_ctx,
3617                           int slice_type,
3618                           int intra_trans_type,
3619                           unsigned int slice_qp,
3620                           unsigned char *mode_cost,
3621                           unsigned char *mv_cost,
3622                           unsigned char *mode_cost_sp,
3623                           unsigned int *simplest_intra_inter_threshold)
3624 {
3625     float had_bias = intra_trans_type == HEVC_ENC_INTRA_TRANS_HADAMARD ?
3626                      1.67f : 2.0f;
3627     double lambda_md, lambda_me;
3628
3629     lambda_md = priv_ctx->lambda_md_table[slice_type][slice_qp];
3630     lambda_me = priv_ctx->lambda_md_table[slice_type][slice_qp];
3631
3632     if (mode_cost) {
3633         mode_cost[0] = map_44_lut_value((unsigned int)(lambda_md * GEN9_HEVC_ENC_Mode_COST[slice_type][0] * had_bias), 0x6f);
3634         mode_cost[1] = map_44_lut_value((unsigned int)(lambda_md * GEN9_HEVC_ENC_Mode_COST[slice_type][1] * had_bias), 0x8f);
3635         mode_cost[2] = map_44_lut_value((unsigned int)(lambda_md * GEN9_HEVC_ENC_Mode_COST[slice_type][2] * had_bias), 0x8f);
3636         mode_cost[3] = map_44_lut_value((unsigned int)(lambda_md * GEN9_HEVC_ENC_Mode_COST[slice_type][3] * had_bias), 0x8f);
3637         mode_cost[4] = map_44_lut_value((unsigned int)(lambda_md * GEN9_HEVC_ENC_Mode_COST[slice_type][4] * had_bias), 0x8f);
3638         mode_cost[5] = map_44_lut_value((unsigned int)(lambda_md * GEN9_HEVC_ENC_Mode_COST[slice_type][5] * had_bias), 0x6f);
3639         mode_cost[6] = map_44_lut_value((unsigned int)(lambda_md * GEN9_HEVC_ENC_Mode_COST[slice_type][6] * had_bias), 0x6f);
3640         mode_cost[7] = map_44_lut_value((unsigned int)(lambda_md * GEN9_HEVC_ENC_Mode_COST[slice_type][7] * had_bias), 0x6f);
3641         mode_cost[8] = map_44_lut_value((unsigned int)(lambda_md * GEN9_HEVC_ENC_Mode_COST[slice_type][8] * had_bias), 0x8f);
3642         mode_cost[9] = map_44_lut_value((unsigned int)(lambda_md * GEN9_HEVC_ENC_Mode_COST[slice_type][9] * had_bias), 0x6f);
3643         mode_cost[10] = map_44_lut_value((unsigned int)(lambda_md * GEN9_HEVC_ENC_Mode_COST[slice_type][10] * had_bias), 0x6f);
3644         mode_cost[11] = map_44_lut_value((unsigned int)(lambda_md * GEN9_HEVC_ENC_Mode_COST[slice_type][11] * had_bias), 0x6f);
3645     }
3646
3647     if (mv_cost) {
3648         mv_cost[0] = map_44_lut_value((unsigned int)(lambda_me * GEN9_HEVC_ENC_MV_COST[slice_type][0] * had_bias), 0x6f);
3649         mv_cost[1] = map_44_lut_value((unsigned int)(lambda_me * GEN9_HEVC_ENC_MV_COST[slice_type][1] * had_bias), 0x6f);
3650         mv_cost[2] = map_44_lut_value((unsigned int)(lambda_me * GEN9_HEVC_ENC_MV_COST[slice_type][2] * had_bias), 0x6f);
3651         mv_cost[3] = map_44_lut_value((unsigned int)(lambda_me * GEN9_HEVC_ENC_MV_COST[slice_type][3] * had_bias), 0x6f);
3652         mv_cost[4] = map_44_lut_value((unsigned int)(lambda_me * GEN9_HEVC_ENC_MV_COST[slice_type][4] * had_bias), 0x6f);
3653         mv_cost[5] = map_44_lut_value((unsigned int)(lambda_me * GEN9_HEVC_ENC_MV_COST[slice_type][5] * had_bias), 0x6f);
3654         mv_cost[6] = map_44_lut_value((unsigned int)(lambda_me * GEN9_HEVC_ENC_MV_COST[slice_type][6] * had_bias), 0x6f);
3655         mv_cost[7] = map_44_lut_value((unsigned int)(lambda_me * GEN9_HEVC_ENC_MV_COST[slice_type][7] * had_bias), 0x6f);
3656     }
3657
3658     if (mode_cost_sp)
3659         *mode_cost_sp = map_44_lut_value((unsigned int)(lambda_md * 45 * had_bias), 0x8f);
3660
3661     if (simplest_intra_inter_threshold) {
3662         lambda_md *= had_bias;
3663         *simplest_intra_inter_threshold = 0;
3664         if (GEN9_HEVC_ENC_Mode_COST[slice_type][1] < GEN9_HEVC_ENC_Mode_COST[slice_type][3])
3665             *simplest_intra_inter_threshold = (unsigned int)(lambda_md *
3666                                                              (GEN9_HEVC_ENC_Mode_COST[slice_type][3] - GEN9_HEVC_ENC_Mode_COST[slice_type][1]) + 0.5);
3667     }
3668 }
3669
3670 static void
3671 gen9_hevc_set_lambda_tables(struct gen9_hevc_encoder_context *priv_ctx,
3672                             int slice_type,
3673                             int intra_trans_type)
3674 {
3675     if (slice_type != HEVC_SLICE_I) {
3676         if (priv_ctx->lambda_table_inited)
3677             return;
3678
3679         memcpy((void *)&priv_ctx->lambda_me_table[slice_type], &GEN9_HEVC_ENC_QP_LAMBDA_ME[slice_type],
3680                sizeof(GEN9_HEVC_ENC_QP_LAMBDA_ME[slice_type]));
3681         memcpy((void *)&priv_ctx->lambda_md_table[slice_type], &GEN9_HEVC_ENC_QP_LAMBDA_ME[slice_type],
3682                sizeof(GEN9_HEVC_ENC_QP_LAMBDA_ME[slice_type]));
3683     } else if (intra_trans_type != priv_ctx->lambda_intra_trans_type ||
3684                !priv_ctx->lambda_table_inited) {
3685         double temp = 0.0;
3686         double lambda = 0.0;
3687         int qp = 0;
3688
3689         for (qp = 0; qp < 52; qp++) {
3690             temp = (double)qp - 12;
3691             lambda = 0.85 * pow(2.0, temp / 3.0);
3692
3693             if ((intra_trans_type != HEVC_ENC_INTRA_TRANS_HAAR) &&
3694                 (intra_trans_type != HEVC_ENC_INTRA_TRANS_HADAMARD))
3695                 lambda *= 0.95;
3696
3697             priv_ctx->lambda_md_table[slice_type][qp] =
3698                 priv_ctx->lambda_me_table[slice_type][qp] = sqrt(lambda);
3699         }
3700
3701         priv_ctx->lambda_intra_trans_type = intra_trans_type;
3702     }
3703 }
3704
3705 static void
3706 gen9_hevc_lambda_tables_init(struct gen9_hevc_encoder_context *priv_ctx)
3707 {
3708     gen9_hevc_set_lambda_tables(priv_ctx, HEVC_SLICE_B, HEVC_ENC_INTRA_TRANS_HAAR);
3709     gen9_hevc_set_lambda_tables(priv_ctx, HEVC_SLICE_P, HEVC_ENC_INTRA_TRANS_HAAR);
3710     gen9_hevc_set_lambda_tables(priv_ctx, HEVC_SLICE_I, HEVC_ENC_INTRA_TRANS_HAAR);
3711
3712     priv_ctx->lambda_table_inited = 1;
3713 }
3714
3715 static void
3716 gen9_hevc_8x8_b_pak_set_curbe(VADriverContextP ctx,
3717                               struct encode_state *encode_state,
3718                               struct intel_encoder_context *encoder_context,
3719                               struct i965_gpe_context *gpe_context)
3720 {
3721     struct i965_driver_data *i965 = i965_driver_data(ctx);
3722     struct encoder_vme_mfc_context *vme_context = NULL;
3723     struct generic_enc_codec_state *generic_state = NULL;
3724     struct gen9_hevc_encoder_state *priv_state = NULL;
3725     gen9_hevc_mbenc_b_pak_curbe_data *cmd = NULL;
3726     VAEncPictureParameterBufferHEVC *pic_param = NULL;
3727     VAEncSliceParameterBufferHEVC *slice_param = NULL;
3728     unsigned int slice_qp = 0;
3729     int bti_idx = 0;
3730
3731     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
3732     generic_state = (struct generic_enc_codec_state *)vme_context->generic_enc_state;
3733     priv_state = (struct gen9_hevc_encoder_state *)vme_context->private_enc_state;
3734     pic_param = (VAEncPictureParameterBufferHEVC *)encode_state->pic_param_ext->buffer;
3735     slice_param = (VAEncSliceParameterBufferHEVC *)encode_state->slice_params_ext[0]->buffer;
3736
3737     slice_qp = pic_param->pic_init_qp + slice_param->slice_qp_delta;
3738     cmd = i965_gpe_context_map_curbe(gpe_context);
3739     if (!cmd)
3740         return;
3741
3742     memset((void *)cmd, 0, sizeof(*cmd));
3743
3744     cmd->dw0.frame_width = ALIGN(priv_state->picture_width, 16);
3745     cmd->dw0.frame_height = ALIGN(priv_state->picture_height, 16);
3746     cmd->dw1.max_vmvr = 511 * 4;
3747     cmd->dw1.qp = slice_qp;
3748     cmd->dw2.brc_enable = generic_state->brc_enabled;
3749     cmd->dw2.lcu_brc_enable = priv_state->lcu_brc_enabled;
3750     cmd->dw2.screen_content = !!pic_param->pic_fields.bits.screen_content_flag;
3751     cmd->dw2.slice_type = priv_state->picture_coding_type;
3752     cmd->dw2.roi_enable = (priv_state->num_roi > 0);
3753     cmd->dw2.fast_surveillance_flag = priv_state->picture_coding_type == HEVC_SLICE_I ?
3754                                       0 : priv_state->video_surveillance_flag;
3755     cmd->dw2.kbl_control_flag = (IS_KBL(i965->intel.device_info) || IS_GLK(i965->intel.device_info));
3756     cmd->dw2.enable_rolling_intra = priv_state->rolling_intra_refresh;
3757     cmd->dw2.simplest_intra_enable = (priv_state->tu_mode == HEVC_TU_BEST_SPEED);
3758     cmd->dw3.widi_intra_refresh_qp_delta = priv_state->widi_intra_refresh_qp_delta;
3759     cmd->dw3.widi_intra_refresh_mb_num = priv_state->widi_intra_insertion_location;
3760     cmd->dw3.widi_intra_refresh_unit_in_mb = priv_state->widi_intra_insertion_size;
3761
3762     cmd->dw16.bti_cu_record = bti_idx++;
3763     cmd->dw17.bti_pak_obj = bti_idx++;
3764     cmd->dw18.bti_slice_map = bti_idx++;
3765     cmd->dw19.bti_brc_input = bti_idx++;
3766     cmd->dw20.bti_lcu_qp = bti_idx++;
3767     cmd->dw21.bti_brc_data = bti_idx++;
3768     cmd->dw22.bti_mb_data = bti_idx++;
3769     cmd->dw23.bti_mvp_surface = bti_idx++;
3770     cmd->dw24.bti_debug = bti_idx++;
3771
3772     i965_gpe_context_unmap_curbe(gpe_context);
3773 }
3774
3775 static void
3776 gen9_hevc_8x8_b_pak_set_surfaces(VADriverContextP ctx,
3777                                  struct encode_state *encode_state,
3778                                  struct intel_encoder_context *encoder_context,
3779                                  struct i965_gpe_context *gpe_context)
3780 {
3781     struct encoder_vme_mfc_context *vme_context = NULL;
3782     struct gen9_hevc_encoder_context *priv_ctx = NULL;
3783     struct gen9_hevc_encoder_state *priv_state = NULL;
3784     int bti_idx = 0;
3785     int size = 0;
3786
3787     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
3788     priv_ctx = (struct gen9_hevc_encoder_context *)vme_context->private_enc_ctx;
3789     priv_state = (struct gen9_hevc_encoder_state *)vme_context->private_enc_state;
3790
3791     size = priv_state->width_in_cu * priv_state->height_in_cu *
3792            priv_state->cu_record_size;
3793     gen9_hevc_set_gpe_1d_surface(ctx, priv_ctx, gpe_context,
3794                                  HEVC_ENC_SURFACE_CU_RECORD, bti_idx++,
3795                                  0, size, priv_state->mb_data_offset, NULL, NULL);
3796
3797     size = priv_state->mb_data_offset;
3798     gen9_hevc_set_gpe_1d_surface(ctx, priv_ctx, gpe_context,
3799                                  HEVC_ENC_SURFACE_HCP_PAK, bti_idx++,
3800                                  0, size, 0, NULL, NULL);
3801
3802     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
3803                                  HEVC_ENC_SURFACE_SLICE_MAP, bti_idx++,
3804                                  0, 1, I965_SURFACEFORMAT_R8_UNORM,
3805                                  NULL, NULL);
3806
3807     gen9_hevc_set_gpe_1d_surface(ctx, priv_ctx, gpe_context,
3808                                  HEVC_ENC_SURFACE_BRC_INPUT, bti_idx++,
3809                                  0, 0, 0, NULL, NULL);
3810
3811     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
3812                                  HEVC_ENC_SURFACE_LCU_QP, bti_idx++,
3813                                  0, 1, I965_SURFACEFORMAT_R8_UNORM,
3814                                  NULL, NULL);
3815
3816     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
3817                                  HEVC_ENC_SURFACE_BRC_DATA, bti_idx++,
3818                                  0, 1, I965_SURFACEFORMAT_R8_UNORM,
3819                                  NULL, NULL);
3820
3821     gen9_hevc_set_gpe_1d_surface(ctx, priv_ctx, gpe_context,
3822                                  HEVC_ENC_SURFACE_MB_MV_INDEX, bti_idx++,
3823                                  0, 0, 0, NULL, NULL);
3824
3825     gen9_hevc_set_gpe_1d_surface(ctx, priv_ctx, gpe_context,
3826                                  HEVC_ENC_SURFACE_MVP_INDEX, bti_idx++,
3827                                  0, 0, 0, NULL, NULL);
3828
3829     gen9_hevc_set_gpe_1d_surface(ctx, priv_ctx, gpe_context,
3830                                  HEVC_ENC_SURFACE_KERNEL_DEBUG, bti_idx++,
3831                                  0, 0, 0, NULL, NULL);
3832 }
3833
3834 static void
3835 gen9_hevc_8x8_b_pak(VADriverContextP ctx,
3836                     struct encode_state *encode_state,
3837                     struct intel_encoder_context *encoder_context)
3838 {
3839     struct encoder_vme_mfc_context *vme_context = NULL;
3840     struct gen9_hevc_encoder_context *priv_ctx = NULL;
3841     struct gen9_hevc_encoder_state *priv_state = NULL;
3842     struct gpe_media_object_walker_parameter param;
3843     struct hevc_enc_kernel_walker_parameter hevc_walker_param;
3844     struct i965_gpe_context *gpe_context = NULL;
3845     int media_state = HEVC_ENC_MEDIA_STATE_HEVC_B_PAK;
3846
3847     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
3848     priv_ctx = (struct gen9_hevc_encoder_context *)vme_context->private_enc_ctx;
3849     priv_state = (struct gen9_hevc_encoder_state *)vme_context->private_enc_state;
3850
3851     gpe_context = &priv_ctx->mbenc_context.gpe_contexts[HEVC_MBENC_BPAK_IDX];
3852
3853     gen8_gpe_context_init(ctx, gpe_context);
3854     gen9_gpe_reset_binding_table(ctx, gpe_context);
3855     gen9_hevc_8x8_b_pak_set_curbe(ctx, encode_state, encoder_context, gpe_context);
3856     gen9_hevc_8x8_b_pak_set_surfaces(ctx, encode_state, encoder_context, gpe_context);
3857     gen8_gpe_setup_interface_data(ctx, gpe_context);
3858
3859     memset((void *)&hevc_walker_param, 0, sizeof(hevc_walker_param));
3860     hevc_walker_param.resolution_x = ALIGN(priv_state->picture_width, 32) >> 5;
3861     hevc_walker_param.resolution_y = ALIGN(priv_state->picture_height, 32) >> 5;
3862     hevc_walker_param.no_dependency = 1;
3863     gen9_hevc_init_object_walker(&hevc_walker_param, &param);
3864     gen9_hevc_run_object_walker(ctx, encoder_context, gpe_context, &param,
3865                                 media_state);
3866 }
3867
3868 static const unsigned char ftq_25i[27] = {
3869     0, 0, 0, 0,
3870     1, 3, 6, 8, 11,
3871     13, 16, 19, 22, 26,
3872     30, 34, 39, 44, 50,
3873     56, 62, 69, 77, 85,
3874     94, 104, 115
3875 };
3876
3877 static void
3878 gen9_hevc_set_forward_coeff_thd(unsigned char *pcoeff,
3879                                 int qp)
3880 {
3881     int idx = (qp + 1) >> 1;
3882
3883     memset((void *)pcoeff, ftq_25i[idx], 7);
3884 }
3885
3886 static int
3887 gen9_hevc_get_qp_from_ref_list(VADriverContextP ctx,
3888                                VAEncSliceParameterBufferHEVC *slice_param,
3889                                int list_idx,
3890                                int ref_frame_idx)
3891 {
3892     struct i965_driver_data *i965 = i965_driver_data(ctx);
3893     struct gen9_hevc_surface_priv *hevc_priv_surface = NULL;
3894     struct object_surface *obj_surface = NULL;
3895     VASurfaceID surface_id;
3896
3897     if (list_idx == 0) {
3898         if (ref_frame_idx < slice_param->num_ref_idx_l0_active_minus1 + 1)
3899             surface_id = slice_param->ref_pic_list0[ref_frame_idx].picture_id;
3900         else
3901             goto FAIL;
3902     } else {
3903         if (ref_frame_idx < slice_param->num_ref_idx_l1_active_minus1 + 1)
3904             surface_id = slice_param->ref_pic_list1[ref_frame_idx].picture_id;
3905         else
3906             goto FAIL;
3907     }
3908
3909     obj_surface = SURFACE(surface_id);
3910     if (obj_surface && obj_surface->private_data) {
3911         hevc_priv_surface = obj_surface->private_data;
3912         return hevc_priv_surface->qp_value;
3913     }
3914
3915 FAIL:
3916     return 0;
3917 }
3918
3919 static short
3920 gen9_hevc_get_poc_diff_from_ref_list(VAEncPictureParameterBufferHEVC *pic_param,
3921                                      VAEncSliceParameterBufferHEVC *slice_param,
3922                                      int list_idx,
3923                                      int ref_frame_idx)
3924 {
3925     short poc_diff = 0;
3926
3927     if (list_idx == 0 &&
3928         ref_frame_idx < slice_param->num_ref_idx_l0_active_minus1 + 1)
3929         poc_diff = pic_param->decoded_curr_pic.pic_order_cnt -
3930                    slice_param->ref_pic_list0[ref_frame_idx].pic_order_cnt;
3931     else if (list_idx == 1 &&
3932              ref_frame_idx < slice_param->num_ref_idx_l1_active_minus1 + 1)
3933         poc_diff = pic_param->decoded_curr_pic.pic_order_cnt -
3934                    slice_param->ref_pic_list1[ref_frame_idx].pic_order_cnt;
3935
3936     return CLAMP(-128, 127, poc_diff);
3937 }
3938
3939 static void
3940 gen9_hevc_set_control_region(VADriverContextP ctx,
3941                              struct encode_state *encode_state,
3942                              struct intel_encoder_context *encoder_context,
3943                              struct i965_gpe_context *gpe_context,
3944                              struct gen9_hevc_walking_pattern_parameter *param)
3945 {
3946     struct encoder_vme_mfc_context *vme_context = NULL;
3947     struct gen9_hevc_encoder_context *priv_ctx = NULL;
3948     struct gen9_hevc_encoder_state *priv_state = NULL;
3949     VAEncSliceParameterBufferHEVC *slice_param = NULL;
3950     gen9_hevc_mbenc_control_region *p_region = NULL;
3951     unsigned int slice, num_regions, height, num_slices, num_units_in_region;
3952     unsigned int frame_width_in_units, frame_height_in_units;
3953     unsigned short region_start_table[64];
3954     unsigned int offset_to_the_region_start[16];
3955     unsigned short temp_data[32][32];
3956     int is_arbitrary_slices = 0;
3957     int slice_start_y[I965_MAX_NUM_SLICE + 1];
3958     int max_height;
3959     int k = 0, i = 0;
3960
3961     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
3962     priv_ctx = (struct gen9_hevc_encoder_context *)vme_context->private_enc_ctx;
3963     priv_state = (struct gen9_hevc_encoder_state *)vme_context->private_enc_state;
3964
3965     memset(slice_start_y, 0, sizeof(int) * (I965_MAX_NUM_SLICE + 1));
3966     memset(region_start_table, 0, sizeof(region_start_table));
3967     memset(temp_data, 0, sizeof(temp_data));
3968     memset(offset_to_the_region_start, 0, sizeof(offset_to_the_region_start));
3969
3970     if (priv_state->num_regions_in_slice < 1)
3971         priv_state->num_regions_in_slice = 1;
3972
3973     if (priv_state->num_regions_in_slice > 16)
3974         priv_state->num_regions_in_slice = 16;
3975
3976     if (priv_state->walking_pattern_26) {
3977         frame_width_in_units = ALIGN(priv_state->picture_width, 16) / 16;
3978         frame_height_in_units = ALIGN(priv_state->picture_height, 16) / 16;
3979     } else {
3980         frame_width_in_units = ALIGN(priv_state->picture_width, 32) / 32;
3981         frame_height_in_units = ALIGN(priv_state->picture_height, 32) / 32;
3982     }
3983
3984     for (slice = 0; slice < encode_state->num_slice_params_ext; slice++) {
3985         slice_param = (VAEncSliceParameterBufferHEVC *)encode_state->slice_params_ext[slice]->buffer;
3986         if (slice_param->slice_segment_address %
3987             ALIGN(priv_state->picture_width, 32)) {
3988             is_arbitrary_slices = 1;
3989         } else {
3990             slice_start_y[slice] = slice_param->slice_segment_address /
3991                                    ALIGN(priv_state->picture_width, 32);
3992
3993             if (priv_state->walking_pattern_26) {
3994                 slice_start_y[slice] *= 2;
3995             }
3996         }
3997     }
3998
3999     slice_start_y[encode_state->num_slice_params_ext] = frame_height_in_units;
4000
4001     region_start_table[0] = 0;
4002     region_start_table[GEN9_HEVC_ENC_REGION_START_Y_OFFSET + 0] = 0;
4003     num_regions = 1;
4004
4005     if (is_arbitrary_slices) {
4006         height = frame_height_in_units;
4007         num_slices = 1;
4008         max_height = height;
4009         if (priv_state->num_regions_in_slice > 1) {
4010             num_units_in_region =
4011                 (frame_width_in_units + 2 * (frame_height_in_units - 1) + priv_state->num_regions_in_slice - 1) / priv_state->num_regions_in_slice;
4012
4013             num_regions = priv_state->num_regions_in_slice;
4014
4015             for (i = 1; i < priv_state->num_regions_in_slice; i++) {
4016                 unsigned int front = i * num_units_in_region;
4017
4018                 if (front < frame_width_in_units)
4019                     region_start_table[i] = (unsigned int)front;
4020                 else if (((front - frame_width_in_units + 1) & 1) == 0)
4021                     region_start_table[i] = (unsigned int)frame_width_in_units - 1;
4022                 else
4023                     region_start_table[i] = (unsigned int)frame_width_in_units - 2;
4024
4025                 region_start_table[GEN9_HEVC_ENC_REGION_START_Y_OFFSET + i] = (unsigned int)((front - region_start_table[i]) >> 1);
4026             }
4027         }
4028     } else {
4029         int start_y;
4030         int slice_is_merged = 0;
4031
4032         max_height = 0;
4033         num_slices = encode_state->num_slice_params_ext;
4034
4035         for (slice = 0; slice < num_slices; slice++) {
4036             int sliceHeight = slice_start_y[slice + 1] - slice_start_y[slice];
4037
4038             if (sliceHeight > max_height)
4039                 max_height = sliceHeight;
4040         }
4041
4042         while (!slice_is_merged) {
4043             int newNumSlices = 1;
4044
4045             start_y = 0;
4046
4047             for (slice = 1; slice < num_slices; slice++) {
4048                 if ((slice_start_y[slice + 1] - start_y) <= max_height)
4049                     slice_start_y[slice] = -1;
4050                 else
4051                     start_y = slice_start_y[slice];
4052             }
4053
4054             for (slice = 1; slice < num_slices; slice++) {
4055                 if (slice_start_y[slice] > 0) {
4056                     slice_start_y[newNumSlices] = slice_start_y[slice];
4057                     newNumSlices++;
4058                 }
4059             }
4060
4061             num_slices = newNumSlices;
4062             slice_start_y[num_slices] = frame_height_in_units;
4063
4064             if (num_slices * priv_state->num_regions_in_slice <= GEN9_HEVC_MEDIA_WALKER_MAX_COLORS)
4065                 slice_is_merged = 1;
4066             else {
4067                 int num = 1;
4068
4069                 max_height = frame_height_in_units;
4070
4071                 for (slice = 0; slice < num_slices - 1; slice++) {
4072                     if ((slice_start_y[slice + 2] - slice_start_y[slice]) <= max_height) {
4073                         max_height = slice_start_y[slice + 2] - slice_start_y[slice];
4074                         num = slice + 1;
4075                     }
4076                 }
4077
4078                 for (slice = num; slice < num_slices; slice++)
4079                     slice_start_y[slice] = slice_start_y[slice + 1];
4080
4081                 num_slices--;
4082             }
4083         }
4084
4085         num_units_in_region = (frame_width_in_units + 2 * (max_height - 1) + priv_state->num_regions_in_slice - 1) /
4086                               priv_state->num_regions_in_slice;
4087         num_regions = num_slices * priv_state->num_regions_in_slice;
4088
4089         for (slice = 0; slice < num_slices; slice++) {
4090             region_start_table[slice * priv_state->num_regions_in_slice] = 0;
4091             region_start_table[GEN9_HEVC_ENC_REGION_START_Y_OFFSET + (slice * priv_state->num_regions_in_slice)] = (unsigned int)slice_start_y[slice];
4092
4093             for (i = 1; i < priv_state->num_regions_in_slice; i++) {
4094                 int front = i * num_units_in_region;
4095
4096                 if ((unsigned int)front < frame_width_in_units)
4097                     region_start_table[slice * priv_state->num_regions_in_slice + i] = (unsigned int)front;
4098                 else if (((front - frame_width_in_units + 1) & 1) == 0)
4099                     region_start_table[slice * priv_state->num_regions_in_slice + i] = (unsigned int)frame_width_in_units - 1;
4100                 else
4101                     region_start_table[slice * priv_state->num_regions_in_slice + i] = (unsigned int)frame_width_in_units - 2;
4102
4103                 region_start_table[GEN9_HEVC_ENC_REGION_START_Y_OFFSET + (slice * priv_state->num_regions_in_slice + i)] = (unsigned int)slice_start_y[slice] +
4104                                                                                                                            ((front - region_start_table[i]) >> 1);
4105             }
4106         }
4107         height = max_height;
4108     }
4109
4110     for (k = 0; k < num_slices; k++) {
4111         int i;
4112         int nearest_reg = 0;
4113         int min_delta = priv_state->picture_height;
4114
4115         if (priv_state->walking_pattern_26) {
4116             int cur_lcu_pel_y = region_start_table[GEN9_HEVC_ENC_REGION_START_Y_OFFSET + (k * priv_state->num_regions_in_slice)] << 4;
4117             int ts_width = priv_state->picture_width >> 4;
4118             int ts_height = height;
4119             int offset_y = -((ts_width + 1) >> 1);
4120             int offset_delta = ((ts_width + ((ts_height - 1) << 1)) + (priv_state->num_regions_in_slice - 1)) / (priv_state->num_regions_in_slice);
4121
4122             for (i = 0; i < (int)num_regions; i++) {
4123                 if (region_start_table[i] == 0) {
4124                     int delta = cur_lcu_pel_y - (region_start_table[GEN9_HEVC_ENC_REGION_START_Y_OFFSET + i] << 4);
4125
4126                     if (delta >= 0) {
4127                         if (delta < min_delta) {
4128                             min_delta = delta;
4129                             nearest_reg = i;
4130                         }
4131                     }
4132                 }
4133
4134                 offset_to_the_region_start[k] = 2 * region_start_table[GEN9_HEVC_ENC_REGION_START_Y_OFFSET + nearest_reg];
4135             }
4136
4137             for (i = 0; i < priv_state->num_regions_in_slice; i++) {
4138                 int tmp_y = region_start_table[GEN9_HEVC_ENC_REGION_START_Y_OFFSET + (nearest_reg + priv_state->num_regions_in_slice)];
4139
4140                 temp_data[k * priv_state->num_regions_in_slice + i][0] = region_start_table[nearest_reg + i];
4141                 temp_data[k * priv_state->num_regions_in_slice + i][1] = region_start_table[GEN9_HEVC_ENC_REGION_START_Y_OFFSET + (nearest_reg + i)];
4142                 temp_data[k * priv_state->num_regions_in_slice + i][2] = region_start_table[GEN9_HEVC_ENC_REGION_START_Y_OFFSET + nearest_reg];
4143                 temp_data[k * priv_state->num_regions_in_slice + i][3] = (unsigned int)((tmp_y != 0) ? tmp_y : (priv_state->picture_height) >> 4);
4144                 temp_data[k * priv_state->num_regions_in_slice + i][4] = offset_to_the_region_start[k] & 0x0FFFF;
4145                 temp_data[k * priv_state->num_regions_in_slice + i][5] = 0;
4146                 temp_data[k * priv_state->num_regions_in_slice + i][6] = 0;
4147                 temp_data[k * priv_state->num_regions_in_slice + i][7] = (unsigned int)(offset_y + region_start_table[GEN9_HEVC_ENC_REGION_START_Y_OFFSET + nearest_reg] + ((i * offset_delta) >> 1));
4148             }
4149         } else {
4150             int cur_lcu_pel_y = region_start_table[GEN9_HEVC_ENC_REGION_START_Y_OFFSET +
4151                                                    (k * priv_state->num_regions_in_slice)] << 5;
4152             int ts_width = (priv_state->picture_width + 16) >> 5;
4153             int ts_height = height;
4154             int offset_y = -4 * ((ts_width + 1) >> 1);
4155             int offset_delta = ((ts_width + ((ts_height - 1) << 1)) + (priv_state->num_regions_in_slice - 1)) / (priv_state->num_regions_in_slice);
4156
4157             for (i = 0; i < (int)num_regions; i++) {
4158                 if (region_start_table[i] == 0) {
4159                     int delta = cur_lcu_pel_y - (region_start_table[GEN9_HEVC_ENC_REGION_START_Y_OFFSET + i] << 5);
4160
4161                     if (delta >= 0) {
4162                         if (delta < min_delta) {
4163                             min_delta = delta;
4164                             nearest_reg = i;
4165                         }
4166                     }
4167                 }
4168
4169                 offset_to_the_region_start[k] = 2 * region_start_table[GEN9_HEVC_ENC_REGION_START_Y_OFFSET + nearest_reg];
4170             }
4171
4172             for (i = 0; i < priv_state->num_regions_in_slice; i++) {
4173                 int tmp_y = 2 * region_start_table[GEN9_HEVC_ENC_REGION_START_Y_OFFSET + (nearest_reg + priv_state->num_regions_in_slice)];
4174
4175                 temp_data[k * priv_state->num_regions_in_slice + i][0] = region_start_table[nearest_reg + i];
4176                 temp_data[k * priv_state->num_regions_in_slice + i][1] = 2 * region_start_table[GEN9_HEVC_ENC_REGION_START_Y_OFFSET + (nearest_reg + i)];
4177                 temp_data[k * priv_state->num_regions_in_slice + i][2] = 2 * region_start_table[GEN9_HEVC_ENC_REGION_START_Y_OFFSET + nearest_reg];
4178                 temp_data[k * priv_state->num_regions_in_slice + i][3] = (unsigned int)((tmp_y != 0) ? tmp_y : (priv_state->picture_height) >> 4);
4179                 temp_data[k * priv_state->num_regions_in_slice + i][4] = offset_to_the_region_start[k] & 0x0FFFF;
4180                 temp_data[k * priv_state->num_regions_in_slice + i][5] = 0;
4181                 temp_data[k * priv_state->num_regions_in_slice + i][6] = 0;
4182                 temp_data[k * priv_state->num_regions_in_slice + i][7] = (unsigned int)(offset_y + 4 * region_start_table[GEN9_HEVC_ENC_REGION_START_Y_OFFSET + nearest_reg] + (4 * ((i * offset_delta) >> 1)));
4183             }
4184         }
4185     }
4186
4187     if (priv_state->walking_pattern_26)
4188         gen9_hevc_init_object_walker_26(priv_state, gpe_context, &param->gpe_param,
4189                                         priv_state->num_regions_in_slice, max_height,
4190                                         priv_state->use_hw_scoreboard,
4191                                         priv_state->use_hw_non_stalling_scoreborad);
4192     else
4193         gen9_hevc_init_object_walker_26z(priv_state, gpe_context, &param->gpe_param,
4194                                          priv_state->num_regions_in_slice, max_height,
4195                                          priv_state->use_hw_scoreboard,
4196                                          priv_state->use_hw_non_stalling_scoreborad);
4197
4198     p_region = (gen9_hevc_mbenc_control_region *)i965_map_gpe_resource(&priv_ctx->res_con_corrent_thread_buffer);
4199     if (!p_region)
4200         return;
4201
4202     memset((void *)p_region, 0, sizeof(*p_region) * GEN9_HEVC_ENC_CONCURRENT_SURFACE_HEIGHT);
4203
4204     for (i = 0; i < 1024 ; i += 64)
4205         memcpy(((unsigned char *)p_region) + i, (unsigned char *)temp_data[i / 64], 32);
4206
4207     param->max_height_in_region = priv_state->walking_pattern_26 ? max_height : max_height * 2;;
4208     param->num_region = num_regions;
4209     param->num_units_in_region = (frame_width_in_units + 2 * (max_height - 1) + priv_state->num_regions_in_slice - 1) /
4210                                  priv_state->num_regions_in_slice;
4211
4212     i965_unmap_gpe_resource(&priv_ctx->res_con_corrent_thread_buffer);
4213 }
4214
4215 static const char hevc_qpc_table[22] = {
4216     29, 30, 31, 32, 32, 33, 34, 34, 35, 35, 36, 36, 37, 37, 37, 38, 38, 38, 39, 39, 39, 39
4217 };
4218
4219 static void
4220 gen9_hevc_8x8_b_mbenc_set_curbe(VADriverContextP ctx,
4221                                 struct encode_state *encode_state,
4222                                 struct intel_encoder_context *encoder_context,
4223                                 struct i965_gpe_context *gpe_context,
4224                                 struct gen9_hevc_walking_pattern_parameter *param)
4225 {
4226     struct encoder_vme_mfc_context *vme_context = NULL;
4227     struct gen9_hevc_encoder_context *priv_ctx = NULL;
4228     struct generic_enc_codec_state *generic_state = NULL;
4229     struct gen9_hevc_encoder_state *priv_state = NULL;
4230     gen9_hevc_mbenc_b_mb_enc_curbe_data *cmd = NULL;
4231     VAEncSequenceParameterBufferHEVC *seq_param = NULL;
4232     VAEncPictureParameterBufferHEVC *pic_param = NULL;
4233     VAEncSliceParameterBufferHEVC *slice_param = NULL;
4234     unsigned int slice_qp = 0, slice_type = HEVC_SLICE_I;
4235     unsigned char mode_cost[12], mv_cost[8], mode_cost_sp;
4236     unsigned char forward_trans_thd[7];
4237     unsigned int simplest_intra_inter_threshold;
4238     int transform_8x8_mode_flag = 1;
4239     int qp_bd_offset_c, q_pi, qp_c;
4240     void *default_curbe_ptr = NULL;
4241     int default_curbe_size = 0;
4242     int max_sp_len = 57;
4243     int bti_idx = 0;
4244
4245     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
4246     priv_ctx = (struct gen9_hevc_encoder_context *)vme_context->private_enc_ctx;
4247     generic_state = (struct generic_enc_codec_state *)vme_context->generic_enc_state;
4248     priv_state = (struct gen9_hevc_encoder_state *)vme_context->private_enc_state;
4249     seq_param = (VAEncSequenceParameterBufferHEVC *)encode_state->seq_param_ext->buffer;
4250     pic_param = (VAEncPictureParameterBufferHEVC *)encode_state->pic_param_ext->buffer;
4251     slice_param = (VAEncSliceParameterBufferHEVC *)encode_state->slice_params_ext[0]->buffer;
4252
4253     slice_qp = pic_param->pic_init_qp + slice_param->slice_qp_delta;
4254     slice_type = priv_state->picture_coding_type;
4255
4256     if (priv_state->tu_mode == HEVC_TU_BEST_SPEED) {
4257         gen9_hevc_set_lambda_tables(priv_ctx, slice_type,
4258                                     HEVC_ENC_INTRA_TRANS_HAAR);
4259
4260         max_sp_len = 25;
4261     }
4262
4263     gen9_hevc_mbenc_set_costs(priv_ctx, slice_type, HEVC_ENC_INTRA_TRANS_REGULAR, slice_qp,
4264                               mode_cost, mv_cost, &mode_cost_sp,
4265                               &simplest_intra_inter_threshold);
4266
4267     gen9_hevc_set_forward_coeff_thd(forward_trans_thd, slice_qp);
4268
4269     gen9_hevc_get_b_mbenc_default_curbe(priv_state->tu_mode,
4270                                         slice_type, &default_curbe_ptr,
4271                                         &default_curbe_size);
4272
4273     cmd = i965_gpe_context_map_curbe(gpe_context);
4274     if (!cmd)
4275         return;
4276
4277     memcpy((void *)cmd, default_curbe_ptr, default_curbe_size);
4278
4279     cmd->dw0.adaptive_en = 1;
4280     cmd->dw0.t_8x8_flag_for_inter_en = transform_8x8_mode_flag;
4281     cmd->dw2.pic_width = priv_state->width_in_mb;
4282     cmd->dw2.len_sp = max_sp_len;
4283     cmd->dw3.src_access = 0;
4284     cmd->dw3.ref_access = 0;
4285     cmd->dw3.ft_enable = (HEVC_ENC_FTQ_BASED_SKIP[priv_state->tu_mode] >> 1) & 0x01;
4286     cmd->dw4.pic_height_minus1 = priv_state->height_in_mb - 1;
4287     cmd->dw4.hme_enable = !!generic_state->hme_enabled;
4288     cmd->dw4.slice_type = slice_type;
4289     cmd->dw4.use_actual_ref_qp_value = 0;
4290     cmd->dw6.frame_width = priv_state->width_in_mb * 16;
4291     cmd->dw6.frame_height = priv_state->height_in_mb * 16;
4292     cmd->dw7.intra_part_mask = 3;
4293     cmd->dw8.mode0_cost = mode_cost[0];
4294     cmd->dw8.mode1_cost = mode_cost[1];
4295     cmd->dw8.mode2_cost = mode_cost[2];
4296     cmd->dw8.mode3_cost = mode_cost[3];
4297     cmd->dw9.mode4_cost = mode_cost[4];
4298     cmd->dw9.mode5_cost = mode_cost[5];
4299     cmd->dw9.mode6_cost = mode_cost[6];
4300     cmd->dw9.mode7_cost = mode_cost[7];
4301     cmd->dw10.mode8_cost = mode_cost[8];
4302     cmd->dw10.mode9_cost = mode_cost[9];
4303     cmd->dw10.ref_id_cost = mode_cost[10];
4304     cmd->dw10.chroma_intra_mode_cost = mode_cost[11];
4305     cmd->dw11.mv0_cost = mv_cost[0];
4306     cmd->dw11.mv1_cost = mv_cost[1];
4307     cmd->dw11.mv2_cost = mv_cost[2];
4308     cmd->dw11.mv3_cost = mv_cost[3];
4309     cmd->dw12.mv4_cost = mv_cost[4];
4310     cmd->dw12.mv5_cost = mv_cost[5];
4311     cmd->dw12.mv6_cost = mv_cost[6];
4312     cmd->dw12.mv7_cost = mv_cost[7];
4313
4314     cmd->dw13.qp_prime_y = slice_qp;
4315     qp_bd_offset_c = 6 * priv_state->bit_depth_chroma_minus8;
4316     q_pi = CLAMP(-qp_bd_offset_c, 51, slice_qp + pic_param->pps_cb_qp_offset);
4317     qp_c = (q_pi < 30) ? q_pi : hevc_qpc_table[q_pi - 30];
4318     cmd->dw13.qp_prime_cb = qp_c;
4319     q_pi = CLAMP(-qp_bd_offset_c, 51, slice_qp + pic_param->pps_cr_qp_offset);
4320     qp_c = (q_pi < 30) ? q_pi : hevc_qpc_table[q_pi - 30];
4321     cmd->dw13.qp_prime_cr = qp_c;
4322
4323     cmd->dw14.sic_fwd_trans_coeff_thread_0 = forward_trans_thd[0];
4324     cmd->dw14.sic_fwd_trans_coeff_thread_1 = forward_trans_thd[1];
4325     cmd->dw14.sic_fwd_trans_coeff_thread_2 = forward_trans_thd[2];
4326     cmd->dw15.sic_fwd_trans_coeff_thread_3 = forward_trans_thd[3];
4327     cmd->dw15.sic_fwd_trans_coeff_thread_4 = forward_trans_thd[4];
4328     cmd->dw15.sic_fwd_trans_coeff_thread_5 = forward_trans_thd[5];
4329     cmd->dw15.sic_fwd_trans_coeff_thread_6 = forward_trans_thd[6];
4330     cmd->dw32.skip_val =
4331         HEVC_ENC_SKIPVAL_B[cmd->dw3.block_based_skip_enable][transform_8x8_mode_flag][slice_qp];
4332
4333     if (priv_state->picture_coding_type == HEVC_SLICE_I)
4334         *(float *)&cmd->dw34.lambda_me = 0.0;
4335     else
4336         *(float *)&cmd->dw34.lambda_me = (float)priv_ctx->lambda_me_table[slice_type][slice_qp];
4337
4338     cmd->dw35.mode_cost_sp = mode_cost_sp;
4339     cmd->dw35.simp_intra_inter_threashold = simplest_intra_inter_threshold;
4340     cmd->dw36.num_refidx_l0_minus_one = slice_param->num_ref_idx_l0_active_minus1;
4341     cmd->dw36.num_refidx_l1_minus_one = slice_param->num_ref_idx_l1_active_minus1;
4342     cmd->dw36.brc_enable = !!generic_state->brc_enabled;
4343     cmd->dw36.lcu_brc_enable = !!priv_state->lcu_brc_enabled;
4344     cmd->dw36.power_saving = priv_state->power_saving;
4345     cmd->dw36.roi_enable = (priv_state->num_roi > 0);
4346     cmd->dw36.fast_surveillance_flag = priv_state->picture_coding_type == HEVC_SLICE_I ?
4347                                        0 : priv_state->video_surveillance_flag;
4348
4349     if (priv_state->picture_coding_type != HEVC_SLICE_I) {
4350         cmd->dw37.actual_qp_refid0_list0 = gen9_hevc_get_qp_from_ref_list(ctx, slice_param, 0, 0);
4351         cmd->dw37.actual_qp_refid1_list0 = gen9_hevc_get_qp_from_ref_list(ctx, slice_param, 0, 1);
4352         cmd->dw37.actual_qp_refid2_list0 = gen9_hevc_get_qp_from_ref_list(ctx, slice_param, 0, 2);
4353         cmd->dw37.actual_qp_refid3_list0 = gen9_hevc_get_qp_from_ref_list(ctx, slice_param, 0, 3);
4354
4355         if (priv_state->picture_coding_type == HEVC_SLICE_B) {
4356             cmd->dw39.actual_qp_refid0_list1 = gen9_hevc_get_qp_from_ref_list(ctx, slice_param, 1, 0);
4357             cmd->dw39.actual_qp_refid1_list1 = gen9_hevc_get_qp_from_ref_list(ctx, slice_param, 1, 1);
4358         }
4359     }
4360
4361     cmd->dw44.max_vmvr = 511 * 4;
4362     cmd->dw44.max_num_merge_candidates = slice_param->max_num_merge_cand;
4363
4364     if (priv_state->picture_coding_type != HEVC_SLICE_I) {
4365         cmd->dw44.max_num_ref_list0 = cmd->dw36.num_refidx_l0_minus_one + 1;
4366         cmd->dw44.max_num_ref_list1 = cmd->dw36.num_refidx_l1_minus_one + 1;
4367         cmd->dw45.temporal_mvp_enable_flag = slice_param->slice_fields.bits.slice_temporal_mvp_enabled_flag;
4368         cmd->dw45.hme_combine_len_pslice = 8;
4369         if (priv_state->picture_coding_type == HEVC_SLICE_B)
4370             cmd->dw45.hme_combine_len_bslice = 8;
4371     }
4372
4373     cmd->dw45.log2_parallel_merge_level = pic_param->log2_parallel_merge_level_minus2 + 2;
4374     cmd->dw46.log2_min_tu_size = seq_param->log2_min_transform_block_size_minus2 + 2;
4375     cmd->dw46.log2_max_tu_size = cmd->dw46.log2_min_tu_size + seq_param->log2_diff_max_min_transform_block_size;
4376     cmd->dw46.log2_min_cu_size = seq_param->log2_min_luma_coding_block_size_minus3 + 3;
4377     cmd->dw46.log2_max_cu_size = cmd->dw46.log2_min_cu_size + seq_param->log2_diff_max_min_luma_coding_block_size;
4378     cmd->dw47.num_regions_in_slice = priv_state->num_regions_in_slice;
4379     cmd->dw47.type_of_walking_pattern = priv_state->walking_pattern_26;
4380     cmd->dw47.chroma_flatness_check_flag = (priv_state->tu_mode == HEVC_TU_BEST_SPEED) ? 0 : 1;
4381     cmd->dw47.enable_intra_early_exit = (priv_state->tu_mode == HEVC_TU_RT_SPEED);
4382     cmd->dw47.skip_intra_krn_flag = (priv_state->tu_mode == HEVC_TU_BEST_SPEED);
4383     cmd->dw47.collocated_from_l0_flag = slice_param->slice_fields.bits.collocated_from_l0_flag;
4384     cmd->dw47.is_low_delay = priv_state->low_delay;
4385     cmd->dw47.screen_content_flag = !!pic_param->pic_fields.bits.screen_content_flag;
4386     cmd->dw47.multi_slice_flag = (encode_state->num_slice_params_ext > 1);
4387     cmd->dw47.arbitary_slice_flag = priv_state->arbitrary_num_mb_in_slice;
4388     cmd->dw47.num_region_minus1 = param->num_region - 1;
4389
4390     if (priv_state->picture_coding_type != HEVC_SLICE_I) {
4391         cmd->dw48.current_td_l0_0 = gen9_hevc_get_poc_diff_from_ref_list(pic_param, slice_param, 0, 0);
4392         cmd->dw48.current_td_l0_1 = gen9_hevc_get_poc_diff_from_ref_list(pic_param, slice_param, 0, 1);
4393         cmd->dw49.current_td_l0_2 = gen9_hevc_get_poc_diff_from_ref_list(pic_param, slice_param, 0, 2);
4394         cmd->dw49.current_td_l0_3 = gen9_hevc_get_poc_diff_from_ref_list(pic_param, slice_param, 0, 3);
4395
4396         if (priv_state->picture_coding_type == HEVC_SLICE_B) {
4397             cmd->dw50.current_td_l1_0 = gen9_hevc_get_poc_diff_from_ref_list(pic_param, slice_param, 1, 0);
4398             cmd->dw50.current_td_l1_1 = gen9_hevc_get_poc_diff_from_ref_list(pic_param, slice_param, 1, 1);
4399         }
4400     }
4401
4402     cmd->dw52.num_of_units_in_region = param->num_units_in_region;
4403     cmd->dw52.max_height_in_region = param->max_height_in_region;
4404
4405     if (priv_state->rolling_intra_refresh) {
4406         cmd->dw35.widi_intra_refresh_en = 1;
4407         cmd->dw35.widi_first_intra_refresh = priv_state->widi_first_intra_refresh;
4408         cmd->dw35.half_update_mixed_lcu = 0;
4409         cmd->dw35.enable_rolling_intra = 1;
4410         cmd->dw38.widi_num_frame_in_gob = priv_state->widi_frame_num_in_gob;
4411         cmd->dw38.widi_num_intra_refresh_off_frames = priv_state->widi_frame_num_without_intra_refresh;
4412         cmd->dw51.widi_intra_refresh_qp_delta = priv_state->widi_intra_refresh_qp_delta;
4413         cmd->dw51.widi_intra_refresh_mb_num = priv_state->widi_intra_insertion_location;
4414         cmd->dw51.widi_intra_refresh_unit_in_mb = priv_state->widi_intra_insertion_size;
4415         cmd->dw53.widi_intra_refresh_ref_height = 40;
4416         cmd->dw53.widi_intra_refresh_ref_width = 48;
4417
4418         priv_state->widi_first_intra_refresh = 0;
4419         priv_state->widi_frame_num_without_intra_refresh = 0;
4420     } else if (priv_state->picture_coding_type != HEVC_SLICE_I)
4421         priv_state->widi_frame_num_without_intra_refresh++;
4422
4423     cmd->dw56.bti_cu_record = bti_idx++;
4424     cmd->dw57.bti_pak_cmd = bti_idx++;
4425     cmd->dw58.bti_src_y = bti_idx++;
4426     bti_idx++;
4427     cmd->dw59.bti_intra_dist = bti_idx++;
4428     cmd->dw60.bti_min_dist = bti_idx++;
4429     cmd->dw61.bti_hme_mv_pred_fwd_bwd_surf_index = bti_idx++;
4430     cmd->dw62.bti_hme_dist_surf_index = bti_idx++;
4431     cmd->dw63.bti_slice_map = bti_idx++;
4432     cmd->dw64.bti_vme_saved_uni_sic = bti_idx++;
4433     cmd->dw65.bti_simplest_intra = bti_idx++;
4434     cmd->dw66.bti_collocated_refframe = bti_idx++;
4435     cmd->dw67.bti_reserved = bti_idx++;
4436     cmd->dw68.bti_brc_input = bti_idx++;
4437     cmd->dw69.bti_lcu_qp = bti_idx++;
4438     cmd->dw70.bti_brc_data = bti_idx++;
4439     cmd->dw71.bti_vme_inter_prediction_surf_index = bti_idx++;
4440     bti_idx += 16;
4441
4442     if (priv_state->picture_coding_type == HEVC_SLICE_P) {
4443         cmd->dw72.bti_concurrent_thread_map = bti_idx++;
4444         cmd->dw73.bti_mb_data_cur_frame = bti_idx++;
4445         cmd->dw74.bti_mvp_cur_frame = bti_idx++;
4446         cmd->dw75.bti_debug = bti_idx++;
4447     } else {
4448         cmd->dw72.bti_vme_inter_prediction_b_surf_index = bti_idx++;
4449         bti_idx += 8;
4450
4451         cmd->dw73.bti_concurrent_thread_map = bti_idx++;
4452         cmd->dw74.bti_mb_data_cur_frame = bti_idx++;
4453         cmd->dw75.bti_mvp_cur_frame = bti_idx++;
4454         cmd->dw76.bti_debug = bti_idx++;
4455     }
4456
4457     i965_gpe_context_unmap_curbe(gpe_context);
4458 }
4459
4460 static void
4461 gen9_hevc_8x8_b_mbenc_set_surfaces(VADriverContextP ctx,
4462                                    struct encode_state *encode_state,
4463                                    struct intel_encoder_context *encoder_context,
4464                                    struct i965_gpe_context *gpe_context)
4465 {
4466     struct i965_driver_data *i965 = i965_driver_data(ctx);
4467     struct encoder_vme_mfc_context *vme_context = NULL;
4468     struct gen9_hevc_encoder_context *priv_ctx = NULL;
4469     struct gen9_hevc_encoder_state *priv_state = NULL;
4470     VAEncSliceParameterBufferHEVC *slice_param = NULL;
4471     VAEncPictureParameterBufferHEVC *pic_param = NULL;
4472     struct gen9_hevc_surface_priv *surface_priv = NULL;
4473     struct object_surface *obj_surface = NULL;
4474     dri_bo *collocated_mv_temporal_bo = NULL;
4475     int bti_idx = 0;
4476     int size = 0;
4477     int i = 0;
4478
4479     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
4480     priv_ctx = (struct gen9_hevc_encoder_context *)vme_context->private_enc_ctx;
4481     priv_state = (struct gen9_hevc_encoder_state *)vme_context->private_enc_state;
4482     pic_param = (VAEncPictureParameterBufferHEVC *)encode_state->pic_param_ext->buffer;
4483     slice_param = (VAEncSliceParameterBufferHEVC *)encode_state->slice_params_ext[0]->buffer;
4484
4485     if (priv_state->picture_coding_type != HEVC_SLICE_I &&
4486         slice_param->slice_fields.bits.slice_temporal_mvp_enabled_flag &&
4487         slice_param->slice_fields.bits.collocated_from_l0_flag) {
4488         if (pic_param->collocated_ref_pic_index != 0xff &&
4489             pic_param->collocated_ref_pic_index < GEN9_MAX_REF_SURFACES) {
4490             VASurfaceID idx = VA_INVALID_SURFACE;
4491
4492             idx = pic_param->reference_frames[pic_param->collocated_ref_pic_index].picture_id;
4493             obj_surface = SURFACE(idx);
4494             if (obj_surface) {
4495                 surface_priv = (struct gen9_hevc_surface_priv *)obj_surface->private_data;
4496                 if (surface_priv)
4497                     collocated_mv_temporal_bo = surface_priv->motion_vector_temporal_bo;
4498             }
4499         }
4500     }
4501
4502     size = priv_state->width_in_cu * priv_state->height_in_cu *
4503            priv_state->cu_record_size;
4504     gen9_hevc_set_gpe_1d_surface(ctx, priv_ctx, gpe_context,
4505                                  HEVC_ENC_SURFACE_CU_RECORD, bti_idx++,
4506                                  0, size, priv_state->mb_data_offset, NULL, NULL);
4507
4508     size = priv_state->mb_data_offset;
4509     gen9_hevc_set_gpe_1d_surface(ctx, priv_ctx, gpe_context,
4510                                  HEVC_ENC_SURFACE_HCP_PAK, bti_idx++,
4511                                  0, size, 0, NULL, NULL);
4512
4513     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
4514                                  HEVC_ENC_SURFACE_RAW_Y_UV, bti_idx++,
4515                                  1, 1, I965_SURFACEFORMAT_R8_UNORM,
4516                                  NULL, NULL);
4517     bti_idx++;
4518
4519     gen9_hevc_set_gpe_1d_surface(ctx, priv_ctx, gpe_context,
4520                                  HEVC_ENC_SURFACE_INTRA_DIST, bti_idx++,
4521                                  0, 0, 0, NULL, NULL);
4522
4523     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
4524                                  HEVC_ENC_SURFACE_MIN_DIST, bti_idx++,
4525                                  0, 1, I965_SURFACEFORMAT_R8_UNORM,
4526                                  NULL, NULL);
4527
4528     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
4529                                  HEVC_ENC_SURFACE_HME_MVP, bti_idx++,
4530                                  0, 1, I965_SURFACEFORMAT_R8_UNORM,
4531                                  NULL, NULL);
4532
4533     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
4534                                  HEVC_ENC_SURFACE_HME_DIST, bti_idx++,
4535                                  0, 1, I965_SURFACEFORMAT_R8_UNORM,
4536                                  NULL, NULL);
4537
4538     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
4539                                  HEVC_ENC_SURFACE_SLICE_MAP, bti_idx++,
4540                                  0, 1, I965_SURFACEFORMAT_R8_UNORM,
4541                                  NULL, NULL);
4542
4543     gen9_hevc_set_gpe_1d_surface(ctx, priv_ctx, gpe_context,
4544                                  HEVC_ENC_SURFACE_VME_UNI_SIC_DATA, bti_idx++,
4545                                  0, 0, 0, NULL, NULL);
4546
4547     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
4548                                  HEVC_ENC_SURFACE_SIMPLIFIED_INTRA, bti_idx++,
4549                                  0, 1, I965_SURFACEFORMAT_R8_UNORM,
4550                                  NULL, NULL);
4551
4552     if (collocated_mv_temporal_bo)
4553         gen9_hevc_set_gpe_1d_surface(ctx, priv_ctx, gpe_context,
4554                                      HEVC_ENC_SURFACE_COL_MB_MV, bti_idx++,
4555                                      0, 0, 0, NULL, collocated_mv_temporal_bo);
4556     else
4557         bti_idx++;
4558
4559     bti_idx++;
4560
4561     gen9_hevc_set_gpe_1d_surface(ctx, priv_ctx, gpe_context,
4562                                  HEVC_ENC_SURFACE_BRC_INPUT, bti_idx++,
4563                                  0, 0, 0, NULL, NULL);
4564
4565     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
4566                                  HEVC_ENC_SURFACE_LCU_QP, bti_idx++,
4567                                  0, 1, I965_SURFACEFORMAT_R8_UNORM,
4568                                  NULL, NULL);
4569
4570     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
4571                                  HEVC_ENC_SURFACE_BRC_DATA, bti_idx++,
4572                                  0, 1, I965_SURFACEFORMAT_R8_UNORM,
4573                                  NULL, NULL);
4574
4575     gen9_hevc_set_gpe_adv_surface(ctx, priv_ctx, gpe_context,
4576                                   HEVC_ENC_SURFACE_RAW_VME, bti_idx++,
4577                                   NULL);
4578
4579     for (i = 0; i < 8; i++) {
4580         if (i <= slice_param->num_ref_idx_l0_active_minus1)
4581             obj_surface = SURFACE(slice_param->ref_pic_list0[i].picture_id);
4582         else
4583             obj_surface = NULL;
4584
4585         if (obj_surface) {
4586             surface_priv = (struct gen9_hevc_surface_priv *)obj_surface->private_data;
4587
4588             gen9_hevc_set_gpe_adv_surface(ctx, priv_ctx, gpe_context,
4589                                           HEVC_ENC_SURFACE_REF_FRAME_VME, bti_idx++,
4590                                           surface_priv->surface_reff);
4591         } else
4592             bti_idx++;
4593
4594         if (i <= slice_param->num_ref_idx_l1_active_minus1)
4595             obj_surface = SURFACE(slice_param->ref_pic_list1[i].picture_id);
4596         else
4597             obj_surface = NULL;
4598
4599         if (obj_surface) {
4600             surface_priv = (struct gen9_hevc_surface_priv *)obj_surface->private_data;
4601
4602             gen9_hevc_set_gpe_adv_surface(ctx, priv_ctx, gpe_context,
4603                                           HEVC_ENC_SURFACE_REF_FRAME_VME, bti_idx++,
4604                                           surface_priv->surface_reff);
4605         } else
4606             bti_idx++;
4607     }
4608
4609     if (priv_state->picture_coding_type != HEVC_SLICE_P) {
4610         gen9_hevc_set_gpe_adv_surface(ctx, priv_ctx, gpe_context,
4611                                       HEVC_ENC_SURFACE_RAW_VME, bti_idx++,
4612                                       NULL);
4613
4614         for (i = 0; i < 4; i++) {
4615             if (i <= slice_param->num_ref_idx_l1_active_minus1)
4616                 obj_surface = SURFACE(slice_param->ref_pic_list1[i].picture_id);
4617             else
4618                 obj_surface = NULL;
4619
4620             if (obj_surface) {
4621                 surface_priv = (struct gen9_hevc_surface_priv *)obj_surface->private_data;
4622
4623                 gen9_hevc_set_gpe_adv_surface(ctx, priv_ctx, gpe_context,
4624                                               HEVC_ENC_SURFACE_REF_FRAME_VME, bti_idx++,
4625                                               surface_priv->surface_reff);
4626             } else
4627                 bti_idx++;
4628
4629             bti_idx++;
4630         }
4631     }
4632
4633     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
4634                                  HEVC_ENC_SURFACE_CONCURRENT_THREAD, bti_idx++,
4635                                  0, 1, I965_SURFACEFORMAT_R8_UNORM,
4636                                  NULL, NULL);
4637
4638     gen9_hevc_set_gpe_1d_surface(ctx, priv_ctx, gpe_context,
4639                                  HEVC_ENC_SURFACE_MB_MV_INDEX, bti_idx++,
4640                                  0, 0, 0, NULL, NULL);
4641
4642     gen9_hevc_set_gpe_1d_surface(ctx, priv_ctx, gpe_context,
4643                                  HEVC_ENC_SURFACE_MVP_INDEX, bti_idx++,
4644                                  0, 0, 0, NULL, NULL);
4645
4646     gen9_hevc_set_gpe_1d_surface(ctx, priv_ctx, gpe_context,
4647                                  HEVC_ENC_SURFACE_KERNEL_DEBUG, bti_idx++,
4648                                  0, 0, 0, NULL, NULL);
4649 }
4650
4651 static void
4652 gen9_hevc_8x8_b_mbenc(VADriverContextP ctx,
4653                       struct encode_state *encode_state,
4654                       struct intel_encoder_context *encoder_context)
4655 {
4656     struct encoder_vme_mfc_context *vme_context = NULL;
4657     struct gen9_hevc_encoder_context *priv_ctx = NULL;
4658     struct gen9_hevc_encoder_state *priv_state = NULL;
4659     struct gen9_hevc_walking_pattern_parameter param;
4660     struct i965_gpe_context *gpe_context = NULL;
4661     int media_state = HEVC_ENC_MEDIA_STATE_HEVC_B_MBENC;
4662     int gpe_idx = HEVC_MBENC_BENC_IDX;
4663
4664     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
4665     priv_ctx = (struct gen9_hevc_encoder_context *)vme_context->private_enc_ctx;
4666     priv_state = (struct gen9_hevc_encoder_state *)vme_context->private_enc_state;
4667
4668     if (priv_state->picture_coding_type == HEVC_SLICE_P)
4669         gpe_idx = priv_state->rolling_intra_refresh ? HEVC_MBENC_P_WIDI_IDX : HEVC_MBENC_PENC_IDX;
4670     else if (priv_state->picture_coding_type != HEVC_SLICE_I)
4671         gpe_idx = priv_state->rolling_intra_refresh ? HEVC_MBENC_MBENC_WIDI_IDX : HEVC_MBENC_BENC_IDX;
4672
4673     gpe_context = &priv_ctx->mbenc_context.gpe_contexts[gpe_idx];
4674
4675     gen8_gpe_context_init(ctx, gpe_context);
4676     gen9_gpe_reset_binding_table(ctx, gpe_context);
4677
4678     memset((void *)&param, 0, sizeof(param));
4679     gen9_hevc_set_control_region(ctx, encode_state, encoder_context, gpe_context, &param);
4680
4681     gen9_hevc_8x8_b_mbenc_set_curbe(ctx, encode_state, encoder_context, gpe_context, &param);
4682     gen9_hevc_8x8_b_mbenc_set_surfaces(ctx, encode_state, encoder_context, gpe_context);
4683     gen8_gpe_setup_interface_data(ctx, gpe_context);
4684
4685     param.gpe_param.color_count_minus1 = param.num_region - 1;
4686     gen9_hevc_run_object_walker(ctx, encoder_context, gpe_context, &param.gpe_param,
4687                                 media_state);
4688 }
4689
4690 static void
4691 gen9_hevc_8x8_pu_fmode_set_curbe(VADriverContextP ctx,
4692                                  struct encode_state *encode_state,
4693                                  struct intel_encoder_context *encoder_context,
4694                                  struct i965_gpe_context *gpe_context)
4695 {
4696     struct i965_driver_data *i965 = i965_driver_data(ctx);
4697     struct encoder_vme_mfc_context *vme_context = NULL;
4698     struct gen9_hevc_encoder_context *priv_ctx = NULL;
4699     struct generic_enc_codec_state *generic_state = NULL;
4700     struct gen9_hevc_encoder_state *priv_state = NULL;
4701     gen9_hevc_mbenc_8x8_pu_fmode_curbe_data *cmd = NULL;
4702     VAEncSequenceParameterBufferHEVC *seq_param = NULL;
4703     VAEncPictureParameterBufferHEVC *pic_param = NULL;
4704     VAEncSliceParameterBufferHEVC *slice_param = NULL;
4705     unsigned int slice_qp = 0;
4706     int bti_idx = 0;
4707
4708     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
4709     priv_ctx = (struct gen9_hevc_encoder_context *)vme_context->private_enc_ctx;
4710     generic_state = (struct generic_enc_codec_state *)vme_context->generic_enc_state;
4711     priv_state = (struct gen9_hevc_encoder_state *)vme_context->private_enc_state;
4712     seq_param = (VAEncSequenceParameterBufferHEVC *)encode_state->seq_param_ext->buffer;
4713     pic_param = (VAEncPictureParameterBufferHEVC *)encode_state->pic_param_ext->buffer;
4714     slice_param = (VAEncSliceParameterBufferHEVC *)encode_state->slice_params_ext[0]->buffer;
4715
4716     slice_qp = pic_param->pic_init_qp + slice_param->slice_qp_delta;
4717
4718     cmd = i965_gpe_context_map_curbe(gpe_context);
4719     if (!cmd)
4720         return;
4721
4722     memset((void *)cmd, 0, sizeof(*cmd));
4723
4724     cmd->dw0.frame_width = ALIGN(priv_state->picture_width, 16);
4725     cmd->dw0.frame_height = ALIGN(priv_state->picture_height, 16);
4726     cmd->dw1.slice_type = priv_state->picture_coding_type;
4727     cmd->dw1.pu_type = 2;
4728     cmd->dw1.pak_reording_flag = priv_state->picture_coding_type == HEVC_SLICE_I ? 1 : 0;
4729
4730     if (seq_param->log2_min_luma_coding_block_size_minus3 + 3 +
4731         seq_param->log2_diff_max_min_luma_coding_block_size == 6)
4732         cmd->dw1.lcu_type = 0;
4733     else
4734         cmd->dw1.lcu_type = 1;
4735
4736     cmd->dw1.screen_content_flag = !!pic_param->pic_fields.bits.screen_content_flag;
4737     cmd->dw1.enable_intra_early_exit = priv_state->tu_mode == HEVC_TU_RT_SPEED ?
4738                                        (priv_state->picture_coding_type == HEVC_SLICE_I ? 0 : 1) : 0;
4739     cmd->dw1.enable_debug_dump = 0;
4740     cmd->dw1.brc_enable = generic_state->brc_enabled;
4741     cmd->dw1.lcu_brc_enable = priv_state->lcu_brc_enabled;
4742     cmd->dw1.roi_enable = (priv_state->num_roi > 0);
4743     cmd->dw1.fast_surveillance_flag = priv_state->picture_coding_type == HEVC_SLICE_I ?
4744                                       0 : priv_state->video_surveillance_flag;
4745     cmd->dw1.enable_rolling_intra = priv_state->rolling_intra_refresh;
4746     cmd->dw1.widi_intra_refresh_en = priv_state->rolling_intra_refresh;
4747     cmd->dw1.half_update_mixed_lcu = 0;
4748     cmd->dw2.luma_lambda = priv_state->fixed_point_lambda_for_luma;
4749
4750     if (priv_state->picture_coding_type != HEVC_SLICE_I) {
4751         double lambda_md;
4752         float had_bias = 2.0f;
4753
4754         lambda_md = priv_ctx->lambda_md_table[cmd->dw1.slice_type][slice_qp];
4755         lambda_md = lambda_md * had_bias;
4756         cmd->dw3.lambda_for_dist_calculation = (unsigned int)(lambda_md * (1 << 10));
4757     }
4758
4759     cmd->dw4.mode_cost_for_8x8_pu_tu8 = 0;
4760     cmd->dw5.mode_cost_for_8x8_pu_tu4 = 0;
4761     cmd->dw6.satd_16x16_pu_threshold = MAX(200 * ((int)slice_qp - 12), 0);
4762     cmd->dw6.bias_factor_toward_8x8 = pic_param->pic_fields.bits.screen_content_flag ? 1024 : 1126 + 102;
4763     cmd->dw7.qp = slice_qp;
4764     cmd->dw7.qp_for_inter = 0;
4765     cmd->dw8.simplified_flag_for_inter = 0;
4766     cmd->dw8.kbl_control_flag = (IS_KBL(i965->intel.device_info) || IS_GLK(i965->intel.device_info));
4767     cmd->dw9.widi_intra_refresh_qp_delta = priv_state->widi_intra_refresh_qp_delta;
4768     cmd->dw9.widi_intra_refresh_mb_num = priv_state->widi_intra_insertion_location;
4769     cmd->dw9.widi_intra_refresh_unit_in_mb = priv_state->widi_intra_insertion_size;
4770
4771     cmd->dw16.bti_pak_object = bti_idx++;
4772     cmd->dw17.bti_vme_8x8_mode = bti_idx++;
4773     cmd->dw18.bti_intra_mode = bti_idx++;
4774     cmd->dw19.bti_pak_command = bti_idx++;
4775     cmd->dw20.bti_slice_map = bti_idx++;
4776     cmd->dw21.bti_intra_dist = bti_idx++;
4777     cmd->dw22.bti_brc_input = bti_idx++;
4778     cmd->dw23.bti_simplest_intra = bti_idx++;
4779     cmd->dw24.bti_lcu_qp_surface = bti_idx++;
4780     cmd->dw25.bti_brc_data = bti_idx++;
4781     cmd->dw26.bti_debug = bti_idx++;
4782
4783     i965_gpe_context_unmap_curbe(gpe_context);
4784 }
4785
4786 static void
4787 gen9_hevc_8x8_pu_fmode_set_surfaces(VADriverContextP ctx,
4788                                     struct encode_state *encode_state,
4789                                     struct intel_encoder_context *encoder_context,
4790                                     struct i965_gpe_context *gpe_context)
4791 {
4792     struct encoder_vme_mfc_context *vme_context = NULL;
4793     struct gen9_hevc_encoder_context *priv_ctx = NULL;
4794     struct gen9_hevc_encoder_state *priv_state = NULL;
4795     int bti_idx = 0;
4796     int size = 0;
4797
4798     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
4799     priv_ctx = (struct gen9_hevc_encoder_context *)vme_context->private_enc_ctx;
4800     priv_state = (struct gen9_hevc_encoder_state *)vme_context->private_enc_state;
4801
4802     size = priv_state->width_in_cu * priv_state->height_in_cu *
4803            priv_state->cu_record_size;
4804     gen9_hevc_set_gpe_1d_surface(ctx, priv_ctx, gpe_context,
4805                                  HEVC_ENC_SURFACE_CU_RECORD, bti_idx++,
4806                                  0, size, priv_state->mb_data_offset, NULL, NULL);
4807
4808     gen9_hevc_set_gpe_1d_surface(ctx, priv_ctx, gpe_context,
4809                                  HEVC_ENC_SURFACE_VME_8x8, bti_idx++,
4810                                  0, 0, 0, NULL, NULL);
4811
4812     gen9_hevc_set_gpe_1d_surface(ctx, priv_ctx, gpe_context,
4813                                  HEVC_ENC_SURFACE_INTRA_MODE, bti_idx++,
4814                                  0, 0, 0, NULL, NULL);
4815
4816     size = priv_state->mb_data_offset;
4817     gen9_hevc_set_gpe_1d_surface(ctx, priv_ctx, gpe_context,
4818                                  HEVC_ENC_SURFACE_HCP_PAK, bti_idx++,
4819                                  0, size, 0, NULL, NULL);
4820
4821     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
4822                                  HEVC_ENC_SURFACE_SLICE_MAP, bti_idx++,
4823                                  0, 1, I965_SURFACEFORMAT_R8_UNORM,
4824                                  NULL, NULL);
4825
4826     gen9_hevc_set_gpe_1d_surface(ctx, priv_ctx, gpe_context,
4827                                  HEVC_ENC_SURFACE_INTRA_DIST, bti_idx++,
4828                                  0, 0, 0, NULL, NULL);
4829
4830     gen9_hevc_set_gpe_1d_surface(ctx, priv_ctx, gpe_context,
4831                                  HEVC_ENC_SURFACE_BRC_INPUT, bti_idx++,
4832                                  0, 0, 0, NULL, NULL);
4833
4834     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
4835                                  HEVC_ENC_SURFACE_SIMPLIFIED_INTRA, bti_idx++,
4836                                  0, 1, I965_SURFACEFORMAT_R8_UNORM,
4837                                  NULL, NULL);
4838
4839     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
4840                                  HEVC_ENC_SURFACE_LCU_QP, bti_idx++,
4841                                  0, 1, I965_SURFACEFORMAT_R8_UNORM,
4842                                  NULL, NULL);
4843
4844     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
4845                                  HEVC_ENC_SURFACE_BRC_DATA, bti_idx++,
4846                                  0, 1, I965_SURFACEFORMAT_R8_UNORM,
4847                                  NULL, NULL);
4848
4849     gen9_hevc_set_gpe_1d_surface(ctx, priv_ctx, gpe_context,
4850                                  HEVC_ENC_SURFACE_KERNEL_DEBUG, bti_idx++,
4851                                  0, 0, 0, NULL, NULL);
4852 }
4853
4854 static void
4855 gen9_hevc_8x8_pu_fmode(VADriverContextP ctx,
4856                        struct encode_state *encode_state,
4857                        struct intel_encoder_context *encoder_context)
4858 {
4859     struct encoder_vme_mfc_context *vme_context = NULL;
4860     struct gen9_hevc_encoder_context *priv_ctx = NULL;
4861     struct gen9_hevc_encoder_state *priv_state = NULL;
4862     struct gpe_media_object_walker_parameter param;
4863     struct hevc_enc_kernel_walker_parameter hevc_walker_param;
4864     struct i965_gpe_context *gpe_context = NULL;
4865     int media_state = HEVC_ENC_MEDIA_STATE_8x8_PU_FMODE;
4866
4867     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
4868     priv_ctx = (struct gen9_hevc_encoder_context *)vme_context->private_enc_ctx;
4869     priv_state = (struct gen9_hevc_encoder_state *)vme_context->private_enc_state;
4870
4871     gpe_context = &priv_ctx->mbenc_context.gpe_contexts[HEVC_MBENC_8x8FMODE_IDX];
4872
4873     gen8_gpe_context_init(ctx, gpe_context);
4874     gen9_gpe_reset_binding_table(ctx, gpe_context);
4875     gen9_hevc_8x8_pu_fmode_set_curbe(ctx, encode_state, encoder_context, gpe_context);
4876     gen9_hevc_8x8_pu_fmode_set_surfaces(ctx, encode_state, encoder_context, gpe_context);
4877     gen8_gpe_setup_interface_data(ctx, gpe_context);
4878
4879     memset((void *)&hevc_walker_param, 0, sizeof(hevc_walker_param));
4880     hevc_walker_param.resolution_x = priv_state->width_in_lcu;
4881     hevc_walker_param.resolution_y = priv_state->height_in_lcu;
4882     hevc_walker_param.no_dependency = 1;
4883     gen9_hevc_init_object_walker(&hevc_walker_param, &param);
4884     gen9_hevc_run_object_walker(ctx, encoder_context, gpe_context, &param,
4885                                 media_state);
4886 }
4887
4888 static void
4889 gen9_hevc_8x8_pu_mode_set_curbe(VADriverContextP ctx,
4890                                 struct encode_state *encode_state,
4891                                 struct intel_encoder_context *encoder_context,
4892                                 struct i965_gpe_context *gpe_context)
4893 {
4894     struct encoder_vme_mfc_context *vme_context = NULL;
4895     struct generic_enc_codec_state *generic_state = NULL;
4896     struct gen9_hevc_encoder_state *priv_state = NULL;
4897     gen9_hevc_mbenc_8x8_pu_curbe_data *cmd = NULL;
4898     VAEncSequenceParameterBufferHEVC *seq_param = NULL;
4899     VAEncPictureParameterBufferHEVC *pic_param = NULL;
4900     VAEncSliceParameterBufferHEVC *slice_param = NULL;
4901     int bti_idx = 0;
4902
4903     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
4904     generic_state = (struct generic_enc_codec_state *)vme_context->generic_enc_state;
4905     priv_state = (struct gen9_hevc_encoder_state *)vme_context->private_enc_state;
4906     seq_param = (VAEncSequenceParameterBufferHEVC *)encode_state->seq_param_ext->buffer;
4907     pic_param = (VAEncPictureParameterBufferHEVC *)encode_state->pic_param_ext->buffer;
4908     slice_param = (VAEncSliceParameterBufferHEVC *)encode_state->slice_params_ext[0]->buffer;
4909
4910     cmd = i965_gpe_context_map_curbe(gpe_context);
4911     if (!cmd)
4912         return;
4913
4914     memset((void *)cmd, 0, sizeof(*cmd));
4915
4916     cmd->dw0.frame_width = ALIGN(priv_state->picture_width, 16);
4917     cmd->dw0.frame_height = ALIGN(priv_state->picture_height, 16);
4918     cmd->dw1.slice_type = priv_state->picture_coding_type;
4919     cmd->dw1.pu_type = 2;
4920     cmd->dw1.dc_filter_flag = 1;
4921     cmd->dw1.angle_refine_flag = 1;
4922     if (seq_param->log2_min_luma_coding_block_size_minus3 + 3 +
4923         seq_param->log2_diff_max_min_luma_coding_block_size == 6)
4924         cmd->dw1.lcu_type = 0;
4925     else
4926         cmd->dw1.lcu_type = 1;
4927     cmd->dw1.screen_content_flag = !!pic_param->pic_fields.bits.screen_content_flag;
4928     cmd->dw1.enable_intra_early_exit = priv_state->tu_mode == HEVC_TU_RT_SPEED ?
4929                                        (priv_state->picture_coding_type == HEVC_SLICE_I ? 0 : 1) : 0;
4930     cmd->dw1.enable_debug_dump = 0;
4931     cmd->dw1.brc_enable = generic_state->brc_enabled;
4932     cmd->dw1.lcu_brc_enable = priv_state->lcu_brc_enabled;
4933     cmd->dw1.roi_enable = (priv_state->num_roi > 0);
4934     cmd->dw1.fast_surveillance_flag = priv_state->picture_coding_type == HEVC_SLICE_I ?
4935                                       0 : priv_state->video_surveillance_flag;
4936     if (priv_state->rolling_intra_refresh) {
4937         cmd->dw1.enable_rolling_intra = 1;
4938         cmd->dw1.widi_intra_refresh_en = 1;
4939         cmd->dw1.half_update_mixed_lcu = 0;
4940
4941         cmd->dw5.widi_intra_refresh_mb_num = priv_state->widi_intra_insertion_location;
4942         cmd->dw5.widi_intra_refresh_mb_num = priv_state->widi_intra_insertion_location;
4943         cmd->dw5.widi_intra_refresh_unit_in_mb = priv_state->widi_intra_insertion_size;
4944
4945         cmd->dw1.qp_value = pic_param->pic_init_qp + slice_param->slice_qp_delta;
4946     }
4947
4948     cmd->dw2.luma_lambda = priv_state->fixed_point_lambda_for_luma;
4949     cmd->dw3.chroma_lambda = priv_state->fixed_point_lambda_for_chroma;
4950     cmd->dw4.simplified_flag_for_inter = 0;
4951     cmd->dw4.harr_trans_form_flag = priv_state->picture_coding_type == HEVC_SLICE_I ? 0 : 1;
4952
4953     cmd->dw8.bti_src_y = bti_idx++;
4954     bti_idx++;
4955     cmd->dw9.bti_slice_map = bti_idx++;
4956     cmd->dw10.bti_vme_8x8_mode = bti_idx++;
4957     cmd->dw11.bti_intra_mode = bti_idx++;
4958     cmd->dw12.bti_brc_input = bti_idx++;
4959     cmd->dw13.bti_simplest_intra = bti_idx++;
4960     cmd->dw14.bti_lcu_qp_surface = bti_idx++;
4961     cmd->dw15.bti_brc_data = bti_idx++;
4962     cmd->dw16.bti_debug = bti_idx++;
4963
4964     i965_gpe_context_unmap_curbe(gpe_context);
4965 }
4966
4967 static void
4968 gen9_hevc_8x8_pu_mode_set_surfaces(VADriverContextP ctx,
4969                                    struct encode_state *encode_state,
4970                                    struct intel_encoder_context *encoder_context,
4971                                    struct i965_gpe_context *gpe_context)
4972 {
4973     struct encoder_vme_mfc_context *vme_context = NULL;
4974     struct gen9_hevc_encoder_context *priv_ctx = NULL;
4975     int bti_idx = 0;
4976
4977     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
4978     priv_ctx = (struct gen9_hevc_encoder_context *)vme_context->private_enc_ctx;
4979
4980     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
4981                                  HEVC_ENC_SURFACE_RAW_Y_UV, bti_idx++,
4982                                  1, 1, I965_SURFACEFORMAT_R8_UNORM,
4983                                  NULL, NULL);
4984     bti_idx++;
4985
4986     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
4987                                  HEVC_ENC_SURFACE_SLICE_MAP, bti_idx++,
4988                                  0, 1, I965_SURFACEFORMAT_R8_UNORM,
4989                                  NULL, NULL);
4990
4991     gen9_hevc_set_gpe_1d_surface(ctx, priv_ctx, gpe_context,
4992                                  HEVC_ENC_SURFACE_VME_8x8, bti_idx++,
4993                                  0, 0, 0, NULL, NULL);
4994
4995     gen9_hevc_set_gpe_1d_surface(ctx, priv_ctx, gpe_context,
4996                                  HEVC_ENC_SURFACE_INTRA_MODE, bti_idx++,
4997                                  0, 0, 0, NULL, NULL);
4998
4999     gen9_hevc_set_gpe_1d_surface(ctx, priv_ctx, gpe_context,
5000                                  HEVC_ENC_SURFACE_BRC_INPUT, bti_idx++,
5001                                  0, 0, 0, NULL, NULL);
5002
5003     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
5004                                  HEVC_ENC_SURFACE_SIMPLIFIED_INTRA, bti_idx++,
5005                                  0, 1, I965_SURFACEFORMAT_R8_UNORM,
5006                                  NULL, NULL);
5007
5008     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
5009                                  HEVC_ENC_SURFACE_LCU_QP, bti_idx++,
5010                                  0, 1, I965_SURFACEFORMAT_R8_UNORM,
5011                                  NULL, NULL);
5012
5013     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
5014                                  HEVC_ENC_SURFACE_BRC_DATA, bti_idx++,
5015                                  0, 1, I965_SURFACEFORMAT_R8_UNORM,
5016                                  NULL, NULL);
5017
5018     gen9_hevc_set_gpe_1d_surface(ctx, priv_ctx, gpe_context,
5019                                  HEVC_ENC_SURFACE_KERNEL_DEBUG, bti_idx++,
5020                                  0, 0, 0, NULL, NULL);
5021 }
5022
5023 static void
5024 gen9_hevc_8x8_pu_mode(VADriverContextP ctx,
5025                       struct encode_state *encode_state,
5026                       struct intel_encoder_context *encoder_context)
5027 {
5028     struct encoder_vme_mfc_context *vme_context = NULL;
5029     struct gen9_hevc_encoder_context *priv_ctx = NULL;
5030     struct gen9_hevc_encoder_state *priv_state = NULL;
5031     struct gpe_media_object_walker_parameter param;
5032     struct hevc_enc_kernel_walker_parameter hevc_walker_param;
5033     struct i965_gpe_context *gpe_context = NULL;
5034     int media_state = HEVC_ENC_MEDIA_STATE_8x8_PU;
5035
5036     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
5037     priv_ctx = (struct gen9_hevc_encoder_context *)vme_context->private_enc_ctx;
5038     priv_state = (struct gen9_hevc_encoder_state *)vme_context->private_enc_state;
5039
5040     gpe_context = &priv_ctx->mbenc_context.gpe_contexts[HEVC_MBENC_8x8PU_IDX];
5041
5042     gen8_gpe_context_init(ctx, gpe_context);
5043     gen9_gpe_reset_binding_table(ctx, gpe_context);
5044     gen9_hevc_8x8_pu_mode_set_curbe(ctx, encode_state, encoder_context, gpe_context);
5045     gen9_hevc_8x8_pu_mode_set_surfaces(ctx, encode_state, encoder_context, gpe_context);
5046     gen8_gpe_setup_interface_data(ctx, gpe_context);
5047
5048     memset((void *)&hevc_walker_param, 0, sizeof(hevc_walker_param));
5049     hevc_walker_param.resolution_x = ALIGN(priv_state->picture_width, 16) >> 3;
5050     hevc_walker_param.resolution_y = ALIGN(priv_state->picture_height, 16) >> 3;
5051     hevc_walker_param.no_dependency = 1;
5052     gen9_hevc_init_object_walker(&hevc_walker_param, &param);
5053     gen9_hevc_run_object_walker(ctx, encoder_context, gpe_context, &param,
5054                                 media_state);
5055 }
5056
5057 static void
5058 gen9_hevc_16x16_pu_mode_set_curbe(VADriverContextP ctx,
5059                                   struct encode_state *encode_state,
5060                                   struct intel_encoder_context *encoder_context,
5061                                   struct i965_gpe_context *gpe_context)
5062 {
5063     struct encoder_vme_mfc_context *vme_context = NULL;
5064     struct gen9_hevc_encoder_context *priv_ctx = NULL;
5065     struct generic_enc_codec_state *generic_state = NULL;
5066     struct gen9_hevc_encoder_state *priv_state = NULL;
5067     gen9_hevc_enc_16x16_pu_curbe_data *cmd = NULL;
5068     VAEncSequenceParameterBufferHEVC *seq_param = NULL;
5069     VAEncPictureParameterBufferHEVC *pic_param = NULL;
5070     VAEncSliceParameterBufferHEVC *slice_param = NULL;
5071     double squred_lambda, qp_lambda, lambda_scaling_factor;
5072     unsigned int slice_qp = 0, slice_type = HEVC_SLICE_I;
5073     unsigned int new_point_lambda_for_luma;
5074     unsigned char mode_cost[12];
5075     int bti_idx = 0;
5076
5077     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
5078     priv_ctx = (struct gen9_hevc_encoder_context *)vme_context->private_enc_ctx;
5079     generic_state = (struct generic_enc_codec_state *)vme_context->generic_enc_state;
5080     priv_state = (struct gen9_hevc_encoder_state *)vme_context->private_enc_state;
5081     seq_param = (VAEncSequenceParameterBufferHEVC *)encode_state->seq_param_ext->buffer;
5082     pic_param = (VAEncPictureParameterBufferHEVC *)encode_state->pic_param_ext->buffer;
5083     slice_param = (VAEncSliceParameterBufferHEVC *)encode_state->slice_params_ext[0]->buffer;
5084
5085     slice_qp = pic_param->pic_init_qp + slice_param->slice_qp_delta;
5086     slice_type = priv_state->picture_coding_type;
5087
5088     lambda_scaling_factor = 0.46 + slice_qp - 22;
5089     if (lambda_scaling_factor < 0)
5090         lambda_scaling_factor = 0.46;
5091     else if (lambda_scaling_factor > 15)
5092         lambda_scaling_factor = 15;
5093
5094     squred_lambda = lambda_scaling_factor * pow(2.0, ((double)slice_qp - 12.0) / 6);
5095     priv_state->fixed_point_lambda_for_luma = (unsigned int)(squred_lambda * (1 << 10));
5096
5097     lambda_scaling_factor = 1.0;
5098     qp_lambda = priv_ctx->lambda_md_table[slice_type][slice_qp];
5099     squred_lambda = qp_lambda * qp_lambda;
5100     priv_state->fixed_point_lambda_for_chroma = (unsigned int)(lambda_scaling_factor * squred_lambda * (1 << 10));
5101
5102     qp_lambda = sqrt(0.57 * pow(2.0, ((double)slice_qp - 12.0) / 3));
5103     squred_lambda = qp_lambda * qp_lambda;
5104     new_point_lambda_for_luma = (unsigned int)(squred_lambda * (1 << 10));
5105
5106     gen9_hevc_mbenc_set_costs(priv_ctx, slice_type, HEVC_ENC_INTRA_TRANS_HAAR, slice_qp,
5107                               mode_cost, NULL, NULL, NULL);
5108
5109     cmd = i965_gpe_context_map_curbe(gpe_context);
5110     if (!cmd)
5111         return;
5112
5113     memset((void *)cmd, 0, sizeof(*cmd));
5114
5115     cmd->dw0.frame_width = ALIGN(priv_state->picture_width, 16);
5116     cmd->dw0.frame_height = ALIGN(priv_state->picture_height, 16);
5117     cmd->dw1.log2_min_cu_size = seq_param->log2_min_luma_coding_block_size_minus3 + 3;
5118     cmd->dw1.log2_max_cu_size = cmd->dw1.log2_min_cu_size + seq_param->log2_diff_max_min_luma_coding_block_size;
5119     cmd->dw1.log2_min_tu_size = seq_param->log2_min_transform_block_size_minus2 + 2;
5120     cmd->dw1.slice_qp = slice_qp;
5121     cmd->dw2.fixed_point_lambda_pred_mode = priv_state->fixed_point_lambda_for_chroma;
5122     cmd->dw3.lambda_scaling_factor = 1;
5123     cmd->dw3.slice_type = slice_type;
5124     cmd->dw3.enable_intra_early_exit = priv_state->tu_mode == HEVC_TU_RT_SPEED ?
5125                                        (priv_state->picture_coding_type == HEVC_SLICE_I ? 0 : 1) : 0;
5126     cmd->dw3.brc_enable = !!generic_state->brc_enabled;
5127     cmd->dw3.lcu_brc_enable = !!priv_state->lcu_brc_enabled;
5128     cmd->dw3.roi_enable = (priv_state->num_roi > 0);
5129     cmd->dw3.fast_surveillance_flag = priv_state->picture_coding_type == HEVC_SLICE_I ?
5130                                       0 : priv_state->video_surveillance_flag;
5131     cmd->dw3.enable_rolling_intra = priv_state->rolling_intra_refresh;
5132     cmd->dw3.widi_intra_refresh_en = priv_state->rolling_intra_refresh;
5133     cmd->dw3.half_update_mixed_lcu = 0;
5134     cmd->dw4.penalty_for_intra_8x8_non_dc_pred_mode = 0;
5135     cmd->dw4.intra_compute_type = 1;
5136     cmd->dw4.avc_intra_8x8_mask = 0;
5137     cmd->dw4.intra_sad_adjust = 2;
5138     cmd->dw5.fixed_point_lambda_cu_mode_for_cost_calculation = new_point_lambda_for_luma;
5139     cmd->dw6.screen_content_flag = !!pic_param->pic_fields.bits.screen_content_flag;
5140     cmd->dw7.mode_cost_intra_non_pred = mode_cost[0];
5141     cmd->dw7.mode_cost_intra_16x16 = mode_cost[1];
5142     cmd->dw7.mode_cost_intra_8x8 = mode_cost[2];
5143     cmd->dw7.mode_cost_intra_4x4 = mode_cost[3];
5144     cmd->dw8.fixed_point_lambda_cu_mode_for_luma = priv_state->fixed_point_lambda_for_luma;
5145
5146     if (priv_state->rolling_intra_refresh) {
5147         cmd->dw9.widi_intra_refresh_mb_num = priv_state->widi_intra_insertion_location;
5148         cmd->dw9.widi_intra_refresh_mb_num = priv_state->widi_intra_insertion_location;
5149         cmd->dw9.widi_intra_refresh_unit_in_mb = priv_state->widi_intra_insertion_size;
5150     }
5151
5152     cmd->dw10.simplified_flag_for_inter = 0;
5153     cmd->dw10.haar_transform_mode = priv_state->picture_coding_type == HEVC_SLICE_I ? 0 : 1;
5154
5155     cmd->dw16.bti_src_y = bti_idx++;
5156     bti_idx++;
5157     cmd->dw17.bti_sad_16x16_pu = bti_idx++;
5158     cmd->dw18.bti_pak_object = bti_idx++;
5159     cmd->dw19.bti_sad_32x32_pu_mode = bti_idx++;
5160     cmd->dw20.bti_vme_mode_8x8 = bti_idx++;
5161     cmd->dw21.bti_slice_map = bti_idx++;
5162     cmd->dw22.bti_vme_src = bti_idx++;
5163     cmd->dw23.bti_brc_input = bti_idx++;
5164     cmd->dw24.bti_simplest_intra = bti_idx++;
5165     cmd->dw25.bti_lcu_qp_surface = bti_idx++;
5166     cmd->dw26.bti_brc_data = bti_idx++;
5167     cmd->dw27.bti_debug = bti_idx++;
5168
5169     i965_gpe_context_unmap_curbe(gpe_context);
5170 }
5171
5172 static void
5173 gen9_hevc_16x16_pu_mode_set_surfaces(VADriverContextP ctx,
5174                                      struct encode_state *encode_state,
5175                                      struct intel_encoder_context *encoder_context,
5176                                      struct i965_gpe_context *gpe_context)
5177 {
5178     struct encoder_vme_mfc_context *vme_context = NULL;
5179     struct gen9_hevc_encoder_context *priv_ctx = NULL;
5180     struct gen9_hevc_encoder_state *priv_state = NULL;
5181     int bti_idx = 0;
5182     int size = 0;
5183
5184     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
5185     priv_ctx = (struct gen9_hevc_encoder_context *)vme_context->private_enc_ctx;
5186     priv_state = (struct gen9_hevc_encoder_state *)vme_context->private_enc_state;
5187
5188     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
5189                                  HEVC_ENC_SURFACE_RAW_Y_UV, bti_idx++,
5190                                  1, 1, I965_SURFACEFORMAT_R8_UNORM,
5191                                  NULL, NULL);
5192     bti_idx++;
5193
5194     gen9_hevc_set_gpe_1d_surface(ctx, priv_ctx, gpe_context,
5195                                  HEVC_ENC_SURFACE_16x16PU_SAD, bti_idx++,
5196                                  0, 0, 0, NULL, NULL);
5197
5198     size = priv_state->width_in_cu * priv_state->height_in_cu *
5199            priv_state->cu_record_size;
5200     gen9_hevc_set_gpe_1d_surface(ctx, priv_ctx, gpe_context,
5201                                  HEVC_ENC_SURFACE_CU_RECORD, bti_idx++,
5202                                  0, size, priv_state->mb_data_offset, NULL, NULL);
5203
5204     gen9_hevc_set_gpe_1d_surface(ctx, priv_ctx, gpe_context,
5205                                  HEVC_ENC_SURFACE_32x32_PU_OUTPUT, bti_idx++,
5206                                  0, 0, 0, NULL, NULL);
5207
5208     gen9_hevc_set_gpe_1d_surface(ctx, priv_ctx, gpe_context,
5209                                  HEVC_ENC_SURFACE_VME_8x8, bti_idx++,
5210                                  0, 0, 0, NULL, NULL);
5211
5212     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
5213                                  HEVC_ENC_SURFACE_SLICE_MAP, bti_idx++,
5214                                  0, 1, I965_SURFACEFORMAT_R8_UNORM,
5215                                  NULL, NULL);
5216
5217     gen9_hevc_set_gpe_adv_surface(ctx, priv_ctx, gpe_context,
5218                                   HEVC_ENC_SURFACE_RAW_VME, bti_idx++,
5219                                   NULL);
5220
5221     gen9_hevc_set_gpe_1d_surface(ctx, priv_ctx, gpe_context,
5222                                  HEVC_ENC_SURFACE_BRC_INPUT, bti_idx++,
5223                                  0, 0, 0, NULL, NULL);
5224
5225     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
5226                                  HEVC_ENC_SURFACE_SIMPLIFIED_INTRA, bti_idx++,
5227                                  0, 1, I965_SURFACEFORMAT_R8_UNORM,
5228                                  NULL, NULL);
5229
5230     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
5231                                  HEVC_ENC_SURFACE_LCU_QP, bti_idx++,
5232                                  0, 1, I965_SURFACEFORMAT_R8_UNORM,
5233                                  NULL, NULL);
5234
5235     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
5236                                  HEVC_ENC_SURFACE_BRC_DATA, bti_idx++,
5237                                  0, 1, I965_SURFACEFORMAT_R8_UNORM,
5238                                  NULL, NULL);
5239
5240     gen9_hevc_set_gpe_1d_surface(ctx, priv_ctx, gpe_context,
5241                                  HEVC_ENC_SURFACE_KERNEL_DEBUG, bti_idx++,
5242                                  0, 0, 0, NULL, NULL);
5243 }
5244
5245 static void
5246 gen9_hevc_16x16_pu_mode(VADriverContextP ctx,
5247                         struct encode_state *encode_state,
5248                         struct intel_encoder_context *encoder_context)
5249 {
5250     struct encoder_vme_mfc_context *vme_context = NULL;
5251     struct gen9_hevc_encoder_context *priv_ctx = NULL;
5252     struct gen9_hevc_encoder_state *priv_state = NULL;
5253     struct gpe_media_object_walker_parameter param;
5254     struct hevc_enc_kernel_walker_parameter hevc_walker_param;
5255     struct i965_gpe_context *gpe_context = NULL;
5256     int media_state = HEVC_ENC_MEDIA_STATE_16x16_PU_MODE_DECISION;
5257
5258     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
5259     priv_ctx = (struct gen9_hevc_encoder_context *)vme_context->private_enc_ctx;
5260     priv_state = (struct gen9_hevc_encoder_state *)vme_context->private_enc_state;
5261
5262     gpe_context = &priv_ctx->mbenc_context.gpe_contexts[HEVC_MBENC_16x16MD_IDX];
5263
5264     gen8_gpe_context_init(ctx, gpe_context);
5265     gen9_gpe_reset_binding_table(ctx, gpe_context);
5266     gen9_hevc_16x16_pu_mode_set_curbe(ctx, encode_state, encoder_context, gpe_context);
5267     gen9_hevc_16x16_pu_mode_set_surfaces(ctx, encode_state, encoder_context, gpe_context);
5268     gen8_gpe_setup_interface_data(ctx, gpe_context);
5269
5270     memset((void *)&hevc_walker_param, 0, sizeof(hevc_walker_param));
5271     hevc_walker_param.resolution_x = ALIGN(priv_state->picture_width, 32) >> 5;
5272     hevc_walker_param.resolution_y = ALIGN(priv_state->picture_height, 32) >> 5;
5273     hevc_walker_param.no_dependency = 1;
5274     gen9_hevc_init_object_walker(&hevc_walker_param, &param);
5275     gen9_hevc_run_object_walker(ctx, encoder_context, gpe_context, &param,
5276                                 media_state);
5277 }
5278
5279 static void
5280 gen9_hevc_16x16_sad_pu_comp_set_curbe(VADriverContextP ctx,
5281                                       struct encode_state *encode_state,
5282                                       struct intel_encoder_context *encoder_context,
5283                                       struct i965_gpe_context *gpe_context)
5284 {
5285     struct encoder_vme_mfc_context *vme_context = NULL;
5286     struct gen9_hevc_encoder_state *priv_state = NULL;
5287     gen9_hevc_mbenc_16x16_sad_curbe_data *cmd = NULL;
5288     VAEncSequenceParameterBufferHEVC *seq_param = NULL;
5289     int bti_idx = 0;
5290
5291     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
5292     priv_state = (struct gen9_hevc_encoder_state *)vme_context->private_enc_state;
5293     seq_param = (VAEncSequenceParameterBufferHEVC *)encode_state->seq_param_ext->buffer;
5294
5295     cmd = i965_gpe_context_map_curbe(gpe_context);
5296     if (!cmd)
5297         return;
5298
5299     memset((void *)cmd, 0, sizeof(*cmd));
5300
5301     cmd->dw0.frame_width = ALIGN(priv_state->picture_width, 16);
5302     cmd->dw0.frame_height = ALIGN(priv_state->picture_height, 16);
5303     cmd->dw1.log2_min_cu_size = seq_param->log2_min_luma_coding_block_size_minus3 + 3;
5304     cmd->dw1.log2_max_cu_size = cmd->dw1.log2_min_cu_size + seq_param->log2_diff_max_min_luma_coding_block_size;
5305     cmd->dw1.log2_min_tu_size = seq_param->log2_min_transform_block_size_minus2 + 2;
5306     cmd->dw1.enable_intra_early_exit = priv_state->tu_mode == HEVC_TU_RT_SPEED ?
5307                                        (priv_state->picture_coding_type == HEVC_SLICE_I ? 0 : 1) : 0;
5308     cmd->dw2.sim_flag_for_inter = 0;
5309     cmd->dw2.slice_type = priv_state->picture_coding_type;
5310     cmd->dw2.fast_surveillance_flag = priv_state->picture_coding_type == HEVC_SLICE_I ?
5311                                       0 : priv_state->video_surveillance_flag;
5312
5313     cmd->dw8.bti_src_y = bti_idx++;
5314     bti_idx++;
5315     cmd->dw9.bti_sad_16x16_pu_output = bti_idx++;
5316     cmd->dw10.bti_32x32_pu_mode_decision = bti_idx++;
5317     cmd->dw11.bti_slice_map = bti_idx++;
5318     cmd->dw12.bti_simplest_intra = bti_idx++;
5319     cmd->dw13.bti_debug = bti_idx++;
5320
5321     i965_gpe_context_unmap_curbe(gpe_context);
5322 }
5323
5324 static void
5325 gen9_hevc_16x16_sad_pu_comp_set_surfaces(VADriverContextP ctx,
5326                                          struct encode_state *encode_state,
5327                                          struct intel_encoder_context *encoder_context,
5328                                          struct i965_gpe_context *gpe_context)
5329 {
5330     struct encoder_vme_mfc_context *vme_context = NULL;
5331     struct gen9_hevc_encoder_context *priv_ctx = NULL;
5332     int bti_idx = 0;
5333
5334     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
5335     priv_ctx = (struct gen9_hevc_encoder_context *)vme_context->private_enc_ctx;
5336
5337     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
5338                                  HEVC_ENC_SURFACE_RAW_Y_UV, bti_idx++,
5339                                  1, 1, I965_SURFACEFORMAT_R8_UNORM,
5340                                  NULL, NULL);
5341     bti_idx++;
5342
5343     gen9_hevc_set_gpe_1d_surface(ctx, priv_ctx, gpe_context,
5344                                  HEVC_ENC_SURFACE_16x16PU_SAD, bti_idx++,
5345                                  0, 0, 0, NULL, NULL);
5346
5347     gen9_hevc_set_gpe_1d_surface(ctx, priv_ctx, gpe_context,
5348                                  HEVC_ENC_SURFACE_32x32_PU_OUTPUT, bti_idx++,
5349                                  0, 0, 0, NULL, NULL);
5350
5351     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
5352                                  HEVC_ENC_SURFACE_SLICE_MAP, bti_idx++,
5353                                  0, 1, I965_SURFACEFORMAT_R8_UNORM,
5354                                  NULL, NULL);
5355
5356     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
5357                                  HEVC_ENC_SURFACE_SIMPLIFIED_INTRA, bti_idx++,
5358                                  0, 1, I965_SURFACEFORMAT_R8_UNORM,
5359                                  NULL, NULL);
5360
5361     gen9_hevc_set_gpe_1d_surface(ctx, priv_ctx, gpe_context,
5362                                  HEVC_ENC_SURFACE_KERNEL_DEBUG, bti_idx++,
5363                                  0, 0, 0, NULL, NULL);
5364 }
5365
5366 static void
5367 gen9_hevc_16x16_sad_pu_computation(VADriverContextP ctx,
5368                                    struct encode_state *encode_state,
5369                                    struct intel_encoder_context *encoder_context)
5370 {
5371     struct encoder_vme_mfc_context *vme_context = NULL;
5372     struct gen9_hevc_encoder_context *priv_ctx = NULL;
5373     struct gen9_hevc_encoder_state *priv_state = NULL;
5374     struct gpe_media_object_walker_parameter param;
5375     struct hevc_enc_kernel_walker_parameter hevc_walker_param;
5376     struct i965_gpe_context *gpe_context = NULL;
5377     int media_state = HEVC_ENC_MEDIA_STATE_16x16_PU_SAD;
5378
5379     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
5380     priv_ctx = (struct gen9_hevc_encoder_context *)vme_context->private_enc_ctx;
5381     priv_state = (struct gen9_hevc_encoder_state *)vme_context->private_enc_state;
5382
5383     gpe_context = &priv_ctx->mbenc_context.gpe_contexts[HEVC_MBENC_16x16SAD_IDX];
5384
5385     gen8_gpe_context_init(ctx, gpe_context);
5386     gen9_gpe_reset_binding_table(ctx, gpe_context);
5387     gen9_hevc_16x16_sad_pu_comp_set_curbe(ctx, encode_state, encoder_context, gpe_context);
5388     gen9_hevc_16x16_sad_pu_comp_set_surfaces(ctx, encode_state, encoder_context, gpe_context);
5389     gen8_gpe_setup_interface_data(ctx, gpe_context);
5390
5391     memset((void *)&hevc_walker_param, 0, sizeof(hevc_walker_param));
5392     hevc_walker_param.resolution_x = ALIGN(priv_state->picture_width, 16) >> 4;
5393     hevc_walker_param.resolution_y = ALIGN(priv_state->picture_height, 16) >> 4;
5394     hevc_walker_param.no_dependency = 1;
5395     gen9_hevc_init_object_walker(&hevc_walker_param, &param);
5396     gen9_hevc_run_object_walker(ctx, encoder_context, gpe_context, &param,
5397                                 media_state);
5398 }
5399
5400 static void
5401 gen9_hevc_32x32_b_intra_set_curbe(VADriverContextP ctx,
5402                                   struct encode_state *encode_state,
5403                                   struct intel_encoder_context *encoder_context,
5404                                   struct i965_gpe_context *gpe_context)
5405 {
5406     struct encoder_vme_mfc_context *vme_context = NULL;
5407     struct generic_enc_codec_state *generic_state = NULL;
5408     struct gen9_hevc_encoder_state *priv_state = NULL;
5409     gen9_hevc_mbenc_b_32x32_pu_intra_curbe_data *cmd = NULL;
5410     VAEncSequenceParameterBufferHEVC *seq_param = NULL;
5411     int bti_idx = 0;
5412
5413     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
5414     generic_state = (struct generic_enc_codec_state *)vme_context->generic_enc_state;
5415     priv_state = (struct gen9_hevc_encoder_state *)vme_context->private_enc_state;
5416     seq_param = (VAEncSequenceParameterBufferHEVC *)encode_state->seq_param_ext->buffer;
5417
5418     cmd = i965_gpe_context_map_curbe(gpe_context);
5419     if (!cmd)
5420         return;
5421
5422     memset((void *)cmd, 0, sizeof(*cmd));
5423
5424     cmd->dw0.frame_width = ALIGN(priv_state->picture_width, 16);
5425     cmd->dw0.frame_height = ALIGN(priv_state->picture_height, 16);
5426     cmd->dw1.enable_debug_dump = 0;
5427     cmd->dw1.enable_intra_early_exit = priv_state->tu_mode == HEVC_TU_RT_SPEED ? 1 : 0;
5428     cmd->dw1.flags = 0;
5429     cmd->dw1.log2_min_tu_size = seq_param->log2_min_transform_block_size_minus2 + 2;
5430     cmd->dw1.slice_type = priv_state->picture_coding_type;
5431     cmd->dw1.hme_enable = generic_state->hme_enabled;
5432     cmd->dw1.fast_surveillance_flag = priv_state->picture_coding_type == HEVC_SLICE_I ?
5433                                       0 : priv_state->video_surveillance_flag;
5434
5435     cmd->dw2.qp_multiplier = 100;
5436     cmd->dw2.qp_value = 0;
5437
5438     cmd->dw8.bti_per_32x32_pu_intra_checck = bti_idx++;
5439     cmd->dw9.bti_src_y = bti_idx++;
5440     bti_idx++;
5441     cmd->dw10.bti_src_y2x = bti_idx++;
5442     cmd->dw11.bti_slice_map = bti_idx++;
5443     cmd->dw12.bti_vme_y2x = bti_idx++;
5444     cmd->dw13.bti_simplest_intra = bti_idx++;
5445     cmd->dw14.bti_hme_mv_pred = bti_idx++;
5446     cmd->dw15.bti_hme_dist = bti_idx++;
5447     cmd->dw16.bti_lcu_skip = bti_idx++;
5448     cmd->dw17.bti_debug = bti_idx++;
5449
5450     i965_gpe_context_unmap_curbe(gpe_context);
5451 }
5452
5453 static void
5454 gen9_hevc_32x32_b_intra_set_surfaces(VADriverContextP ctx,
5455                                      struct encode_state *encode_state,
5456                                      struct intel_encoder_context *encoder_context,
5457                                      struct i965_gpe_context *gpe_context)
5458 {
5459     struct encoder_vme_mfc_context *vme_context = NULL;
5460     struct gen9_hevc_encoder_context *priv_ctx = NULL;
5461     int bti_idx = 0;
5462
5463     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
5464     priv_ctx = (struct gen9_hevc_encoder_context *)vme_context->private_enc_ctx;
5465
5466     gen9_hevc_set_gpe_1d_surface(ctx, priv_ctx, gpe_context,
5467                                  HEVC_ENC_SURFACE_32x32_PU_OUTPUT, bti_idx++,
5468                                  0, 0, 0, NULL, NULL);
5469
5470     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
5471                                  HEVC_ENC_SURFACE_RAW_Y_UV, bti_idx++,
5472                                  1, 1, I965_SURFACEFORMAT_R8_UNORM,
5473                                  NULL, NULL);
5474     bti_idx++;
5475
5476     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
5477                                  HEVC_ENC_SURFACE_Y_2X, bti_idx++,
5478                                  0, 1, I965_SURFACEFORMAT_R8_UNORM,
5479                                  NULL, NULL);
5480
5481     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
5482                                  HEVC_ENC_SURFACE_SLICE_MAP, bti_idx++,
5483                                  0, 1, I965_SURFACEFORMAT_R8_UNORM,
5484                                  NULL, NULL);
5485
5486     gen9_hevc_set_gpe_adv_surface(ctx, priv_ctx, gpe_context,
5487                                   HEVC_ENC_SURFACE_Y_2X_VME, bti_idx++,
5488                                   NULL);
5489
5490     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
5491                                  HEVC_ENC_SURFACE_SIMPLIFIED_INTRA, bti_idx++,
5492                                  0, 1, I965_SURFACEFORMAT_R8_UNORM,
5493                                  NULL, NULL);
5494
5495     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
5496                                  HEVC_ENC_SURFACE_HME_MVP, bti_idx++,
5497                                  0, 1, I965_SURFACEFORMAT_R8_UNORM,
5498                                  NULL, NULL);
5499
5500     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
5501                                  HEVC_ENC_SURFACE_HME_DIST, bti_idx++,
5502                                  0, 1, I965_SURFACEFORMAT_R8_UNORM,
5503                                  NULL, NULL);
5504
5505     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
5506                                  HEVC_ENC_SURFACE_LCU_QP, bti_idx++,
5507                                  0, 1, I965_SURFACEFORMAT_R8_UNORM,
5508                                  NULL, NULL);
5509
5510     gen9_hevc_set_gpe_1d_surface(ctx, priv_ctx, gpe_context,
5511                                  HEVC_ENC_SURFACE_KERNEL_DEBUG, bti_idx,
5512                                  0, 0, 0, NULL, NULL);
5513 }
5514
5515 static void
5516 gen9_hevc_32x32_b_intra(VADriverContextP ctx,
5517                         struct encode_state *encode_state,
5518                         struct intel_encoder_context *encoder_context)
5519 {
5520     struct encoder_vme_mfc_context *vme_context = NULL;
5521     struct gen9_hevc_encoder_context *priv_ctx = NULL;
5522     struct gen9_hevc_encoder_state *priv_state = NULL;
5523     struct gpe_media_object_walker_parameter param;
5524     struct hevc_enc_kernel_walker_parameter hevc_walker_param;
5525     struct i965_gpe_context *gpe_context = NULL;
5526     int media_state = HEVC_ENC_MEDIA_STATE_32x32_B_INTRA_CHECK;
5527
5528     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
5529     priv_ctx = (struct gen9_hevc_encoder_context *)vme_context->private_enc_ctx;
5530     priv_state = (struct gen9_hevc_encoder_state *)vme_context->private_enc_state;
5531
5532     gpe_context = &priv_ctx->mbenc_context.gpe_contexts[HEVC_MBENC_32x32INTRACHECK_IDX];
5533
5534     gen8_gpe_context_init(ctx, gpe_context);
5535     gen9_gpe_reset_binding_table(ctx, gpe_context);
5536     gen9_hevc_32x32_b_intra_set_curbe(ctx, encode_state, encoder_context, gpe_context);
5537     gen9_hevc_32x32_b_intra_set_surfaces(ctx, encode_state, encoder_context, gpe_context);
5538     gen8_gpe_setup_interface_data(ctx, gpe_context);
5539
5540     memset((void *)&hevc_walker_param, 0, sizeof(hevc_walker_param));
5541     hevc_walker_param.resolution_x = ALIGN(priv_state->picture_width, 32) >> 5;
5542     hevc_walker_param.resolution_y = ALIGN(priv_state->picture_height, 32) >> 5;
5543     hevc_walker_param.no_dependency = 1;
5544     gen9_hevc_init_object_walker(&hevc_walker_param, &param);
5545     gen9_hevc_run_object_walker(ctx, encoder_context, gpe_context, &param,
5546                                 media_state);
5547 }
5548
5549 static void
5550 gen9_hevc_32x32_pu_mode_set_curbe(VADriverContextP ctx,
5551                                   struct encode_state *encode_state,
5552                                   struct intel_encoder_context *encoder_context,
5553                                   struct i965_gpe_context *gpe_context)
5554 {
5555     struct encoder_vme_mfc_context *vme_context = NULL;
5556     struct gen9_hevc_encoder_context *priv_ctx = NULL;
5557     struct generic_enc_codec_state *generic_state = NULL;
5558     struct gen9_hevc_encoder_state *priv_state = NULL;
5559     VAEncPictureParameterBufferHEVC *pic_param = NULL;
5560     VAEncSliceParameterBufferHEVC *slice_param = NULL;
5561     gen9_hevc_mbenc_32x32_pu_mode_curbe_data *cmd = NULL;
5562     double lambda_scaling_factor = 1.0, qp_lambda = 0.0, squared_qp_lambda = 0.0;
5563     unsigned int slice_qp = 0, fixed_point_lambda = 0;
5564     int bti_idx = 0;
5565
5566     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
5567     priv_ctx = (struct gen9_hevc_encoder_context *)vme_context->private_enc_ctx;
5568     generic_state = (struct generic_enc_codec_state *)vme_context->generic_enc_state;
5569     priv_state = (struct gen9_hevc_encoder_state *)vme_context->private_enc_state;
5570     pic_param = (VAEncPictureParameterBufferHEVC *)encode_state->pic_param_ext->buffer;
5571     slice_param = (VAEncSliceParameterBufferHEVC *)encode_state->slice_params_ext[0]->buffer;
5572
5573     slice_qp = pic_param->pic_init_qp + slice_param->slice_qp_delta;
5574     gen9_hevc_set_lambda_tables(priv_ctx, HEVC_SLICE_I, HEVC_ENC_INTRA_TRANS_HAAR);
5575     lambda_scaling_factor = 1.0;
5576     qp_lambda = priv_ctx->lambda_md_table[HEVC_SLICE_I][slice_qp];
5577     squared_qp_lambda = qp_lambda * qp_lambda;
5578     fixed_point_lambda = (unsigned int)(lambda_scaling_factor * squared_qp_lambda * (1 << 10));
5579
5580     cmd = i965_gpe_context_map_curbe(gpe_context);
5581     if (!cmd)
5582         return;
5583
5584     memset((void *)cmd, 0, sizeof(*cmd));
5585
5586     cmd->dw0.frame_width = ALIGN(priv_state->picture_width, 16);
5587     cmd->dw0.frame_height = ALIGN(priv_state->picture_height, 16);
5588     cmd->dw1.enable_debug_dump = 0;
5589     cmd->dw1.lcu_type = priv_state->lcu_size == 64 ? 0 : 1;
5590     cmd->dw1.pu_type = 0;
5591     cmd->dw1.brc_enable = !!generic_state->brc_enabled;
5592     cmd->dw1.lcu_brc_enable = priv_state->lcu_brc_enabled;
5593     cmd->dw1.slice_type = priv_state->picture_coding_type;
5594     cmd->dw1.fast_surveillance_flag = priv_state->picture_coding_type == HEVC_SLICE_I ?
5595                                       0 : priv_state->video_surveillance_flag;
5596     cmd->dw1.roi_enable = (priv_state->num_roi > 0);
5597
5598     cmd->dw2.lambda = fixed_point_lambda;
5599     cmd->dw3.mode_cost_32x32 = 0;
5600     cmd->dw4.early_exit = (unsigned int) - 1;
5601
5602     cmd->dw8.bti_32x32_pu_output = bti_idx++;
5603     cmd->dw9.bti_src_y = bti_idx++;
5604     bti_idx++;
5605     cmd->dw10.bti_src_y2x = bti_idx++;
5606     cmd->dw11.bti_slice_map = bti_idx++;
5607     cmd->dw12.bti_src_y2x_vme = bti_idx++;
5608     cmd->dw13.bti_brc_input = bti_idx++;
5609     cmd->dw14.bti_lcu_qp_surface = bti_idx++;
5610     cmd->dw15.bti_brc_data = bti_idx++;
5611     cmd->dw16.bti_kernel_debug = bti_idx++;
5612
5613     i965_gpe_context_unmap_curbe(gpe_context);
5614 }
5615
5616 static void
5617 gen9_hevc_32x32_pu_mode_set_surfaces(VADriverContextP ctx,
5618                                      struct encode_state *encode_state,
5619                                      struct intel_encoder_context *encoder_context,
5620                                      struct i965_gpe_context *gpe_context)
5621 {
5622     struct encoder_vme_mfc_context *vme_context = NULL;
5623     struct gen9_hevc_encoder_context *priv_ctx = NULL;
5624     int bti_idx = 0;
5625
5626     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
5627     priv_ctx = (struct gen9_hevc_encoder_context *)vme_context->private_enc_ctx;
5628
5629     gen9_hevc_set_gpe_1d_surface(ctx, priv_ctx, gpe_context,
5630                                  HEVC_ENC_SURFACE_32x32_PU_OUTPUT, bti_idx++,
5631                                  0, 0, 0, NULL, NULL);
5632
5633     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
5634                                  HEVC_ENC_SURFACE_RAW_Y, bti_idx++,
5635                                  1, 1, I965_SURFACEFORMAT_R8_UNORM,
5636                                  NULL, NULL);
5637     bti_idx++;
5638
5639     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
5640                                  HEVC_ENC_SURFACE_Y_2X, bti_idx++,
5641                                  0, 1, I965_SURFACEFORMAT_R8_UNORM,
5642                                  NULL, NULL);
5643
5644     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
5645                                  HEVC_ENC_SURFACE_SLICE_MAP, bti_idx++,
5646                                  0, 1, I965_SURFACEFORMAT_R8_UNORM,
5647                                  NULL, NULL);
5648
5649     gen9_hevc_set_gpe_adv_surface(ctx, priv_ctx, gpe_context,
5650                                   HEVC_ENC_SURFACE_Y_2X_VME, bti_idx++,
5651                                   NULL);
5652
5653     gen9_hevc_set_gpe_1d_surface(ctx, priv_ctx, gpe_context,
5654                                  HEVC_ENC_SURFACE_BRC_INPUT, bti_idx++,
5655                                  0, 0, 0, NULL, NULL);
5656
5657     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
5658                                  HEVC_ENC_SURFACE_LCU_QP, bti_idx++,
5659                                  0, 1, I965_SURFACEFORMAT_R8_UNORM,
5660                                  NULL, NULL);
5661
5662     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
5663                                  HEVC_ENC_SURFACE_BRC_DATA, bti_idx++,
5664                                  0, 1, I965_SURFACEFORMAT_R8_UNORM,
5665                                  NULL, NULL);
5666
5667     gen9_hevc_set_gpe_1d_surface(ctx, priv_ctx, gpe_context,
5668                                  HEVC_ENC_SURFACE_KERNEL_DEBUG, bti_idx,
5669                                  0, 0, 0, NULL, NULL);
5670 }
5671
5672 static void
5673 gen9_hevc_32x32_pu_mode(VADriverContextP ctx,
5674                         struct encode_state *encode_state,
5675                         struct intel_encoder_context *encoder_context)
5676 {
5677     struct encoder_vme_mfc_context *vme_context = NULL;
5678     struct gen9_hevc_encoder_context *priv_ctx = NULL;
5679     struct gen9_hevc_encoder_state *priv_state = NULL;
5680     struct gpe_media_object_walker_parameter param;
5681     struct hevc_enc_kernel_walker_parameter hevc_walker_param;
5682     struct i965_gpe_context *gpe_context = NULL;
5683     int media_state = HEVC_ENC_MEDIA_STATE_32x32_PU_MODE_DECISION;
5684
5685     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
5686     priv_ctx = (struct gen9_hevc_encoder_context *)vme_context->private_enc_ctx;
5687     priv_state = (struct gen9_hevc_encoder_state *)vme_context->private_enc_state;
5688
5689     gpe_context = &priv_ctx->mbenc_context.gpe_contexts[HEVC_MBENC_32x32MD_IDX];
5690
5691     gen8_gpe_context_init(ctx, gpe_context);
5692     gen9_gpe_reset_binding_table(ctx, gpe_context);
5693     gen9_hevc_32x32_pu_mode_set_curbe(ctx, encode_state, encoder_context, gpe_context);
5694     gen9_hevc_32x32_pu_mode_set_surfaces(ctx, encode_state, encoder_context, gpe_context);
5695     gen8_gpe_setup_interface_data(ctx, gpe_context);
5696
5697     memset((void *)&hevc_walker_param, 0, sizeof(hevc_walker_param));
5698     hevc_walker_param.resolution_x = ALIGN(priv_state->picture_width, 32) >> 5;
5699     hevc_walker_param.resolution_y = ALIGN(priv_state->picture_height, 32) >> 5;
5700     hevc_walker_param.no_dependency = 1;
5701     gen9_hevc_init_object_walker(&hevc_walker_param, &param);
5702     gen9_hevc_run_object_walker(ctx, encoder_context, gpe_context, &param,
5703                                 media_state);
5704 }
5705
5706 static void
5707 gen9_hevc_2x_scaling_set_curbe(VADriverContextP ctx,
5708                                struct encode_state *encode_state,
5709                                struct intel_encoder_context *encoder_context,
5710                                struct i965_gpe_context *gpe_context)
5711 {
5712     struct encoder_vme_mfc_context *vme_context = NULL;
5713     struct gen9_hevc_encoder_state *priv_state = NULL;
5714     gen9_hevc_mbenc_downscaling2x_curbe_data *cmd = NULL;
5715     int bti_idx = 0;
5716
5717     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
5718     priv_state = (struct gen9_hevc_encoder_state *)vme_context->private_enc_state;
5719
5720     cmd = i965_gpe_context_map_curbe(gpe_context);
5721     if (!cmd)
5722         return;
5723
5724     memset((void *)cmd, 0, sizeof(gen9_hevc_mbenc_downscaling2x_curbe_data));
5725
5726     cmd->dw0.pic_width = ALIGN(priv_state->picture_width, 16);
5727     cmd->dw0.pic_height = ALIGN(priv_state->picture_height, 16);
5728
5729     cmd->dw8.bti_src_y = bti_idx++;
5730     cmd->dw9.bit_dst_y = bti_idx;
5731
5732     i965_gpe_context_unmap_curbe(gpe_context);
5733 }
5734
5735 static void
5736 gen9_hevc_2x_scaling_set_surfaces(VADriverContextP ctx,
5737                                   struct encode_state *encode_state,
5738                                   struct intel_encoder_context *encoder_context,
5739                                   struct i965_gpe_context *gpe_context)
5740 {
5741     struct encoder_vme_mfc_context *vme_context = NULL;
5742     struct gen9_hevc_encoder_context *priv_ctx = NULL;
5743     int bti_idx = 0;
5744
5745     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
5746     priv_ctx = (struct gen9_hevc_encoder_context *)vme_context->private_enc_ctx;
5747
5748     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
5749                                  HEVC_ENC_SURFACE_RAW_Y, bti_idx++,
5750                                  0, 1, I965_SURFACEFORMAT_R16_UNORM,
5751                                  NULL, NULL);
5752
5753     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
5754                                  HEVC_ENC_SURFACE_Y_2X, bti_idx,
5755                                  0, 1, I965_SURFACEFORMAT_R16_UNORM,
5756                                  NULL, NULL);
5757 }
5758
5759 static void
5760 gen9_hevc_2x_scaling(VADriverContextP ctx,
5761                      struct encode_state *encode_state,
5762                      struct intel_encoder_context *encoder_context)
5763 {
5764     struct encoder_vme_mfc_context *vme_context = NULL;
5765     struct gen9_hevc_encoder_context *priv_ctx = NULL;
5766     struct gen9_hevc_encoder_state *priv_state = NULL;
5767     struct gpe_media_object_walker_parameter param;
5768     struct hevc_enc_kernel_walker_parameter hevc_walker_param;
5769     struct i965_gpe_context *gpe_context = NULL;
5770     int media_state = HEVC_ENC_MEDIA_STATE_2X_SCALING;
5771
5772     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
5773     priv_ctx = (struct gen9_hevc_encoder_context *)vme_context->private_enc_ctx;
5774     priv_state = (struct gen9_hevc_encoder_state *)vme_context->private_enc_state;
5775
5776     gpe_context = &priv_ctx->mbenc_context.gpe_contexts[HEVC_MBENC_2xSCALING_IDX];
5777
5778     gen8_gpe_context_init(ctx, gpe_context);
5779     gen9_gpe_reset_binding_table(ctx, gpe_context);
5780     gen9_hevc_2x_scaling_set_curbe(ctx, encode_state, encoder_context, gpe_context);
5781     gen9_hevc_2x_scaling_set_surfaces(ctx, encode_state, encoder_context, gpe_context);
5782     gen8_gpe_setup_interface_data(ctx, gpe_context);
5783
5784     memset((void *)&hevc_walker_param, 0, sizeof(hevc_walker_param));
5785     hevc_walker_param.resolution_x = ALIGN(priv_state->picture_width, 32) >> 5;
5786     hevc_walker_param.resolution_y = ALIGN(priv_state->picture_height, 32) >> 5;
5787     hevc_walker_param.no_dependency = 1;
5788     gen9_hevc_init_object_walker(&hevc_walker_param, &param);
5789
5790     gen9_hevc_run_object_walker(ctx, encoder_context, gpe_context, &param,
5791                                 media_state);
5792 }
5793
5794 static void
5795 gen9_hevc_mbenc(VADriverContextP ctx,
5796                 struct encode_state *encode_state,
5797                 struct intel_encoder_context *encoder_context)
5798 {
5799     struct encoder_vme_mfc_context *vme_context = NULL;
5800     struct gen9_hevc_encoder_state *priv_state = NULL;
5801     int fast_encoding = 0;
5802
5803     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
5804     priv_state = (struct gen9_hevc_encoder_state *)vme_context->private_enc_state;
5805     fast_encoding = (priv_state->tu_mode == HEVC_TU_BEST_SPEED);
5806
5807     if (!fast_encoding) {
5808         if (!priv_state->bit_depth_luma_minus8)
5809             gen9_hevc_2x_scaling(ctx, encode_state, encoder_context);
5810
5811         if (priv_state->picture_coding_type == HEVC_SLICE_I)
5812             gen9_hevc_32x32_pu_mode(ctx, encode_state, encoder_context);
5813         else
5814             gen9_hevc_32x32_b_intra(ctx, encode_state, encoder_context);
5815
5816         gen9_hevc_16x16_sad_pu_computation(ctx, encode_state, encoder_context);
5817         gen9_hevc_16x16_pu_mode(ctx, encode_state, encoder_context);
5818         gen9_hevc_8x8_pu_mode(ctx, encode_state, encoder_context);
5819         gen9_hevc_8x8_pu_fmode(ctx, encode_state, encoder_context);
5820     }
5821
5822     if (priv_state->picture_coding_type != HEVC_SLICE_I ||
5823         fast_encoding) {
5824         gen9_hevc_8x8_b_mbenc(ctx, encode_state, encoder_context);
5825         gen9_hevc_8x8_b_pak(ctx, encode_state, encoder_context);
5826     }
5827 }
5828
5829 static VAStatus
5830 gen9_hevc_vme_gpe_init(VADriverContextP ctx,
5831                        struct encode_state *encode_state,
5832                        struct intel_encoder_context *encoder_context)
5833 {
5834     struct encoder_vme_mfc_context *vme_context = NULL;
5835     struct gen9_hevc_encoder_context *priv_ctx = NULL;
5836     struct gen9_hevc_encoder_state *priv_state = NULL;
5837     VAEncSliceParameterBufferHEVC *slice_param = NULL;
5838     int i = 0, j = 0;
5839
5840     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
5841     priv_ctx = (struct gen9_hevc_encoder_context *)vme_context->private_enc_ctx;
5842     priv_state = (struct gen9_hevc_encoder_state *)vme_context->private_enc_state;
5843
5844     i965_zero_gpe_resource(&priv_ctx->res_mb_code_surface);
5845
5846     i965_zero_gpe_resource(&priv_ctx->res_slice_map_buffer);
5847     if (encode_state->num_slice_params_ext > 1) {
5848         struct gen9_hevc_slice_map *pslice_map = NULL;
5849         int width = priv_state->width_in_lcu;
5850         int pitch = ALIGN(priv_state->frame_width_in_max_lcu >> 3, 64);
5851         void *ptr_start = NULL;
5852         int lcu_count = 0;
5853
5854         ptr_start = (void *)i965_map_gpe_resource(&priv_ctx->res_slice_map_buffer);
5855
5856         if (!ptr_start)
5857             return VA_STATUS_ERROR_UNKNOWN;
5858
5859         pslice_map = (struct gen9_hevc_slice_map *)ptr_start;
5860         for (i = 0; i < encode_state->num_slice_params_ext; i++) {
5861             slice_param = (VAEncSliceParameterBufferHEVC *)encode_state->slice_params_ext[i]->buffer;
5862
5863             for (j = 0; j < slice_param->num_ctu_in_slice; j++) {
5864                 pslice_map[lcu_count++].slice_id = i;
5865
5866                 if (lcu_count >= width) {
5867                     lcu_count = 0;
5868                     ptr_start += pitch;
5869
5870                     pslice_map = (struct gen9_hevc_slice_map *)ptr_start;
5871                 }
5872             }
5873         }
5874
5875         i965_unmap_gpe_resource(&priv_ctx->res_slice_map_buffer);
5876     }
5877
5878     return VA_STATUS_SUCCESS;
5879 }
5880
5881 static VAStatus
5882 gen9_hevc_vme_gpe_run(VADriverContextP ctx,
5883                       struct encode_state *encode_state,
5884                       struct intel_encoder_context *encoder_context)
5885 {
5886     struct encoder_vme_mfc_context *vme_context = NULL;
5887     struct generic_enc_codec_state *generic_state = NULL;
5888     struct gen9_hevc_encoder_state *priv_state = NULL;
5889
5890     vme_context = encoder_context->vme_context;
5891     generic_state = vme_context->generic_enc_state;
5892     priv_state = vme_context->private_enc_state;
5893
5894     if (generic_state->brc_enabled &&
5895         (generic_state->brc_need_reset || !generic_state->brc_inited)) {
5896         gen9_hevc_brc_init_reset(ctx, encode_state, encoder_context,
5897                                  generic_state->brc_inited ? 1 : 0);
5898         generic_state->brc_need_reset = 0;
5899         generic_state->brc_inited = 1;
5900     }
5901
5902     if (generic_state->hme_supported || generic_state->brc_enabled) {
5903         gen9_hevc_hme_scaling(ctx, encode_state, encoder_context);
5904
5905         if (generic_state->brc_enabled)
5906             gen9_hevc_brc_intra_dist(ctx, encode_state, encoder_context);
5907
5908         if (generic_state->hme_enabled)
5909             gen9_hevc_hme_encode_me(ctx, encode_state, encoder_context);
5910
5911         if (generic_state->brc_enabled)
5912             gen9_hevc_brc_update(ctx, encode_state, encoder_context);
5913     }
5914
5915     if (priv_state->num_roi && !generic_state->brc_enabled)
5916         gen9_hevc_brc_update_lcu_based(ctx, encode_state, encoder_context);
5917
5918     if (priv_state->bit_depth_luma_minus8)
5919         gen9_hevc_ref_frame_depth_conversion(ctx, encode_state, encoder_context);
5920
5921     gen9_hevc_mbenc(ctx, encode_state, encoder_context);
5922
5923     return VA_STATUS_SUCCESS;
5924 }
5925
5926 static VAStatus
5927 gen9_hevc_vme_pipeline(VADriverContextP ctx,
5928                        VAProfile profile,
5929                        struct encode_state *encode_state,
5930                        struct intel_encoder_context *encoder_context)
5931 {
5932     VAStatus va_status = VA_STATUS_SUCCESS;
5933
5934     va_status = gen9_hevc_enc_init_parameters(ctx, encode_state, encoder_context);
5935     if (va_status != VA_STATUS_SUCCESS)
5936         goto EXIT;
5937
5938     va_status = gen9_hevc_vme_gpe_init(ctx, encode_state, encoder_context);
5939     if (va_status != VA_STATUS_SUCCESS)
5940         goto EXIT;
5941
5942     va_status = gen9_hevc_vme_gpe_run(ctx, encode_state, encoder_context);
5943
5944 EXIT:
5945     return va_status;
5946 }
5947
5948 static void
5949 gen9_hevc_vme_scaling_context_init(VADriverContextP ctx,
5950                                    struct intel_encoder_context *encoder_context)
5951 {
5952     struct encoder_vme_mfc_context *vme_context = NULL;
5953     struct gen9_hevc_encoder_context *priv_ctx = NULL;
5954     struct gen9_hevc_encoder_state *priv_state = NULL;
5955     struct i965_gpe_context *gpe_context = NULL;
5956     struct i965_kernel kernel_info;
5957     struct gen9_hevc_scaling_context *scaling_ctx = NULL;
5958     GEN9_ENC_OPERATION kernel_idx;
5959     int curbe_size = 0;
5960     int i = 0;
5961
5962     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
5963     priv_ctx = (struct gen9_hevc_encoder_context *)vme_context->private_enc_ctx;
5964     priv_state = (struct gen9_hevc_encoder_state *)vme_context->private_enc_state;
5965
5966     scaling_ctx = &priv_ctx->scaling_context;
5967
5968     for (i = 0; i < NUM_HEVC_ENC_SCALING; i++) {
5969         if (i == HEVC_ENC_SCALING_4X ||
5970             i == HEVC_ENC_SCALING_16X) {
5971             curbe_size = sizeof(gen9_hevc_scaling4x_curbe_data);
5972             kernel_idx = GEN9_ENC_SCALING4X;
5973         } else if (i == HEVC_ENC_SCALING_32X) {
5974             curbe_size = sizeof(gen9_hevc_scaling2x_curbe_data);
5975             kernel_idx = GEN9_ENC_SCALING2X;
5976         }
5977
5978         gpe_context = &scaling_ctx->gpe_contexts[i];
5979
5980         gen9_hevc_vme_init_gpe_context(ctx, gpe_context,
5981                                        curbe_size,
5982                                        curbe_size);
5983         gen9_hevc_vme_init_scoreboard(gpe_context,
5984                                       0xFF,
5985                                       priv_state->use_hw_scoreboard,
5986                                       priv_state->use_hw_non_stalling_scoreborad);
5987
5988         memset(&kernel_info, 0, sizeof(kernel_info));
5989         gen9_hevc_get_kernel_header_and_size((void *)hevc_enc_kernel_ptr,
5990                                              hevc_enc_kernel_size,
5991                                              kernel_idx,
5992                                              0,
5993                                              &kernel_info);
5994         gen8_gpe_load_kernels(ctx,
5995                               gpe_context,
5996                               &kernel_info,
5997                               1);
5998     }
5999 }
6000
6001 static void
6002 gen9_hevc_vme_me_context_init(VADriverContextP ctx,
6003                               struct intel_encoder_context *encoder_context)
6004 {
6005     struct encoder_vme_mfc_context *vme_context = NULL;
6006     struct gen9_hevc_encoder_context *priv_ctx = NULL;
6007     struct gen9_hevc_encoder_state *priv_state = NULL;
6008     struct i965_gpe_context *gpe_context = NULL;
6009     struct i965_kernel kernel_info;
6010     struct gen9_hevc_me_context *me_ctx = NULL;
6011     int i = 0, j = 0;
6012
6013     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
6014     priv_ctx = (struct gen9_hevc_encoder_context *)vme_context->private_enc_ctx;
6015     priv_state = (struct gen9_hevc_encoder_state *)vme_context->private_enc_state;
6016
6017     me_ctx = &priv_ctx->me_context;
6018
6019     for (i = 0; i < NUM_HEVC_ENC_ME; i++) {
6020         for (j = 0; j < NUM_HEVC_ENC_ME_TYPES; j++) {
6021             gpe_context = &me_ctx->gpe_context[j][i];
6022
6023             gen9_hevc_vme_init_gpe_context(ctx, gpe_context,
6024                                            sizeof(gen9_hevc_me_curbe_data),
6025                                            0);
6026             gen9_hevc_vme_init_scoreboard(gpe_context,
6027                                           0xFF,
6028                                           priv_state->use_hw_scoreboard,
6029                                           priv_state->use_hw_non_stalling_scoreborad);
6030
6031             memset(&kernel_info, 0, sizeof(kernel_info));
6032             gen9_hevc_get_kernel_header_and_size((void *)hevc_enc_kernel_ptr,
6033                                                  hevc_enc_kernel_size,
6034                                                  GEN9_ENC_ME,
6035                                                  i,
6036                                                  &kernel_info);
6037             gen8_gpe_load_kernels(ctx,
6038                                   gpe_context,
6039                                   &kernel_info,
6040                                   1);
6041         }
6042     }
6043 }
6044
6045 static void
6046 gen9_hevc_vme_mbenc_context_init(VADriverContextP ctx,
6047                                  struct intel_encoder_context *encoder_context)
6048 {
6049     struct encoder_vme_mfc_context *vme_context = NULL;
6050     struct gen9_hevc_encoder_context *priv_ctx = NULL;
6051     struct gen9_hevc_encoder_state *priv_state = NULL;
6052     struct i965_gpe_context *gpe_context = NULL;
6053     struct i965_kernel kernel_info;
6054     struct gen9_hevc_mbenc_context *mbenc_ctx = NULL;
6055     int i = 0;
6056
6057     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
6058     priv_ctx = (struct gen9_hevc_encoder_context *)vme_context->private_enc_ctx;
6059     priv_state = (struct gen9_hevc_encoder_state *)vme_context->private_enc_state;
6060
6061     mbenc_ctx = &priv_ctx->mbenc_context;
6062
6063     mbenc_ctx->kernel_num = GEN8_HEVC_ENC_MBENC_TOTAL_NUM;
6064
6065     for (i = 0; i < mbenc_ctx->kernel_num; i++) {
6066         gpe_context = &mbenc_ctx->gpe_contexts[i];
6067
6068         gen9_hevc_vme_init_gpe_context(ctx, gpe_context,
6069                                        hevc_mbenc_curbe_size[i],
6070                                        0);
6071         gen9_hevc_vme_init_scoreboard(gpe_context,
6072                                       0xFF,
6073                                       priv_state->use_hw_scoreboard,
6074                                       priv_state->use_hw_non_stalling_scoreborad);
6075
6076         memset(&kernel_info, 0, sizeof(kernel_info));
6077         gen9_hevc_get_kernel_header_and_size((void *)hevc_enc_kernel_ptr,
6078                                              hevc_enc_kernel_size,
6079                                              GEN9_ENC_MBENC,
6080                                              i,
6081                                              &kernel_info);
6082         gen8_gpe_load_kernels(ctx,
6083                               gpe_context,
6084                               &kernel_info,
6085                               1);
6086     }
6087 }
6088
6089 static void
6090 gen9_hevc_vme_brc_context_init(VADriverContextP ctx,
6091                                struct intel_encoder_context *encoder_context)
6092 {
6093     struct encoder_vme_mfc_context *vme_context = NULL;
6094     struct gen9_hevc_encoder_context *priv_ctx = NULL;
6095     struct gen9_hevc_encoder_state *priv_state = NULL;
6096     struct i965_gpe_context *gpe_context = NULL;
6097     struct i965_kernel kernel_info;
6098     struct gen9_hevc_brc_context *brc_ctx = NULL;
6099     int i = 0;
6100
6101     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
6102     priv_ctx = (struct gen9_hevc_encoder_context *)vme_context->private_enc_ctx;
6103     priv_state = (struct gen9_hevc_encoder_state *)vme_context->private_enc_state;
6104
6105     brc_ctx = &priv_ctx->brc_context;
6106
6107     for (i = 0; i < GEN9_HEVC_ENC_BRC_NUM; i++) {
6108         gpe_context = &brc_ctx->gpe_contexts[i];
6109
6110         gen9_hevc_vme_init_gpe_context(ctx, gpe_context,
6111                                        hevc_brc_curbe_size[i],
6112                                        0);
6113         gen9_hevc_vme_init_scoreboard(gpe_context,
6114                                       0xFF,
6115                                       priv_state->use_hw_scoreboard,
6116                                       priv_state->use_hw_non_stalling_scoreborad);
6117
6118         memset(&kernel_info, 0, sizeof(kernel_info));
6119         gen9_hevc_get_kernel_header_and_size((void *)hevc_enc_kernel_ptr,
6120                                              hevc_enc_kernel_size,
6121                                              GEN9_ENC_BRC,
6122                                              i,
6123                                              &kernel_info);
6124         gen8_gpe_load_kernels(ctx,
6125                               gpe_context,
6126                               &kernel_info,
6127                               1);
6128     }
6129 }
6130
6131 static void
6132 gen9_hevc_vme_scaling_context_destroy(struct gen9_hevc_scaling_context *scaling_context)
6133 {
6134     int i;
6135
6136     for (i = 0; i < NUM_HEVC_ENC_SCALING; i++)
6137         gen8_gpe_context_destroy(&scaling_context->gpe_contexts[i]);
6138 }
6139
6140 static void
6141 gen9_hevc_vme_me_context_destroy(struct gen9_hevc_me_context *me_context)
6142 {
6143     int i, j;
6144
6145     for (i = 0; i < NUM_HEVC_ENC_ME; i++)
6146         for (j = 0; j < NUM_HEVC_ENC_ME_TYPES; j++)
6147             gen8_gpe_context_destroy(&me_context->gpe_context[j][i]);
6148 }
6149
6150 static void
6151 gen9_hevc_vme_mbenc_context_destroy(struct gen9_hevc_mbenc_context *mbenc_context)
6152 {
6153     int i;
6154
6155     for (i = 0; i < mbenc_context->kernel_num; i++)
6156         gen8_gpe_context_destroy(&mbenc_context->gpe_contexts[i]);
6157 }
6158
6159 static void
6160 gen9_hevc_vme_brc_context_destroy(struct gen9_hevc_brc_context *brc_context)
6161 {
6162     int i;
6163
6164     for (i = 0; i < GEN9_HEVC_ENC_BRC_NUM; i++)
6165         gen8_gpe_context_destroy(&brc_context->gpe_contexts[i]);
6166 }
6167
6168 static void
6169 gen9_hevc_vme_kernels_context_init(VADriverContextP ctx,
6170                                    struct intel_encoder_context *encoder_context)
6171 {
6172     gen9_hevc_vme_scaling_context_init(ctx, encoder_context);
6173     gen9_hevc_vme_me_context_init(ctx, encoder_context);
6174     gen9_hevc_vme_mbenc_context_init(ctx, encoder_context);
6175     gen9_hevc_vme_brc_context_init(ctx, encoder_context);
6176 }
6177
6178 static void
6179 gen9_hevc_vme_kernels_context_destroy(struct encoder_vme_mfc_context *vme_context)
6180 {
6181     struct gen9_hevc_encoder_context *priv_ctx = NULL;
6182
6183     priv_ctx = (struct gen9_hevc_encoder_context *)vme_context->private_enc_ctx;
6184
6185     gen9_hevc_vme_scaling_context_destroy(&priv_ctx->scaling_context);
6186     gen9_hevc_vme_me_context_destroy(&priv_ctx->me_context);
6187     gen9_hevc_vme_mbenc_context_destroy(&priv_ctx->mbenc_context);
6188     gen9_hevc_vme_brc_context_destroy(&priv_ctx->brc_context);
6189 }
6190
6191 static void
6192 gen9_hevc_vme_context_destroy(void *context)
6193 {
6194     struct encoder_vme_mfc_context *vme_context = (struct encoder_vme_mfc_context *)context;
6195
6196     if (!vme_context)
6197         return;
6198
6199     gen9_hevc_enc_free_resources(vme_context);
6200
6201     gen9_hevc_vme_kernels_context_destroy(vme_context);
6202
6203     if (vme_context->private_enc_ctx) free(vme_context->private_enc_ctx);
6204     if (vme_context->generic_enc_state) free(vme_context->generic_enc_state);
6205     if (vme_context->private_enc_state) free(vme_context->private_enc_state);
6206
6207     free(vme_context);
6208 }
6209
6210 #define PAK_IMPLEMENTATION_START
6211
6212 static void
6213 gen9_hevc_pak_pipe_mode_select(VADriverContextP ctx,
6214                                struct encode_state *encode_state,
6215                                struct intel_encoder_context *encoder_context)
6216 {
6217     struct intel_batchbuffer *batch = encoder_context->base.batch;
6218
6219     BEGIN_BCS_BATCH(batch, 6);
6220
6221     OUT_BCS_BATCH(batch, HCP_PIPE_MODE_SELECT | (6 - 2));
6222     OUT_BCS_BATCH(batch,
6223                   (0 << 5) |
6224                   (0 << 3) |
6225                   HCP_CODEC_SELECT_ENCODE);
6226     OUT_BCS_BATCH(batch, 0);
6227     OUT_BCS_BATCH(batch, 0);
6228     OUT_BCS_BATCH(batch, 0);
6229     OUT_BCS_BATCH(batch, 0);
6230
6231     ADVANCE_BCS_BATCH(batch);
6232 }
6233
6234 static void
6235 gen9_hevc_pak_add_surface_state(VADriverContextP ctx,
6236                                 struct encode_state *encode_state,
6237                                 struct intel_encoder_context *encoder_context,
6238                                 struct object_surface *obj_surface,
6239                                 enum GEN9_HEVC_ENC_SURFACE_TYPE type)
6240 {
6241     struct intel_batchbuffer *batch = encoder_context->base.batch;
6242
6243     BEGIN_BCS_BATCH(batch, 3);
6244
6245     OUT_BCS_BATCH(batch, HCP_SURFACE_STATE | (3 - 2));
6246     OUT_BCS_BATCH(batch,
6247                   (type << 28) |
6248                   (obj_surface->width - 1));
6249     OUT_BCS_BATCH(batch,
6250                   (((obj_surface->fourcc == VA_FOURCC_P010) ?
6251                     SURFACE_FORMAT_P010 :
6252                     SURFACE_FORMAT_PLANAR_420_8) << 28) |
6253                   (obj_surface->y_cb_offset));
6254
6255     ADVANCE_BCS_BATCH(batch);
6256 }
6257
6258 #define OUT_BUFFER(buf_bo, is_target, ma)                               \
6259     do {                                                                \
6260         if (buf_bo) {                                                   \
6261             OUT_RELOC64(batch,                                          \
6262                           buf_bo,                                       \
6263                           I915_GEM_DOMAIN_RENDER,                       \
6264                           is_target ? I915_GEM_DOMAIN_RENDER : 0,       \
6265                           0);                                           \
6266         } else {                                                        \
6267             OUT_BCS_BATCH(batch, 0);                                    \
6268             OUT_BCS_BATCH(batch, 0);                                    \
6269         }                                                               \
6270         if (ma)                                                         \
6271             OUT_BCS_BATCH(batch, priv_ctx->mocs);                       \
6272     } while (0);
6273
6274 #define OUT_BUFFER_MA_TARGET(buf_bo)       OUT_BUFFER(buf_bo, 1, 1)
6275 #define OUT_BUFFER_MA_REFERENCE(buf_bo)    OUT_BUFFER(buf_bo, 0, 1)
6276 #define OUT_BUFFER_NMA_REFERENCE(buf_bo)   OUT_BUFFER(buf_bo, 0, 0)
6277
6278 static void
6279 gen9_hevc_pak_add_pipe_buf_addr_state(VADriverContextP ctx,
6280                                       struct encode_state *encode_state,
6281                                       struct intel_encoder_context *encoder_context)
6282 {
6283     struct i965_driver_data *i965 = i965_driver_data(ctx);
6284     struct intel_batchbuffer *batch = encoder_context->base.batch;
6285     struct encoder_vme_mfc_context *pak_context = NULL;
6286     struct gen9_hevc_encoder_context *priv_ctx = NULL;
6287     dri_bo *bo = NULL;
6288     int i = 0;
6289
6290     pak_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
6291     priv_ctx = (struct gen9_hevc_encoder_context *)pak_context->private_enc_ctx;
6292
6293     if (IS_KBL(i965->intel.device_info) || IS_GLK(i965->intel.device_info)) {
6294         BEGIN_BCS_BATCH(batch, 104);
6295
6296         OUT_BCS_BATCH(batch, HCP_PIPE_BUF_ADDR_STATE | (104 - 2));
6297     } else {
6298         BEGIN_BCS_BATCH(batch, 95);
6299
6300         OUT_BCS_BATCH(batch, HCP_PIPE_BUF_ADDR_STATE | (95 - 2));
6301     }
6302
6303     OUT_BUFFER_MA_TARGET(priv_ctx->reconstructed_object.obj_surface->bo);
6304     OUT_BUFFER_MA_TARGET(priv_ctx->deblocking_filter_line_buffer.bo);
6305     OUT_BUFFER_MA_TARGET(priv_ctx->deblocking_filter_tile_line_buffer.bo);
6306     OUT_BUFFER_MA_TARGET(priv_ctx->deblocking_filter_tile_column_buffer.bo);
6307     OUT_BUFFER_MA_TARGET(priv_ctx->metadata_line_buffer.bo);
6308     OUT_BUFFER_MA_TARGET(priv_ctx->metadata_tile_line_buffer.bo);
6309     OUT_BUFFER_MA_TARGET(priv_ctx->metadata_tile_column_buffer.bo);
6310     OUT_BUFFER_MA_TARGET(priv_ctx->sao_line_buffer.bo);
6311     OUT_BUFFER_MA_TARGET(priv_ctx->sao_tile_line_buffer.bo);
6312     OUT_BUFFER_MA_TARGET(priv_ctx->sao_tile_column_buffer.bo);
6313     OUT_BUFFER_MA_TARGET(priv_ctx->
6314                          mv_temporal_buffer[GEN9_MAX_MV_TEMPORAL_BUFFERS - 1].bo);
6315     OUT_BUFFER_MA_TARGET(NULL);
6316
6317     for (i = 0; i < GEN9_MAX_REF_SURFACES; i++) {
6318         if (priv_ctx->reference_surfaces[i].obj_surface &&
6319             priv_ctx->reference_surfaces[i].obj_surface->bo) {
6320             bo = priv_ctx->reference_surfaces[i].obj_surface->bo;
6321
6322             OUT_BUFFER_NMA_REFERENCE(bo);
6323         } else
6324             OUT_BUFFER_NMA_REFERENCE(NULL);
6325     }
6326     OUT_BCS_BATCH(batch, priv_ctx->mocs);
6327
6328     OUT_BUFFER_MA_TARGET(priv_ctx->
6329                          uncompressed_picture_source.obj_surface->bo);
6330     OUT_BUFFER_MA_TARGET(NULL);
6331     OUT_BUFFER_MA_TARGET(NULL);
6332     OUT_BUFFER_MA_TARGET(NULL);
6333
6334     for (i = 0; i < GEN9_MAX_MV_TEMPORAL_BUFFERS - 1; i++) {
6335         bo = priv_ctx->mv_temporal_buffer[i].bo;
6336
6337         if (bo) {
6338             OUT_BUFFER_NMA_REFERENCE(bo);
6339         } else
6340             OUT_BUFFER_NMA_REFERENCE(NULL);
6341     }
6342     OUT_BCS_BATCH(batch, priv_ctx->mocs);
6343
6344     OUT_BUFFER_MA_TARGET(NULL);
6345     OUT_BUFFER_MA_TARGET(NULL);
6346     OUT_BUFFER_MA_TARGET(NULL);
6347     OUT_BUFFER_MA_TARGET(NULL);
6348
6349     if (IS_KBL(i965->intel.device_info) || IS_GLK(i965->intel.device_info)) {
6350         for (i = 0; i < 9; i++)
6351             OUT_BCS_BATCH(batch, 0);
6352     }
6353
6354     ADVANCE_BCS_BATCH(batch);
6355 }
6356
6357 static void
6358 gen9_hevc_pak_add_ind_obj_base_addr_state(VADriverContextP ctx,
6359                                           struct encode_state *encode_state,
6360                                           struct intel_encoder_context *encoder_context)
6361 {
6362     struct intel_batchbuffer *batch = encoder_context->base.batch;
6363     struct encoder_vme_mfc_context *pak_context = NULL;
6364     struct gen9_hevc_encoder_context *priv_ctx = NULL;
6365     struct gen9_hevc_encoder_state *priv_state = NULL;
6366
6367     pak_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
6368     priv_ctx = (struct gen9_hevc_encoder_context *)pak_context->private_enc_ctx;
6369     priv_state = (struct gen9_hevc_encoder_state *)pak_context->private_enc_state;
6370
6371     BEGIN_BCS_BATCH(batch, 14);
6372
6373     OUT_BCS_BATCH(batch, HCP_IND_OBJ_BASE_ADDR_STATE | (14 - 2));
6374     OUT_BUFFER_MA_REFERENCE(NULL);
6375     OUT_BUFFER_NMA_REFERENCE(NULL);
6376
6377     OUT_RELOC64(batch,
6378                 priv_ctx->res_mb_code_surface.bo,
6379                 I915_GEM_DOMAIN_INSTRUCTION, 0,
6380                 priv_state->mb_data_offset);
6381     OUT_BCS_BATCH(batch, priv_ctx->mocs);
6382
6383     OUT_RELOC64(batch,
6384                 priv_ctx->indirect_pak_bse_object.bo,
6385                 I915_GEM_DOMAIN_RENDER, I915_GEM_DOMAIN_RENDER,
6386                 priv_ctx->indirect_pak_bse_object.offset);
6387     OUT_BCS_BATCH(batch, priv_ctx->mocs);
6388
6389     OUT_RELOC64(batch,
6390                 priv_ctx->indirect_pak_bse_object.bo,
6391                 I915_GEM_DOMAIN_RENDER, 0,
6392                 priv_ctx->indirect_pak_bse_object.end_offset);
6393
6394     ADVANCE_BCS_BATCH(batch);
6395 }
6396
6397 static void
6398 gen9_hevc_pak_set_qm(VADriverContextP ctx,
6399                      int size_id,
6400                      int color_component,
6401                      int pred_type,
6402                      int dc,
6403                      unsigned int *qm,
6404                      int qm_length,
6405                      struct intel_encoder_context *encoder_context)
6406 {
6407     struct intel_batchbuffer *batch = encoder_context->base.batch;
6408     unsigned int qm_buffer[16];
6409
6410     memset(qm_buffer, 0, sizeof(qm_buffer));
6411     memcpy(qm_buffer, qm, qm_length * 4);
6412
6413     BEGIN_BCS_BATCH(batch, 18);
6414
6415     OUT_BCS_BATCH(batch, HCP_QM_STATE | (18 - 2));
6416     OUT_BCS_BATCH(batch,
6417                   dc << 5 |
6418                   color_component << 3 |
6419                   size_id << 1 |
6420                   pred_type);
6421     intel_batchbuffer_data(batch, qm_buffer, 16 * 4);
6422
6423     ADVANCE_BCS_BATCH(batch);
6424 }
6425
6426 static unsigned int qm_default[16] = {
6427     0x10101010, 0x10101010, 0x10101010, 0x10101010,
6428     0x10101010, 0x10101010, 0x10101010, 0x10101010,
6429     0x10101010, 0x10101010, 0x10101010, 0x10101010,
6430     0x10101010, 0x10101010, 0x10101010, 0x10101010
6431 };
6432
6433 static void
6434 gen9_hevc_pak_add_qm_state(VADriverContextP ctx,
6435                            struct encode_state *encode_state,
6436                            struct intel_encoder_context *encoder_context)
6437 {
6438     int i;
6439
6440     for (i = 0; i < 6; i++)
6441         gen9_hevc_pak_set_qm(ctx,
6442                              0, i % 3, i / 3, 0,
6443                              qm_default, 4,
6444                              encoder_context);
6445
6446     for (i = 0; i < 6; i++)
6447         gen9_hevc_pak_set_qm(ctx,
6448                              1, i % 3, i / 3, 0,
6449                              qm_default, 16,
6450                              encoder_context);
6451
6452     for (i = 0; i < 6; i++)
6453         gen9_hevc_pak_set_qm(ctx,
6454                              2, i % 3, i / 3, 16,
6455                              qm_default, 16,
6456                              encoder_context);
6457
6458     for (i = 0; i < 2; i++)
6459         gen9_hevc_pak_set_qm(ctx,
6460                              3, 0, i % 2, 16,
6461                              qm_default, 16,
6462                              encoder_context);
6463 }
6464
6465 static void
6466 gen9_hevc_pak_set_fqm(VADriverContextP ctx,
6467                       int size_id,
6468                       int color_component,
6469                       int pred_type,
6470                       int dc,
6471                       unsigned int *fqm,
6472                       int fqm_length,
6473                       struct intel_encoder_context *encoder_context)
6474 {
6475     struct intel_batchbuffer *batch = encoder_context->base.batch;
6476     unsigned int fqm_buffer[32];
6477
6478     memset(fqm_buffer, 0, sizeof(fqm_buffer));
6479     memcpy(fqm_buffer, fqm, fqm_length * 4);
6480
6481     BEGIN_BCS_BATCH(batch, 34);
6482
6483     OUT_BCS_BATCH(batch, HCP_FQM_STATE | (34 - 2));
6484     OUT_BCS_BATCH(batch,
6485                   dc << 16 |
6486                   color_component << 3 |
6487                   size_id << 1 |
6488                   pred_type);
6489     intel_batchbuffer_data(batch, fqm_buffer, 32 * 4);
6490
6491     ADVANCE_BCS_BATCH(batch);
6492 }
6493
6494 static unsigned int fm_default[32] = {
6495     0x10001000, 0x10001000, 0x10001000, 0x10001000,
6496     0x10001000, 0x10001000, 0x10001000, 0x10001000,
6497     0x10001000, 0x10001000, 0x10001000, 0x10001000,
6498     0x10001000, 0x10001000, 0x10001000, 0x10001000,
6499     0x10001000, 0x10001000, 0x10001000, 0x10001000,
6500     0x10001000, 0x10001000, 0x10001000, 0x10001000,
6501     0x10001000, 0x10001000, 0x10001000, 0x10001000,
6502     0x10001000, 0x10001000, 0x10001000, 0x10001000
6503 };
6504
6505 static void
6506 gen9_hevc_pak_add_fm_state(VADriverContextP ctx,
6507                            struct encode_state *encode_state,
6508                            struct intel_encoder_context *encoder_context)
6509 {
6510     gen9_hevc_pak_set_fqm(ctx,
6511                           0, 0, 0, 0,
6512                           fm_default, 8,
6513                           encoder_context);
6514     gen9_hevc_pak_set_fqm(ctx,
6515                           0, 0, 1, 0,
6516                           fm_default, 8,
6517                           encoder_context);
6518     gen9_hevc_pak_set_fqm(ctx,
6519                           1, 0, 0, 0,
6520                           fm_default, 32,
6521                           encoder_context);
6522     gen9_hevc_pak_set_fqm(ctx,
6523                           1, 0, 1, 0,
6524                           fm_default, 32,
6525                           encoder_context);
6526     gen9_hevc_pak_set_fqm(ctx,
6527                           2, 0, 0, 0x1000,
6528                           fm_default, 0,
6529                           encoder_context);
6530     gen9_hevc_pak_set_fqm(ctx,
6531                           2, 0, 1, 0x1000,
6532                           fm_default, 0,
6533                           encoder_context);
6534     gen9_hevc_pak_set_fqm(ctx,
6535                           3, 0, 0, 0x1000,
6536                           fm_default, 0,
6537                           encoder_context);
6538     gen9_hevc_pak_set_fqm(ctx,
6539                           3, 0, 1, 0x1000,
6540                           fm_default, 0,
6541                           encoder_context);
6542 }
6543
6544 static void
6545 gen9_hevc_set_reflist(VADriverContextP ctx,
6546                       struct gen9_hevc_encoder_context *priv_ctx,
6547                       VAEncPictureParameterBufferHEVC *pic_param,
6548                       VAEncSliceParameterBufferHEVC *slice_param,
6549                       int list_idx,
6550                       struct intel_batchbuffer *batch)
6551 {
6552     struct i965_driver_data *i965 = i965_driver_data(ctx);
6553     int num_ref_minus1 = (list_idx ?
6554                           slice_param->num_ref_idx_l1_active_minus1 :
6555                           slice_param->num_ref_idx_l0_active_minus1);
6556     VAPictureHEVC *ref_list = (list_idx ?
6557                                slice_param->ref_pic_list1 :
6558                                slice_param->ref_pic_list0);
6559     VAPictureHEVC *curr_pic = &pic_param->decoded_curr_pic;
6560     struct object_surface *obj_surface = NULL;
6561     int frame_idx;
6562     int i = 0, j = 0;
6563
6564     BEGIN_BCS_BATCH(batch, 18);
6565
6566     OUT_BCS_BATCH(batch, HCP_REF_IDX_STATE | (18 - 2));
6567     OUT_BCS_BATCH(batch,
6568                   num_ref_minus1 << 1 |
6569                   list_idx);
6570
6571     for (i = 0; i < 16; i++) {
6572         frame_idx = -1;
6573         obj_surface = SURFACE(ref_list[i].picture_id);
6574         if (i < MIN((num_ref_minus1 + 1), GEN9_MAX_REF_SURFACES) && obj_surface) {
6575             for (j = 0; j < GEN9_MAX_REF_SURFACES; j++) {
6576                 if (obj_surface == priv_ctx->reference_surfaces[j].obj_surface) {
6577                     frame_idx = j;
6578                     break;
6579                 }
6580             }
6581         }
6582
6583         if (i < MIN((num_ref_minus1 + 1), GEN9_MAX_REF_SURFACES) &&
6584             frame_idx >= 0) {
6585             OUT_BCS_BATCH(batch,
6586                           1 << 15 |
6587                           0 << 14 |
6588                           !!(ref_list[i].flags & VA_PICTURE_HEVC_LONG_TERM_REFERENCE) << 13 |
6589                           0 << 12 |
6590                           0 << 11 |
6591                           frame_idx << 8 |
6592                           (CLAMP(-128, 127, curr_pic->pic_order_cnt -
6593                                  ref_list[i].pic_order_cnt) & 0xff));
6594         } else {
6595             OUT_BCS_BATCH(batch, 0);
6596         }
6597     }
6598
6599     ADVANCE_BCS_BATCH(batch);
6600 }
6601
6602 static void
6603 gen9_hevc_pak_add_slice_state(VADriverContextP ctx,
6604                               struct encode_state *encode_state,
6605                               struct intel_encoder_context *encoder_context,
6606                               int slice_idx,
6607                               struct intel_batchbuffer *batch)
6608 {
6609     struct i965_driver_data *i965 = i965_driver_data(ctx);
6610     struct encoder_vme_mfc_context *pak_context = NULL;
6611     struct gen9_hevc_encoder_state *priv_state = NULL;
6612     VAEncPictureParameterBufferHEVC *pic_param = NULL;
6613     VAEncSliceParameterBufferHEVC *slice_param = NULL;
6614     int slice_hor_pos, slice_ver_pos, next_slice_hor_pos, next_slice_ver_pos;
6615     int slice_type = 0, slice_end = 0, last_slice = 0;
6616     int collocated_ref_idx = 0;
6617
6618     pak_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
6619     priv_state = (struct gen9_hevc_encoder_state *)pak_context->private_enc_state;
6620     pic_param = (VAEncPictureParameterBufferHEVC *)encode_state->pic_param_ext->buffer;
6621     slice_param = (VAEncSliceParameterBufferHEVC *)encode_state->slice_params_ext[slice_idx]->buffer;
6622
6623     slice_type = slice_param->slice_type;
6624     slice_end = slice_param->slice_segment_address + slice_param->num_ctu_in_slice;
6625     slice_hor_pos = slice_param->slice_segment_address % priv_state->width_in_lcu;
6626     slice_ver_pos = slice_param->slice_segment_address / priv_state->width_in_lcu;
6627     next_slice_hor_pos = slice_end % priv_state->width_in_lcu;
6628     next_slice_ver_pos = slice_end / priv_state->width_in_lcu;
6629
6630     if (slice_end >= priv_state->width_in_lcu * priv_state->height_in_lcu ||
6631         slice_idx == encode_state->num_slice_params_ext - 1)
6632         last_slice = 1;
6633
6634     if (priv_state->picture_coding_type != HEVC_SLICE_I &&
6635         slice_param->slice_fields.bits.slice_temporal_mvp_enabled_flag &&
6636         slice_param->slice_fields.bits.collocated_from_l0_flag)
6637         collocated_ref_idx = pic_param->collocated_ref_pic_index;
6638
6639     if (IS_KBL(i965->intel.device_info) || IS_GLK(i965->intel.device_info)) {
6640         BEGIN_BCS_BATCH(batch, 11);
6641
6642         OUT_BCS_BATCH(batch, HCP_SLICE_STATE | (11 - 2));
6643     } else {
6644         BEGIN_BCS_BATCH(batch, 9);
6645
6646         OUT_BCS_BATCH(batch, HCP_SLICE_STATE | (9 - 2));
6647     }
6648
6649     OUT_BCS_BATCH(batch,
6650                   slice_ver_pos << 16 |
6651                   slice_hor_pos);
6652     OUT_BCS_BATCH(batch,
6653                   next_slice_ver_pos << 16 |
6654                   next_slice_hor_pos);
6655     OUT_BCS_BATCH(batch,
6656                   (slice_param->slice_cr_qp_offset & 0x1f) << 17 |
6657                   (slice_param->slice_cb_qp_offset & 0x1f) << 12 |
6658                   (pic_param->pic_init_qp + slice_param->slice_qp_delta) << 6 |
6659                   slice_param->slice_fields.bits.slice_temporal_mvp_enabled_flag << 5 |
6660                   slice_param->slice_fields.bits.dependent_slice_segment_flag << 4 |
6661                   last_slice << 2 |
6662                   slice_type);
6663     OUT_BCS_BATCH(batch,
6664                   collocated_ref_idx << 26 |
6665                   (slice_param->max_num_merge_cand - 1)  << 23 |
6666                   slice_param->slice_fields.bits.cabac_init_flag << 22 |
6667                   slice_param->luma_log2_weight_denom << 19 |
6668                   (slice_param->luma_log2_weight_denom + slice_param->delta_chroma_log2_weight_denom) << 16 |
6669                   slice_param->slice_fields.bits.collocated_from_l0_flag << 15 |
6670                   priv_state->low_delay << 14 |
6671                   slice_param->slice_fields.bits.mvd_l1_zero_flag << 13 |
6672                   slice_param->slice_fields.bits.slice_sao_luma_flag << 12 |
6673                   slice_param->slice_fields.bits.slice_sao_chroma_flag << 11 |
6674                   slice_param->slice_fields.bits.slice_loop_filter_across_slices_enabled_flag << 10 |
6675                   (slice_param->slice_beta_offset_div2 & 0xf) << 5 |
6676                   (slice_param->slice_tc_offset_div2 & 0xf) << 1 |
6677                   slice_param->slice_fields.bits.slice_deblocking_filter_disabled_flag);
6678     OUT_BCS_BATCH(batch, 0);
6679
6680     if (!pic_param->pic_fields.bits.reference_pic_flag &&
6681         priv_state->picture_coding_type != HEVC_SLICE_I)
6682         OUT_BCS_BATCH(batch, 0 << 26 |
6683                       8 << 20);
6684     else
6685         OUT_BCS_BATCH(batch, 5 << 26 |
6686                       11 << 20);
6687
6688     OUT_BCS_BATCH(batch,
6689                   1 << 10 |
6690                   1 << 9  |
6691                   1 << 2  |
6692                   1 << 1  |
6693                   0);
6694     OUT_BCS_BATCH(batch, 0);
6695
6696     if (IS_KBL(i965->intel.device_info) || IS_GLK(i965->intel.device_info)) {
6697         OUT_BCS_BATCH(batch, 0);
6698         OUT_BCS_BATCH(batch, 0);
6699     }
6700
6701     ADVANCE_BCS_BATCH(batch);
6702 }
6703
6704 static int
6705 gen9_hevc_find_skipemulcnt(unsigned char *buf, unsigned int bits_length)
6706 {
6707     int skip_cnt = 0, i = 0;
6708
6709     if ((bits_length >> 3) < 6)
6710         return 0;
6711
6712     for (i = 0; i < 3; i++)
6713         if (buf[i] != 0)
6714             break;
6715
6716     if (i > 1) {
6717         if (buf[i] == 1)
6718             skip_cnt = i + 3;
6719     }
6720
6721     return skip_cnt;
6722 }
6723
6724 static void
6725 gen9_hevc_pak_insert_object(unsigned int *data_buffer,
6726                             unsigned int data_size,
6727                             unsigned char emulation_flag,
6728                             int is_last_header,
6729                             int is_end_of_slice,
6730                             int skip_emul_byte_cnt,
6731                             struct intel_batchbuffer *batch)
6732 {
6733     int length_in_dws = ALIGN(data_size, 32) >> 5;
6734     int data_bits_in_last_dw = data_size & 0x1f;
6735     int skip_cnt = skip_emul_byte_cnt;
6736
6737     if (data_bits_in_last_dw == 0)
6738         data_bits_in_last_dw = 32;
6739
6740     if (emulation_flag) {
6741         if (!skip_cnt)
6742             skip_cnt = gen9_hevc_find_skipemulcnt((unsigned char *)data_buffer,
6743                                                   data_size);
6744     }
6745
6746     BEGIN_BCS_BATCH(batch, length_in_dws + 2);
6747
6748     OUT_BCS_BATCH(batch, HCP_INSERT_PAK_OBJECT | (length_in_dws + 2 - 2));
6749     OUT_BCS_BATCH(batch,
6750                   (0 << 31) |
6751                   (0 << 16) |
6752                   (0 << 15) |
6753                   (data_bits_in_last_dw << 8) |
6754                   (skip_cnt << 4) |
6755                   ((!!emulation_flag) << 3) |
6756                   ((!!is_last_header) << 2) |
6757                   ((!!is_end_of_slice) << 1) |
6758                   (0 << 0));
6759     intel_batchbuffer_data(batch, data_buffer, length_in_dws * 4);
6760
6761     ADVANCE_BCS_BATCH(batch);
6762 }
6763
6764 static void
6765 gen9_hevc_pak_add_refs(VADriverContextP ctx,
6766                        struct encode_state *encode_state,
6767                        struct intel_encoder_context *encoder_context,
6768                        int slice_idx,
6769                        struct intel_batchbuffer *batch)
6770 {
6771     struct encoder_vme_mfc_context *pak_context = NULL;
6772     struct gen9_hevc_encoder_context *priv_ctx = NULL;
6773     VAEncPictureParameterBufferHEVC *pic_param = NULL;
6774     VAEncSliceParameterBufferHEVC *slice_param = NULL;
6775
6776     pak_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
6777     priv_ctx = (struct gen9_hevc_encoder_context *)pak_context->private_enc_ctx;
6778     pic_param = (VAEncPictureParameterBufferHEVC *)encode_state->pic_param_ext->buffer;
6779     slice_param = (VAEncSliceParameterBufferHEVC *)encode_state->slice_params_ext[slice_idx]->buffer;
6780
6781     if (slice_param->slice_type == HEVC_SLICE_I)
6782         return;
6783
6784     gen9_hevc_set_reflist(ctx, priv_ctx, pic_param, slice_param, 0, batch);
6785
6786     if (slice_param->slice_type == HEVC_SLICE_P)
6787         return;
6788
6789     gen9_hevc_set_reflist(ctx, priv_ctx, pic_param, slice_param, 1, batch);
6790 }
6791
6792 static void
6793 gen9_hevc_pak_insert_packed_data(VADriverContextP ctx,
6794                                  struct encode_state *encode_state,
6795                                  struct intel_encoder_context *encoder_context,
6796                                  int slice_idx,
6797                                  struct intel_batchbuffer *batch)
6798 {
6799     VAEncPackedHeaderParameterBuffer *param = NULL;
6800     unsigned int *header_data = NULL;
6801     unsigned int length_in_bits = 0;
6802     int packed_type = 0;
6803     int idx = 0, idx_offset = 0;
6804     int i = 0;
6805
6806     for (i = 0; i < 4; i++) {
6807         idx_offset = 0;
6808         switch (i) {
6809         case 0:
6810             packed_type = VAEncPackedHeaderHEVC_VPS;
6811             break;
6812         case 1:
6813             packed_type = VAEncPackedHeaderHEVC_VPS;
6814             idx_offset = 1;
6815             break;
6816         case 2:
6817             packed_type = VAEncPackedHeaderHEVC_PPS;
6818             break;
6819         case 3:
6820             packed_type = VAEncPackedHeaderHEVC_SEI;
6821             break;
6822         default:
6823             break;
6824         }
6825
6826         idx = va_enc_packed_type_to_idx(packed_type) + idx_offset;
6827         if (encode_state->packed_header_data[idx]) {
6828             param = (VAEncPackedHeaderParameterBuffer *)encode_state->packed_header_param[idx]->buffer;
6829             header_data = (unsigned int *)encode_state->packed_header_data[idx]->buffer;
6830             length_in_bits = param->bit_length;
6831
6832             gen9_hevc_pak_insert_object(header_data, length_in_bits,
6833                                         !param->has_emulation_bytes, 0, 0, 0,
6834                                         batch);
6835         }
6836     }
6837 }
6838
6839 static void
6840 gen9_hevc_pak_insert_slice_header(VADriverContextP ctx,
6841                                   struct encode_state *encode_state,
6842                                   struct intel_encoder_context *encoder_context,
6843                                   int slice_idx,
6844                                   struct intel_batchbuffer *batch)
6845 {
6846     VAEncPackedHeaderParameterBuffer *param = NULL;
6847     unsigned int *header_data = NULL;
6848     unsigned int length_in_bits = 0;
6849     int count = 0, start_index = -1;
6850     int i = 0;
6851
6852     count = encode_state->slice_rawdata_count[slice_idx];
6853     start_index = encode_state->slice_rawdata_index[slice_idx] &
6854                   SLICE_PACKED_DATA_INDEX_MASK;
6855
6856     for (i = 0; i < count; i++) {
6857         param = (VAEncPackedHeaderParameterBuffer *)
6858                 (encode_state->packed_header_params_ext[start_index + i]->buffer);
6859
6860         if (param->type == VAEncPackedHeaderSlice)
6861             continue;
6862
6863         header_data = (unsigned int *)encode_state->packed_header_data_ext[start_index]->buffer;
6864         length_in_bits = param->bit_length;
6865         gen9_hevc_pak_insert_object(header_data, length_in_bits,
6866                                     !param->has_emulation_bytes, 0, 0, 0,
6867                                     batch);
6868     }
6869
6870     start_index = -1;
6871     if (encode_state->slice_header_index[slice_idx] & SLICE_PACKED_DATA_INDEX_TYPE)
6872         start_index = encode_state->slice_header_index[slice_idx] &
6873                       SLICE_PACKED_DATA_INDEX_MASK;
6874     if (start_index == -1) {
6875         VAEncSequenceParameterBufferHEVC *seq_param = (VAEncSequenceParameterBufferHEVC *)encode_state->seq_param_ext->buffer;
6876         VAEncPictureParameterBufferHEVC *pic_param = (VAEncPictureParameterBufferHEVC *)encode_state->pic_param_ext->buffer;
6877         VAEncSliceParameterBufferHEVC *slice_param = (VAEncSliceParameterBufferHEVC *)encode_state->slice_params_ext[slice_idx]->buffer;
6878         unsigned char *slice_header = NULL;
6879         int slice_header_bits = 0;
6880
6881         slice_header_bits = build_hevc_slice_header(seq_param,
6882                                                     pic_param,
6883                                                     slice_param,
6884                                                     &slice_header,
6885                                                     0);
6886
6887         gen9_hevc_pak_insert_object((unsigned int *)slice_header, slice_header_bits,
6888                                     1, 1, 0, 5,
6889                                     batch);
6890
6891         free(slice_header);
6892     } else {
6893         param = (VAEncPackedHeaderParameterBuffer *)
6894                 (encode_state->packed_header_params_ext[start_index]->buffer);
6895         header_data = (unsigned int *)encode_state->packed_header_data_ext[start_index]->buffer;
6896         length_in_bits = param->bit_length;
6897
6898         gen9_hevc_pak_insert_object(header_data, length_in_bits,
6899                                     !param->has_emulation_bytes, 1, 0, 0,
6900                                     batch);
6901     }
6902 }
6903
6904 static VAStatus
6905 gen9_hevc_pak_slice_level(VADriverContextP ctx,
6906                           struct encode_state *encode_state,
6907                           struct intel_encoder_context *encoder_context,
6908                           int slice_idx)
6909 {
6910     struct intel_batchbuffer *batch = encoder_context->base.batch;
6911     struct encoder_vme_mfc_context *pak_context = NULL;
6912     struct gen9_hevc_encoder_context *priv_ctx = NULL;
6913     struct generic_enc_codec_state *generic_state = NULL;
6914     struct gen9_hevc_encoder_state *priv_state = NULL;
6915     VAEncSliceParameterBufferHEVC *slice_param = NULL;
6916     struct intel_batchbuffer *slice_batch = NULL;
6917     struct gpe_mi_batch_buffer_start_parameter second_level_batch;
6918     VAStatus va_status = VA_STATUS_SUCCESS;
6919     int slice_offset = 0;
6920
6921     pak_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
6922     priv_ctx = (struct gen9_hevc_encoder_context *)pak_context->private_enc_ctx;
6923     generic_state = (struct generic_enc_codec_state *)pak_context->generic_enc_state;
6924     priv_state = (struct gen9_hevc_encoder_state *)pak_context->private_enc_state;
6925     slice_param = (VAEncSliceParameterBufferHEVC *)encode_state->slice_params_ext[slice_idx]->buffer;
6926
6927     slice_batch = priv_ctx->res_pak_slice_batch_buffer;
6928
6929     if (generic_state->curr_pak_pass == 0) {
6930         slice_offset = intel_batchbuffer_used_size(slice_batch);
6931         priv_state->slice_batch_offset[slice_idx] = slice_offset;
6932
6933         if (slice_idx < encode_state->num_slice_params_ext - 1)
6934             priv_state->slice_start_lcu[slice_idx + 1] =
6935                 priv_state->slice_start_lcu[slice_idx] +
6936                 slice_param->num_ctu_in_slice;
6937
6938         gen9_hevc_pak_add_refs(ctx, encode_state, encoder_context,
6939                                slice_idx, slice_batch);
6940         gen9_hevc_pak_add_slice_state(ctx, encode_state, encoder_context,
6941                                       slice_idx, slice_batch);
6942
6943         if (slice_idx == 0)
6944             gen9_hevc_pak_insert_packed_data(ctx, encode_state, encoder_context,
6945                                              slice_idx, slice_batch);
6946
6947         gen9_hevc_pak_insert_slice_header(ctx, encode_state, encoder_context,
6948                                           slice_idx, slice_batch);
6949
6950         BEGIN_BCS_BATCH(slice_batch, 2);
6951         OUT_BCS_BATCH(slice_batch, 0);
6952         OUT_BCS_BATCH(slice_batch, MI_BATCH_BUFFER_END);
6953         ADVANCE_BCS_BATCH(slice_batch);
6954     } else
6955         slice_offset = priv_state->slice_batch_offset[slice_idx];
6956
6957     memset(&second_level_batch, 0, sizeof(second_level_batch));
6958     second_level_batch.offset = slice_offset;
6959     second_level_batch.is_second_level = 1;
6960     second_level_batch.bo = slice_batch->buffer;
6961     gen8_gpe_mi_batch_buffer_start(ctx, batch, &second_level_batch);
6962
6963     memset(&second_level_batch, 0, sizeof(second_level_batch));
6964     second_level_batch.offset = priv_state->slice_start_lcu[slice_idx] *
6965                                 priv_state->pak_obj_size;
6966     second_level_batch.is_second_level = 1;
6967     second_level_batch.bo = priv_ctx->res_mb_code_surface.bo;
6968     gen8_gpe_mi_batch_buffer_start(ctx, batch, &second_level_batch);
6969
6970     return va_status;
6971 }
6972
6973 static VAStatus
6974 gen9_hevc_pak_pipeline_prepare(VADriverContextP ctx,
6975                                struct encode_state *encode_state,
6976                                struct intel_encoder_context *encoder_context)
6977 {
6978     struct i965_driver_data *i965 = i965_driver_data(ctx);
6979     struct encoder_vme_mfc_context *pak_context = NULL;
6980     struct gen9_hevc_encoder_context *priv_ctx = NULL;
6981     struct gen9_hevc_encoder_state *priv_state = NULL;
6982     struct gen9_hevc_surface_priv *surface_priv;
6983     VAEncPictureParameterBufferHEVC *pic_param = NULL;
6984     struct i965_coded_buffer_segment *coded_buffer_segment = NULL;
6985     struct object_surface *obj_surface = NULL;
6986     struct object_buffer *obj_buffer = NULL;
6987     dri_bo *bo = NULL;
6988     int i = 0;
6989
6990     pak_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
6991     priv_ctx = (struct gen9_hevc_encoder_context *)pak_context->private_enc_ctx;
6992     priv_state = (struct gen9_hevc_encoder_state *)pak_context->private_enc_state;
6993     pic_param = (VAEncPictureParameterBufferHEVC *)encode_state->pic_param_ext->buffer;
6994
6995     if (priv_ctx->uncompressed_picture_source.obj_surface &&
6996         priv_ctx->uncompressed_picture_source.obj_surface->bo)
6997         dri_bo_unreference(priv_ctx->uncompressed_picture_source.obj_surface->bo);
6998     priv_ctx->uncompressed_picture_source.obj_surface = encode_state->input_yuv_object;
6999     priv_ctx->uncompressed_picture_source.surface_id = encoder_context->input_yuv_surface;
7000     dri_bo_reference(priv_ctx->uncompressed_picture_source.obj_surface->bo);
7001
7002     if (priv_ctx->reconstructed_object.obj_surface &&
7003         priv_ctx->reconstructed_object.obj_surface->bo)
7004         dri_bo_unreference(priv_ctx->reconstructed_object.obj_surface->bo);
7005     priv_ctx->reconstructed_object.obj_surface = encode_state->reconstructed_object;
7006     priv_ctx->reconstructed_object.surface_id = pic_param->decoded_curr_pic.picture_id;
7007     dri_bo_reference(priv_ctx->reconstructed_object.obj_surface->bo);
7008
7009     surface_priv = (struct gen9_hevc_surface_priv *)encode_state->reconstructed_object->private_data;
7010     if (surface_priv) {
7011         if (priv_ctx->mv_temporal_buffer[GEN9_MAX_MV_TEMPORAL_BUFFERS - 1].bo)
7012             dri_bo_unreference(priv_ctx->mv_temporal_buffer[GEN9_MAX_MV_TEMPORAL_BUFFERS - 1].bo);
7013         priv_ctx->mv_temporal_buffer[GEN9_MAX_MV_TEMPORAL_BUFFERS - 1].bo =
7014             surface_priv->motion_vector_temporal_bo;
7015         dri_bo_reference(surface_priv->motion_vector_temporal_bo);
7016     }
7017
7018     if (priv_state->picture_coding_type != HEVC_SLICE_I) {
7019         for (i = 0; i < GEN9_MAX_REF_SURFACES; i++) {
7020             obj_surface = encode_state->reference_objects[i];
7021             if (obj_surface && obj_surface->bo) {
7022                 if (priv_ctx->reference_surfaces[i].obj_surface &&
7023                     priv_ctx->reference_surfaces[i].obj_surface->bo)
7024                     dri_bo_unreference(priv_ctx->reference_surfaces[i].obj_surface->bo);
7025                 priv_ctx->reference_surfaces[i].obj_surface = obj_surface;
7026                 priv_ctx->reference_surfaces[i].surface_id = pic_param->reference_frames[i].picture_id;
7027                 dri_bo_reference(obj_surface->bo);
7028
7029                 surface_priv = (struct gen9_hevc_surface_priv *) obj_surface->private_data;
7030                 if (surface_priv) {
7031                     if (priv_ctx->mv_temporal_buffer[i].bo)
7032                         dri_bo_unreference(priv_ctx->mv_temporal_buffer[i].bo);
7033                     priv_ctx->mv_temporal_buffer[i].bo = surface_priv->motion_vector_temporal_bo;
7034                     dri_bo_reference(surface_priv->motion_vector_temporal_bo);
7035                 }
7036             } else {
7037                 break;
7038             }
7039         }
7040     }
7041
7042     obj_buffer = encode_state->coded_buf_object;
7043     bo = obj_buffer->buffer_store->bo;
7044
7045     if (priv_ctx->indirect_pak_bse_object.bo)
7046         dri_bo_unreference(priv_ctx->indirect_pak_bse_object.bo);
7047     priv_ctx->indirect_pak_bse_object.offset = I965_CODEDBUFFER_HEADER_SIZE;
7048     priv_ctx->indirect_pak_bse_object.end_offset = ALIGN((obj_buffer->size_element - 0x1000), 0x1000);
7049     priv_ctx->indirect_pak_bse_object.bo = bo;
7050     dri_bo_reference(priv_ctx->indirect_pak_bse_object.bo);
7051
7052     dri_bo_map(bo, 1);
7053
7054     if (!bo->virtual)
7055         return VA_STATUS_ERROR_INVALID_VALUE;
7056
7057     coded_buffer_segment = (struct i965_coded_buffer_segment *)bo->virtual;
7058     coded_buffer_segment->mapped = 0;
7059     coded_buffer_segment->codec = encoder_context->codec;
7060     coded_buffer_segment->status_support = 1;
7061
7062     dri_bo_unmap(bo);
7063
7064     if (priv_ctx->res_pak_slice_batch_buffer)
7065         intel_batchbuffer_free(priv_ctx->res_pak_slice_batch_buffer);
7066
7067     priv_ctx->res_pak_slice_batch_buffer =
7068         intel_batchbuffer_new(&i965->intel, I915_EXEC_BSD,
7069                               GEN9_HEVC_ENC_PAK_SLICE_STATE_SIZE *
7070                               encode_state->num_slice_params_ext);
7071     if (!priv_ctx->res_pak_slice_batch_buffer)
7072         return VA_STATUS_ERROR_ALLOCATION_FAILED;
7073
7074     for (i = 0; i < I965_MAX_NUM_SLICE; i++) {
7075         priv_state->slice_batch_offset[i] = 0;
7076         priv_state->slice_start_lcu[i] = 0;
7077     }
7078
7079     return VA_STATUS_SUCCESS;
7080 }
7081
7082 static VAStatus
7083 gen9_hevc_pak_picture_level(VADriverContextP ctx,
7084                             struct encode_state *encode_state,
7085                             struct intel_encoder_context *encoder_context)
7086 {
7087     struct intel_batchbuffer *batch = encoder_context->base.batch;
7088     struct encoder_vme_mfc_context *pak_context = NULL;
7089     struct gen9_hevc_encoder_context *priv_ctx = NULL;
7090     struct generic_enc_codec_state *generic_state = NULL;
7091     struct gen9_hevc_encoder_state *priv_state = NULL;
7092     struct gpe_mi_batch_buffer_start_parameter second_level_batch;
7093     struct hevc_encode_status_buffer *status_buffer = NULL;
7094     VAStatus va_status = VA_STATUS_SUCCESS;
7095     int i = 0;
7096
7097     pak_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
7098     priv_ctx = (struct gen9_hevc_encoder_context *)pak_context->private_enc_ctx;
7099     generic_state = (struct generic_enc_codec_state *)pak_context->generic_enc_state;
7100     priv_state = (struct gen9_hevc_encoder_state *)pak_context->private_enc_state;
7101     status_buffer = &priv_state->status_buffer;
7102
7103     if (generic_state->brc_enabled &&
7104         generic_state->curr_pak_pass) {
7105         gen9_hevc_conditional_end(ctx,
7106                                   batch, status_buffer->bo,
7107                                   status_buffer->status_image_mask_offset,
7108                                   0);
7109
7110         gen9_hevc_load_reg_mem(ctx,
7111                                batch, status_buffer->bo,
7112                                status_buffer->status_image_ctrl_offset,
7113                                status_buffer->mmio_image_ctrl_offset);
7114
7115         gen9_hevc_store_reg_mem(ctx, batch, priv_ctx->res_brc_pak_statistic_buffer.bo,
7116                                 offsetof(GEN9_HEVC_PAK_STATES, HEVC_ENC_IMAGE_STATUS_CONTROL_FOR_LAST_PASS),
7117                                 status_buffer->mmio_image_ctrl_offset);
7118
7119         gen9_hevc_store_reg_mem(ctx, batch, status_buffer->bo,
7120                                 status_buffer->status_image_ctrl_last_pass_offset,
7121                                 status_buffer->mmio_image_ctrl_offset);
7122     }
7123
7124     gen9_hevc_pak_pipe_mode_select(ctx, encode_state, encoder_context);
7125     gen9_hevc_pak_add_surface_state(ctx, encode_state, encoder_context,
7126                                     encode_state->input_yuv_object,
7127                                     GEN9_HEVC_ENC_SURFACE_SOURCE);
7128     gen9_hevc_pak_add_surface_state(ctx, encode_state, encoder_context,
7129                                     encode_state->reconstructed_object,
7130                                     GEN9_HEVC_ENC_SURFACE_RECON);
7131     gen9_hevc_pak_add_pipe_buf_addr_state(ctx, encode_state, encoder_context);
7132     gen9_hevc_pak_add_ind_obj_base_addr_state(ctx, encode_state, encoder_context);
7133     gen9_hevc_pak_add_qm_state(ctx, encode_state, encoder_context);
7134     gen9_hevc_pak_add_fm_state(ctx, encode_state, encoder_context);
7135
7136     if (generic_state->brc_enabled) {
7137         memset(&second_level_batch, 0, sizeof(second_level_batch));
7138
7139         second_level_batch.offset = generic_state->curr_pak_pass * priv_state->pic_state_size;
7140         second_level_batch.bo = priv_ctx->res_brc_pic_states_write_buffer.bo;
7141         second_level_batch.is_second_level = 1;
7142         gen8_gpe_mi_batch_buffer_start(ctx, batch, &second_level_batch);
7143     } else
7144         gen9_hevc_add_pic_state(ctx, encode_state, encoder_context, NULL, 0, 0);
7145
7146     for (i = 0; i < encode_state->num_slice_params_ext; i++) {
7147         va_status = gen9_hevc_pak_slice_level(ctx, encode_state, encoder_context, i);
7148         if (va_status != VA_STATUS_SUCCESS)
7149             goto EXIT;
7150     }
7151
7152 EXIT:
7153     return va_status;
7154 }
7155
7156 static void
7157 gen9_hevc_pak_read_status(VADriverContextP ctx,
7158                           struct intel_encoder_context *encoder_context)
7159 {
7160     struct intel_batchbuffer *batch = encoder_context->base.batch;
7161     struct encoder_vme_mfc_context *pak_context = NULL;
7162     struct gen9_hevc_encoder_context *priv_ctx = NULL;
7163     struct generic_enc_codec_state *generic_state = NULL;
7164     struct gen9_hevc_encoder_state *priv_state = NULL;
7165     struct gpe_mi_flush_dw_parameter mi_flush_dw_param;
7166     struct hevc_encode_status_buffer *status_buffer = NULL;
7167
7168     pak_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
7169     priv_ctx = (struct gen9_hevc_encoder_context *)pak_context->private_enc_ctx;
7170     generic_state = (struct generic_enc_codec_state *)pak_context->generic_enc_state;
7171     priv_state = (struct gen9_hevc_encoder_state *)pak_context->private_enc_state;
7172
7173     status_buffer = &priv_state->status_buffer;
7174
7175     memset(&mi_flush_dw_param, 0, sizeof(mi_flush_dw_param));
7176     gen8_gpe_mi_flush_dw(ctx, batch, &mi_flush_dw_param);
7177
7178     gen9_hevc_store_reg_mem(ctx, batch, status_buffer->bo,
7179                             status_buffer->status_bs_byte_count_offset,
7180                             status_buffer->mmio_bs_frame_offset);
7181
7182     gen9_hevc_store_reg_mem(ctx, batch, status_buffer->bo,
7183                             status_buffer->status_image_mask_offset,
7184                             status_buffer->mmio_image_mask_offset);
7185
7186     gen9_hevc_store_reg_mem(ctx, batch, status_buffer->bo,
7187                             status_buffer->status_image_ctrl_offset,
7188                             status_buffer->mmio_image_ctrl_offset);
7189
7190     if (generic_state->brc_enabled) {
7191         gen9_hevc_store_reg_mem(ctx, batch, priv_ctx->res_brc_pak_statistic_buffer.bo,
7192                                 offsetof(GEN9_HEVC_PAK_STATES, HEVC_ENC_BYTECOUNT_FRAME),
7193                                 status_buffer->mmio_bs_frame_offset);
7194         gen9_hevc_store_reg_mem(ctx, batch, priv_ctx->res_brc_pak_statistic_buffer.bo,
7195                                 offsetof(GEN9_HEVC_PAK_STATES, HEVC_ENC_BYTECOUNT_FRAME_NOHEADER),
7196                                 status_buffer->mmio_bs_frame_no_header_offset);
7197         gen9_hevc_store_reg_mem(ctx, batch, priv_ctx->res_brc_pak_statistic_buffer.bo,
7198                                 offsetof(GEN9_HEVC_PAK_STATES, HEVC_ENC_IMAGE_STATUS_CONTROL),
7199                                 status_buffer->mmio_image_ctrl_offset);
7200     }
7201
7202     gen8_gpe_mi_flush_dw(ctx, batch, &mi_flush_dw_param);
7203 }
7204
7205 static VAStatus
7206 gen9_hevc_pak_pipeline(VADriverContextP ctx,
7207                        VAProfile profile,
7208                        struct encode_state *encode_state,
7209                        struct intel_encoder_context *encoder_context)
7210 {
7211     struct i965_driver_data *i965 = i965_driver_data(ctx);
7212     struct intel_batchbuffer *batch = encoder_context->base.batch;
7213     struct encoder_vme_mfc_context *pak_context = encoder_context->vme_context;
7214     struct gen9_hevc_encoder_context *priv_ctx = NULL;
7215     struct generic_enc_codec_state *generic_state = NULL;
7216     struct gen9_hevc_encoder_state *priv_state = NULL;
7217     VAStatus va_status = VA_STATUS_SUCCESS;
7218
7219     if (!pak_context || !pak_context->generic_enc_state || !batch) {
7220         va_status = VA_STATUS_ERROR_INVALID_CONTEXT;
7221         goto EXIT;
7222     }
7223
7224     va_status = gen9_hevc_pak_pipeline_prepare(ctx, encode_state, encoder_context);
7225     if (va_status != VA_STATUS_SUCCESS)
7226         goto EXIT;
7227
7228     if (i965->intel.has_bsd2)
7229         intel_batchbuffer_start_atomic_bcs_override(batch, 0x1000, BSD_RING0);
7230     else
7231         intel_batchbuffer_start_atomic_bcs(batch, 0x1000);
7232
7233     intel_batchbuffer_emit_mi_flush(batch);
7234
7235     generic_state = (struct generic_enc_codec_state *)pak_context->generic_enc_state;
7236     priv_state = (struct gen9_hevc_encoder_state *)pak_context->private_enc_state;
7237     priv_ctx = (struct gen9_hevc_encoder_context *)pak_context->private_enc_ctx;
7238
7239     for (generic_state->curr_pak_pass = 0;
7240          generic_state->curr_pak_pass < generic_state->num_pak_passes;
7241          generic_state->curr_pak_pass++) {
7242         va_status = gen9_hevc_pak_picture_level(ctx, encode_state, encoder_context);
7243         if (va_status != VA_STATUS_SUCCESS)
7244             goto EXIT;
7245
7246         gen9_hevc_pak_read_status(ctx, encoder_context);
7247     }
7248
7249     if (priv_ctx->res_pak_slice_batch_buffer) {
7250         intel_batchbuffer_free(priv_ctx->res_pak_slice_batch_buffer);
7251
7252         priv_ctx->res_pak_slice_batch_buffer = NULL;
7253     }
7254
7255     intel_batchbuffer_end_atomic(batch);
7256
7257     intel_batchbuffer_flush(batch);
7258
7259     priv_state->frame_number++;
7260
7261 EXIT:
7262     return va_status;
7263 }
7264
7265 static void
7266 gen9_hevc_pak_context_destroy(void *context)
7267 {
7268     struct encoder_vme_mfc_context *pak_context = (struct encoder_vme_mfc_context *)context;
7269     struct gen9_hevc_encoder_context *priv_ctx = NULL;
7270     int i = 0;
7271
7272     priv_ctx = (struct gen9_hevc_encoder_context *)pak_context->private_enc_ctx;
7273
7274     if (priv_ctx->res_pak_slice_batch_buffer) {
7275         intel_batchbuffer_free(priv_ctx->res_pak_slice_batch_buffer);
7276         priv_ctx->res_pak_slice_batch_buffer = NULL;
7277     }
7278
7279     dri_bo_unreference(priv_ctx->indirect_pak_bse_object.bo);
7280     priv_ctx->indirect_pak_bse_object.bo = NULL;
7281
7282     if (priv_ctx->uncompressed_picture_source.obj_surface &&
7283         priv_ctx->uncompressed_picture_source.obj_surface->bo)
7284         i965_destroy_surface_storage(priv_ctx->uncompressed_picture_source.obj_surface);
7285
7286     if (priv_ctx->reconstructed_object.obj_surface &&
7287         priv_ctx->reconstructed_object.obj_surface->bo)
7288         i965_destroy_surface_storage(priv_ctx->reconstructed_object.obj_surface);
7289
7290     for (i = 0; i < GEN9_MAX_REF_SURFACES; i++)
7291         if (priv_ctx->reference_surfaces[i].obj_surface &&
7292             priv_ctx->reference_surfaces[i].obj_surface->bo)
7293             i965_destroy_surface_storage(priv_ctx->reference_surfaces[i].obj_surface);
7294 }
7295
7296 #define STATUS_IMPLEMENTATION_START
7297
7298 static void
7299 gen9_hevc_status_buffer_init(struct hevc_encode_status_buffer *status_buffer)
7300 {
7301     uint32_t base_offset = offsetof(struct i965_coded_buffer_segment, codec_private_data);
7302
7303     status_buffer->mmio_bs_frame_offset = MMIO_HCP_ENC_BITSTREAM_BYTECOUNT_FRAME_OFFSET;
7304     status_buffer->mmio_bs_frame_no_header_offset = MMIO_HCP_ENC_BITSTREAM_BYTECOUNT_FRAME_NO_HEADER_OFFSET;
7305     status_buffer->mmio_image_mask_offset = MMIO_HCP_ENC_IMAGE_STATUS_MASK_OFFSET;
7306     status_buffer->mmio_image_ctrl_offset = MMIO_HCP_ENC_IMAGE_STATUS_CTRL_OFFSET;
7307
7308     status_buffer->status_image_mask_offset = base_offset +
7309                                               offsetof(struct hevc_encode_status, image_status_mask);
7310     status_buffer->status_image_ctrl_offset = base_offset +
7311                                               offsetof(struct hevc_encode_status, image_status_ctrl);
7312     status_buffer->status_image_ctrl_last_pass_offset = base_offset +
7313                                                         offsetof(struct hevc_encode_status, image_status_ctrl_last_pass);
7314     status_buffer->status_bs_byte_count_offset = base_offset +
7315                                                  offsetof(struct hevc_encode_status, bs_byte_count);
7316     status_buffer->status_media_state_offset = base_offset +
7317                                                offsetof(struct hevc_encode_status, media_state);
7318     status_buffer->status_pass_num_offset = base_offset +
7319                                             offsetof(struct hevc_encode_status, pass_num);
7320 }
7321
7322 static VAStatus
7323 gen9_hevc_get_coded_status(VADriverContextP ctx,
7324                            struct intel_encoder_context *encoder_context,
7325                            struct i965_coded_buffer_segment *coded_buf_seg)
7326 {
7327     struct hevc_encode_status *encode_status;
7328     struct hevc_enc_image_status_ctrl *image_status_ctrl, *image_status_ctrl_last_pass;
7329
7330     if (!encoder_context || !coded_buf_seg)
7331         return VA_STATUS_ERROR_INVALID_BUFFER;
7332
7333     encode_status = (struct hevc_encode_status *)coded_buf_seg->codec_private_data;
7334
7335     coded_buf_seg->base.size = encode_status->bs_byte_count;
7336
7337     image_status_ctrl = (struct hevc_enc_image_status_ctrl *)&encode_status->image_status_ctrl;
7338     image_status_ctrl_last_pass = (struct hevc_enc_image_status_ctrl *)&encode_status->image_status_ctrl_last_pass;
7339
7340     if (image_status_ctrl->total_pass && image_status_ctrl->cumulative_frame_delta_qp == 0)
7341         image_status_ctrl->cumulative_frame_delta_qp = image_status_ctrl_last_pass->cumulative_frame_delta_qp;
7342
7343     image_status_ctrl_last_pass->cumulative_frame_delta_qp = 0;
7344
7345     return VA_STATUS_SUCCESS;
7346 }
7347
7348 // External initial APIs
7349
7350 Bool
7351 gen9_hevc_vme_context_init(VADriverContextP ctx,
7352                            struct intel_encoder_context *encoder_context)
7353 {
7354     struct i965_driver_data *i965 = i965_driver_data(ctx);
7355     struct encoder_vme_mfc_context *vme_context = NULL;
7356     struct gen9_hevc_encoder_context *priv_ctx = NULL;
7357     struct generic_enc_codec_state *generic_state = NULL;
7358     struct gen9_hevc_encoder_state *priv_state = NULL;
7359
7360     hevc_enc_kernel_ptr = (void *)gen9_hevc_encoder_kernels;
7361     hevc_enc_kernel_size = sizeof(gen9_hevc_encoder_kernels);
7362
7363     vme_context = calloc(1, sizeof(*vme_context));
7364     priv_ctx = calloc(1, sizeof(*priv_ctx));
7365     generic_state = calloc(1, sizeof(*generic_state));
7366     priv_state = calloc(1, sizeof(*priv_state));
7367
7368     if (!vme_context || !generic_state ||
7369         !priv_ctx || !priv_state) {
7370         if (vme_context) free(vme_context);
7371         if (generic_state) free(generic_state);
7372         if (priv_ctx) free(priv_ctx);
7373         if (priv_state) free(priv_state);
7374
7375         return false;
7376     }
7377
7378     encoder_context->vme_context = (void *)vme_context;
7379     vme_context->private_enc_ctx = (void *)priv_ctx;
7380     vme_context->generic_enc_state = (void *)generic_state;
7381     vme_context->private_enc_state = (void *)priv_state;
7382
7383     priv_ctx->ctx = ctx;
7384     priv_ctx->mocs = i965->intel.mocs_state;
7385
7386     generic_state->num_pak_passes = 1;
7387     generic_state->brc_enabled = 0;
7388
7389     priv_state->tu_mode = HEVC_TU_RT_SPEED;
7390     priv_state->use_hw_scoreboard = 1;
7391     priv_state->use_hw_non_stalling_scoreborad = 1;
7392     priv_state->rolling_intra_refresh = 0;
7393     priv_state->flatness_check_supported = 0;
7394     priv_state->walking_pattern_26 = 0;
7395     priv_state->num_regions_in_slice = 4;
7396     priv_state->frames_per_100s = 30000;
7397     priv_state->user_max_frame_size = 0;
7398     priv_state->brc_method = HEVC_BRC_CQP;
7399     priv_state->lcu_brc_enabled = 0;
7400     priv_state->parallel_brc = 0;
7401     priv_state->pak_obj_size = ((IS_KBL(i965->intel.device_info) || IS_GLK(i965->intel.device_info)) ?
7402                                 GEN95_HEVC_ENC_PAK_OBJ_SIZE :
7403                                 GEN9_HEVC_ENC_PAK_OBJ_SIZE) *
7404                                4;
7405     priv_state->cu_record_size = GEN9_HEVC_ENC_PAK_CU_RECORD_SIZE * 4;
7406     priv_state->pic_state_size = GEN9_HEVC_ENC_BRC_PIC_STATE_SIZE;
7407
7408     gen9_hevc_status_buffer_init(&priv_state->status_buffer);
7409     gen9_hevc_vme_kernels_context_init(ctx, encoder_context);
7410     gen9_hevc_lambda_tables_init(priv_ctx);
7411
7412     encoder_context->vme_pipeline = gen9_hevc_vme_pipeline;
7413     encoder_context->vme_context_destroy = gen9_hevc_vme_context_destroy;
7414     return true;
7415 }
7416
7417 Bool
7418 gen9_hevc_pak_context_init(VADriverContextP ctx,
7419                            struct intel_encoder_context *encoder_context)
7420 {
7421     struct encoder_vme_mfc_context *pak_context = encoder_context->vme_context;
7422
7423     if (!pak_context)
7424         return false;
7425
7426     encoder_context->mfc_context = pak_context;
7427     encoder_context->mfc_context_destroy = gen9_hevc_pak_context_destroy;
7428     encoder_context->mfc_pipeline = gen9_hevc_pak_pipeline;
7429     encoder_context->mfc_brc_prepare = gen9_hevc_brc_prepare;
7430     encoder_context->get_status = gen9_hevc_get_coded_status;
7431     return true;
7432 }