src/gen9_hevc_encoder.c

   1 /*
   2  * Copyright © 2017 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the
   6  * "Software"), to deal in the Software without restriction, including
   7  * without limitation the rights to use, copy, modify, merge, publish,
   8  * distribute, sub license, and/or sell copies of the Software, and to
   9  * permit persons to whom the Software is furnished to do so, subject to
  10  * the following conditions:
  11  *
  12  * The above copyright notice and this permission notice (including the
  13  * next paragraph) shall be included in all copies or substantial portions
  14  * of the Software.
  15  *
  16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
  17  * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  18  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
  19  * IN NO EVENT SHALL PRECISION INSIGHT AND/OR ITS SUPPLIERS BE LIABLE FOR
  20  * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
  21  * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
  22  * SOFTWAR OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  23  *
  24  * Authors:
  25  *    Chen, Peng <chen.c.peng@intel.com>
  26  *
  27  */
  28
  29 #include <stdio.h>
  30 #include <stdlib.h>
  31 #include <stdbool.h>
  32 #include <string.h>
  33 #include <math.h>
  34 #include <assert.h>
  35 #include <va/va.h>
  36
  37 #include "intel_batchbuffer.h"
  38 #include "intel_driver.h"
  39
  40 #include "i965_defines.h"
  41 #include "i965_drv_video.h"
  42 #include "i965_encoder.h"
  43 #include "i965_encoder_common.h"
  44 #include "i965_encoder_utils.h"
  45 #include "i965_encoder_api.h"
  46 #include "gen9_hevc_enc_kernels.h"
  47 #include "gen9_hevc_enc_kernels_binary.h"
  48 #include "gen9_hevc_enc_utils.h"
  49 #include "gen9_hevc_encoder.h"
  50 #include "gen9_hevc_enc_const_def.h"
  51
  52 static void *hevc_enc_kernel_ptr = NULL;
  53 static int hevc_enc_kernel_size = 0;
  54
  55 static bool
  56 gen9_hevc_get_kernel_header_and_size(void *pvbinary,
  57                                      int binary_size,
  58                                      GEN9_ENC_OPERATION operation,
  59                                      int krnstate_idx,
  60                                      struct i965_kernel *ret_kernel)
  61 {
  62     typedef uint32_t BIN_PTR[4];
  63
  64     char *bin_start;
  65     gen9_hevc_enc_kernels_header_bxt *pkh_table = NULL;
  66     gen9_hevc_enc_kernel_header *pcurr_header = NULL, *pinvalid_entry, *pnext_header;
  67     int next_krnoffset;
  68
  69     if (!pvbinary || !ret_kernel)
  70         return false;
  71
  72     bin_start = (char *)pvbinary;
  73     pkh_table = (gen9_hevc_enc_kernels_header_bxt *)pvbinary;
  74     pinvalid_entry = (gen9_hevc_enc_kernel_header *)(pkh_table + 1);
  75     next_krnoffset = binary_size;
  76
  77     switch (operation) {
  78     case GEN9_ENC_SCALING2X:
  79         pcurr_header = &pkh_table->HEVC_ENC_I_2xDownSampling_Kernel;
  80         break;
  81     case GEN9_ENC_SCALING4X:
  82         pcurr_header = &pkh_table->HEVC_ENC_I_DS4HME;
  83         break;
  84     case GEN9_ENC_ME:
  85         if (krnstate_idx)
  86             pcurr_header = &pkh_table->HEVC_ENC_P_HME;
  87         else
  88             pcurr_header = &pkh_table->HEVC_ENC_B_HME;
  89         break;
  90     case GEN9_ENC_BRC:
  91         switch (krnstate_idx) {
  92         case GEN9_HEVC_ENC_BRC_COARSE_INTRA:
  93             pcurr_header = &pkh_table->HEVC_ENC_I_COARSE;
  94             break;
  95         case GEN9_HEVC_ENC_BRC_INIT:
  96             pcurr_header = &pkh_table->HEVC_ENC_BRC_Init;
  97             break;
  98         case GEN9_HEVC_ENC_BRC_RESET:
  99             pcurr_header = &pkh_table->HEVC_ENC_BRC_Reset;
 100             break;
 101         case GEN9_HEVC_ENC_BRC_FRAME_UPDATE:
 102             pcurr_header = &pkh_table->HEVC_ENC_BRC_Update;
 103             break;
 104         case GEN9_HEVC_ENC_BRC_LCU_UPDATE:
 105             pcurr_header = &pkh_table->HEVC_ENC_BRC_LCU_Update;
 106             break;
 107         default:
 108             break;
 109         }
 110         break;
 111     case GEN9_ENC_MBENC:
 112         switch (krnstate_idx) {
 113         case GEN9_HEVC_ENC_MBENC_2xSCALING:
 114         case GEN9_HEVC_ENC_MBENC_32x32MD:
 115         case GEN9_HEVC_ENC_MBENC_16x16SAD:
 116         case GEN9_HEVC_ENC_MBENC_16x16MD:
 117         case GEN9_HEVC_ENC_MBENC_8x8PU:
 118         case GEN9_HEVC_ENC_MBENC_8x8FMODE:
 119         case GEN9_HEVC_ENC_MBENC_32x32INTRACHECK:
 120         case GEN9_HEVC_ENC_MBENC_BENC:
 121             pcurr_header = &pkh_table->HEVC_ENC_I_2xDownSampling_Kernel;
 122             pcurr_header += krnstate_idx;
 123             break;
 124         case GEN9_HEVC_ENC_MBENC_BPAK:
 125             pcurr_header = &pkh_table->HEVC_ENC_PB_Pak;
 126             break;
 127         case GEN9_HEVC_ENC_MBENC_WIDI:
 128             pcurr_header = &pkh_table->HEVC_ENC_PB_Widi;
 129             break;
 130         case GEN9_HEVC_MBENC_PENC:
 131             pcurr_header = &pkh_table->HEVC_ENC_P_MB;
 132             break;
 133         case GEN9_HEVC_MBENC_P_WIDI:
 134             pcurr_header = &pkh_table->HEVC_ENC_P_Widi;
 135             break;
 136         case GEN9_HEVC_ENC_MBENC_DS_COMBINED:
 137             pcurr_header = &pkh_table->HEVC_ENC_DS_Combined;
 138             break;
 139         default:
 140             break;
 141         }
 142         break;
 143     default:
 144         break;
 145     }
 146
 147     if (!pcurr_header ||
 148         ((pcurr_header->kernel_start_pointer << 6) >= binary_size))
 149         return false;
 150
 151     ret_kernel->bin = (const BIN_PTR *)(bin_start + (pcurr_header->kernel_start_pointer << 6));
 152
 153     pnext_header = (pcurr_header + 1);
 154     if (pnext_header < pinvalid_entry)
 155         next_krnoffset = pnext_header->kernel_start_pointer << 6;
 156
 157     ret_kernel->size = next_krnoffset - (pcurr_header->kernel_start_pointer << 6);
 158
 159     return true;
 160 }
 161
 162 static void
 163 gen9_hevc_store_reg_mem(VADriverContextP ctx,
 164                         struct intel_batchbuffer *batch,
 165                         dri_bo *bo,
 166                         unsigned int offset,
 167                         unsigned int mmio_offset)
 168 {
 169     struct gpe_mi_store_register_mem_parameter mi_store_reg_mem_param;
 170
 171     memset((void *)&mi_store_reg_mem_param, 0,
 172            sizeof(mi_store_reg_mem_param));
 173     mi_store_reg_mem_param.bo = bo;
 174     mi_store_reg_mem_param.offset = offset;
 175     mi_store_reg_mem_param.mmio_offset = mmio_offset;
 176     gen8_gpe_mi_store_register_mem(ctx, batch, &mi_store_reg_mem_param);
 177 }
 178
 179 static void
 180 gen9_hevc_load_reg_mem(VADriverContextP ctx,
 181                        struct intel_batchbuffer *batch,
 182                        dri_bo *bo,
 183                        unsigned int offset,
 184                        unsigned int mmio_offset)
 185 {
 186     struct gpe_mi_load_register_mem_parameter mi_load_reg_mem_param;
 187
 188     memset((void *)&mi_load_reg_mem_param, 0,
 189            sizeof(mi_load_reg_mem_param));
 190     mi_load_reg_mem_param.bo = bo;
 191     mi_load_reg_mem_param.offset = offset;
 192     mi_load_reg_mem_param.mmio_offset = mmio_offset;
 193     gen8_gpe_mi_load_register_mem(ctx, batch, &mi_load_reg_mem_param);
 194 }
 195
 196 static void
 197 gen9_hevc_conditional_end(VADriverContextP ctx,
 198                           struct intel_batchbuffer *batch,
 199                           dri_bo *bo,
 200                           unsigned int offset,
 201                           unsigned int data)
 202 {
 203     struct gpe_mi_conditional_batch_buffer_end_parameter mi_cond_end;
 204
 205     memset(&mi_cond_end, 0, sizeof(mi_cond_end));
 206     mi_cond_end.offset = offset;
 207     mi_cond_end.bo = bo;
 208     mi_cond_end.compare_data = data;
 209     mi_cond_end.compare_mask_mode_disabled = 0;
 210     gen9_gpe_mi_conditional_batch_buffer_end(ctx, batch,
 211                                              &mi_cond_end);
 212 }
 213
 214 static VAStatus
 215 gen9_hevc_ensure_surface(VADriverContextP ctx,
 216                          struct gen9_hevc_encoder_state *priv_state,
 217                          struct object_surface *obj_surface,
 218                          int reallocate_flag)
 219 {
 220     VAStatus va_status = VA_STATUS_SUCCESS;
 221     int update = 0;
 222     unsigned int fourcc = VA_FOURCC_NV12;
 223
 224     if (!obj_surface) {
 225         va_status = VA_STATUS_ERROR_INVALID_PARAMETER;
 226
 227         goto EXIT;
 228     }
 229
 230     if ((priv_state->bit_depth_luma_minus8 > 0)
 231         || (priv_state->bit_depth_chroma_minus8 > 0)) {
 232         if (obj_surface->fourcc != VA_FOURCC_P010) {
 233             update = 1;
 234             fourcc = VA_FOURCC_P010;
 235         }
 236     } else if (obj_surface->fourcc != VA_FOURCC_NV12) {
 237         update = 1;
 238         fourcc = VA_FOURCC_NV12;
 239     }
 240
 241     /* (Re-)allocate the underlying surface buffer store, if necessary */
 242     if (!obj_surface->bo || update) {
 243         if (reallocate_flag) {
 244             struct i965_driver_data * const i965 = i965_driver_data(ctx);
 245
 246             i965_destroy_surface_storage(obj_surface);
 247
 248             va_status = i965_check_alloc_surface_bo(ctx,
 249                                                     obj_surface,
 250                                                     i965->codec_info->has_tiled_surface,
 251                                                     fourcc,
 252                                                     SUBSAMPLE_YUV420);
 253         } else
 254             va_status = VA_STATUS_ERROR_INVALID_PARAMETER;
 255     }
 256
 257 EXIT:
 258     return va_status;
 259 }
 260
 261 static void
 262 gen9_hevc_free_surface_private(void **data)
 263 {
 264     struct gen9_hevc_surface_priv *surface_priv = NULL;
 265     int i = 0;
 266
 267     surface_priv = (struct gen9_hevc_surface_priv *)(*data);
 268     if (!surface_priv)
 269         return;
 270
 271     for (i = 0; i < HEVC_SCALED_SURFS_NUM; i++) {
 272         if (surface_priv->scaled_surface_obj[i]) {
 273             i965_DestroySurfaces(surface_priv->ctx, &surface_priv->scaled_surface_id[i], 1);
 274             surface_priv->scaled_surface_id[i] = VA_INVALID_SURFACE;
 275             surface_priv->scaled_surface_obj[i] = NULL;
 276         }
 277     }
 278
 279     if (surface_priv->surface_obj_nv12) {
 280         i965_DestroySurfaces(surface_priv->ctx, &surface_priv->surface_id_nv12, 1);
 281         surface_priv->surface_id_nv12 = VA_INVALID_SURFACE;
 282         surface_priv->surface_obj_nv12 = NULL;
 283     }
 284
 285     if (surface_priv->motion_vector_temporal_bo)
 286         dri_bo_unreference(surface_priv->motion_vector_temporal_bo);
 287
 288     free(surface_priv);
 289 }
 290
 291 static VAStatus
 292 gen9_hevc_init_surface_private(VADriverContextP ctx,
 293                                struct generic_enc_codec_state *generic_state,
 294                                struct gen9_hevc_encoder_state *priv_state,
 295                                struct object_surface *obj_surface)
 296 {
 297     struct i965_driver_data *i965 = i965_driver_data(ctx);
 298     struct gen9_hevc_surface_priv *surface_priv = NULL;
 299     int size = 0;
 300
 301     if (!obj_surface || !obj_surface->bo)
 302         return VA_STATUS_ERROR_INVALID_SURFACE;
 303
 304     if (obj_surface->private_data &&
 305         obj_surface->free_private_data != gen9_hevc_free_surface_private) {
 306         obj_surface->free_private_data(&obj_surface->private_data);
 307         obj_surface->private_data = NULL;
 308     }
 309
 310     if (obj_surface->private_data) {
 311         surface_priv = (struct gen9_hevc_surface_priv *)obj_surface->private_data;
 312
 313         surface_priv->surface_nv12_valid = 0;
 314     } else {
 315         surface_priv = calloc(1, sizeof(*surface_priv));
 316         if (!surface_priv)
 317             return VA_STATUS_ERROR_ALLOCATION_FAILED;
 318
 319         surface_priv->ctx = ctx;
 320         surface_priv->surface_reff = obj_surface;
 321
 322         obj_surface->private_data = (void *)surface_priv;
 323         obj_surface->free_private_data = gen9_hevc_free_surface_private;
 324
 325         // alloc motion vector temporal buffer
 326         size = MAX(((priv_state->picture_width + 63) >> 6) *
 327                    ((priv_state->picture_height + 15) >> 4),
 328                    ((priv_state->picture_width + 31) >> 5) *
 329                    ((priv_state->picture_height + 31) >> 5));
 330         size = ALIGN(size, 2) * 64;
 331         surface_priv->motion_vector_temporal_bo =
 332             dri_bo_alloc(i965->intel.bufmgr,
 333                          "motion vector temporal buffer",
 334                          size,
 335                          0x1000);
 336         if (!surface_priv->motion_vector_temporal_bo)
 337             return VA_STATUS_ERROR_ALLOCATION_FAILED;
 338
 339         //alloc HME surfaces
 340         i965_CreateSurfaces(ctx,
 341                             priv_state->frame_width_4x,
 342                             priv_state->frame_height_4x,
 343                             VA_RT_FORMAT_YUV420,
 344                             1,
 345                             &surface_priv->scaled_surface_id[HEVC_SCALED_SURF_4X_ID]);
 346
 347         surface_priv->scaled_surface_obj[HEVC_SCALED_SURF_4X_ID] =
 348             SURFACE(surface_priv->scaled_surface_id[HEVC_SCALED_SURF_4X_ID]);
 349
 350         if (!surface_priv->scaled_surface_obj[HEVC_SCALED_SURF_4X_ID])
 351             return VA_STATUS_ERROR_ALLOCATION_FAILED;
 352
 353         i965_check_alloc_surface_bo(ctx, surface_priv->scaled_surface_obj[HEVC_SCALED_SURF_4X_ID], 1,
 354                                     VA_FOURCC('N', 'V', '1', '2'), SUBSAMPLE_YUV420);
 355
 356         if (generic_state->b16xme_supported) {
 357             i965_CreateSurfaces(ctx,
 358                                 priv_state->frame_width_16x,
 359                                 priv_state->frame_height_16x,
 360                                 VA_RT_FORMAT_YUV420,
 361                                 1,
 362                                 &surface_priv->scaled_surface_id[HEVC_SCALED_SURF_16X_ID]);
 363             surface_priv->scaled_surface_obj[HEVC_SCALED_SURF_16X_ID] =
 364                 SURFACE(surface_priv->scaled_surface_id[HEVC_SCALED_SURF_16X_ID]);
 365
 366             if (!surface_priv->scaled_surface_obj[HEVC_SCALED_SURF_16X_ID])
 367                 return VA_STATUS_ERROR_ALLOCATION_FAILED;
 368
 369             i965_check_alloc_surface_bo(ctx, surface_priv->scaled_surface_obj[HEVC_SCALED_SURF_16X_ID], 1,
 370                                         VA_FOURCC('N', 'V', '1', '2'), SUBSAMPLE_YUV420);
 371         }
 372
 373         if (generic_state->b32xme_supported) {
 374             i965_CreateSurfaces(ctx,
 375                                 priv_state->frame_width_32x,
 376                                 priv_state->frame_height_32x,
 377                                 VA_RT_FORMAT_YUV420,
 378                                 1,
 379                                 &surface_priv->scaled_surface_id[HEVC_SCALED_SURF_32X_ID]);
 380             surface_priv->scaled_surface_obj[HEVC_SCALED_SURF_32X_ID] =
 381                 SURFACE(surface_priv->scaled_surface_id[HEVC_SCALED_SURF_32X_ID]);
 382
 383             if (!surface_priv->scaled_surface_obj[HEVC_SCALED_SURF_32X_ID])
 384                 return VA_STATUS_ERROR_ALLOCATION_FAILED;
 385
 386             i965_check_alloc_surface_bo(ctx, surface_priv->scaled_surface_obj[HEVC_SCALED_SURF_32X_ID], 1,
 387                                         VA_FOURCC('N', 'V', '1', '2'), SUBSAMPLE_YUV420);
 388         }
 389
 390         if (obj_surface->fourcc == VA_FOURCC_P010) {
 391             i965_CreateSurfaces(ctx,
 392                                 priv_state->frame_width_in_max_lcu,
 393                                 priv_state->frame_height_in_max_lcu,
 394                                 VA_RT_FORMAT_YUV420,
 395                                 1,
 396                                 &surface_priv->surface_id_nv12);
 397             surface_priv->surface_obj_nv12 = SURFACE(surface_priv->surface_id_nv12);
 398
 399             if (!surface_priv->surface_obj_nv12)
 400                 return VA_STATUS_ERROR_ALLOCATION_FAILED;
 401
 402             i965_check_alloc_surface_bo(ctx, surface_priv->surface_obj_nv12, 1,
 403                                         VA_FOURCC('N', 'V', '1', '2'), SUBSAMPLE_YUV420);
 404         }
 405     }
 406
 407     return VA_STATUS_SUCCESS;
 408 }
 409
 410 static void
 411 gen9_hevc_enc_free_resources(struct encoder_vme_mfc_context *vme_context)
 412 {
 413     struct gen9_hevc_encoder_context *priv_ctx = NULL;
 414
 415     priv_ctx = (struct gen9_hevc_encoder_context *)vme_context->private_enc_ctx;
 416
 417     i965_free_gpe_resource(&priv_ctx->res_brc_pic_states_write_buffer);
 418     i965_free_gpe_resource(&priv_ctx->res_brc_history_buffer);
 419     i965_free_gpe_resource(&priv_ctx->res_brc_intra_dist_buffer);
 420     i965_free_gpe_resource(&priv_ctx->res_brc_pak_statistic_buffer);
 421     i965_free_gpe_resource(&priv_ctx->res_brc_input_buffer_for_enc_kernels);
 422     i965_free_gpe_resource(&priv_ctx->res_brc_constant_data_buffer);
 423
 424     i965_free_gpe_resource(&priv_ctx->res_mb_code_surface);
 425
 426     // free VME buffers
 427     i965_free_gpe_resource(&priv_ctx->res_flatness_check_surface);
 428     i965_free_gpe_resource(&priv_ctx->res_brc_me_dist_buffer);
 429     i965_free_gpe_resource(&priv_ctx->s4x_memv_distortion_buffer);
 430     i965_free_gpe_resource(&priv_ctx->s4x_memv_data_buffer);
 431     i965_free_gpe_resource(&priv_ctx->s16x_memv_data_buffer);
 432     i965_free_gpe_resource(&priv_ctx->s32x_memv_data_buffer);
 433     i965_free_gpe_resource(&priv_ctx->res_32x32_pu_output_buffer);
 434     i965_free_gpe_resource(&priv_ctx->res_simplest_intra_buffer);
 435     i965_free_gpe_resource(&priv_ctx->res_kernel_debug);
 436     i965_free_gpe_resource(&priv_ctx->res_sad_16x16_pu_buffer);
 437     i965_free_gpe_resource(&priv_ctx->res_vme_8x8_mode_buffer);
 438     i965_free_gpe_resource(&priv_ctx->res_intra_mode_buffer);
 439     i965_free_gpe_resource(&priv_ctx->res_intra_distortion_buffer);
 440     i965_free_gpe_resource(&priv_ctx->res_vme_uni_sic_buffer);
 441     i965_free_gpe_resource(&priv_ctx->res_con_corrent_thread_buffer);
 442     i965_free_gpe_resource(&priv_ctx->res_mv_index_buffer);
 443     i965_free_gpe_resource(&priv_ctx->res_mvp_index_buffer);
 444     i965_free_gpe_resource(&priv_ctx->res_roi_buffer);
 445     i965_free_gpe_resource(&priv_ctx->res_mb_statistics_buffer);
 446
 447     if (priv_ctx->scaled_2x_surface_obj) {
 448         i965_DestroySurfaces(priv_ctx->ctx, &priv_ctx->scaled_2x_surface_id, 1);
 449         priv_ctx->scaled_2x_surface_obj = NULL;
 450     }
 451
 452     // free PAK buffers
 453     i965_free_gpe_resource(&priv_ctx->deblocking_filter_line_buffer);
 454     i965_free_gpe_resource(&priv_ctx->deblocking_filter_tile_line_buffer);
 455     i965_free_gpe_resource(&priv_ctx->deblocking_filter_tile_column_buffer);
 456     i965_free_gpe_resource(&priv_ctx->metadata_line_buffer);
 457     i965_free_gpe_resource(&priv_ctx->metadata_tile_line_buffer);
 458     i965_free_gpe_resource(&priv_ctx->metadata_tile_column_buffer);
 459     i965_free_gpe_resource(&priv_ctx->sao_line_buffer);
 460     i965_free_gpe_resource(&priv_ctx->sao_tile_line_buffer);
 461     i965_free_gpe_resource(&priv_ctx->sao_tile_column_buffer);
 462
 463     priv_ctx->res_inited = 0;
 464 }
 465
 466 #define ALLOC_GPE_RESOURCE(RES, NAME, SIZE)                 \
 467     do{                                                     \
 468         i965_free_gpe_resource(&priv_ctx->RES);             \
 469         if (!i965_allocate_gpe_resource(i965->intel.bufmgr, \
 470                                  &priv_ctx->RES,            \
 471                                  SIZE,                      \
 472                                  NAME))                     \
 473             goto FAIL;                                      \
 474     } while(0);
 475
 476 #define ALLOC_GPE_2D_RESOURCE(RES, NAME, W, H, P)               \
 477     do{                                                         \
 478         i965_free_gpe_resource(&priv_ctx->RES);                 \
 479         if (!i965_gpe_allocate_2d_resource(i965->intel.bufmgr,  \
 480                                  &priv_ctx->RES,                \
 481                                  (ALIGN(W, 64)), H,             \
 482                                  (ALIGN(P, 64)),                \
 483                                  NAME))                         \
 484             goto FAIL;                                          \
 485     } while(0);
 486
 487 static VAStatus
 488 gen9_hevc_enc_alloc_resources(VADriverContextP ctx,
 489                               struct encode_state *encode_state,
 490                               struct intel_encoder_context *encoder_context)
 491 {
 492     struct i965_driver_data *i965 = i965_driver_data(ctx);
 493     struct encoder_vme_mfc_context *vme_context = NULL;
 494     struct gen9_hevc_encoder_context *priv_ctx = NULL;
 495     struct generic_enc_codec_state *generic_state = NULL;
 496     struct gen9_hevc_encoder_state *priv_state = NULL;
 497     int res_size = 0, size_shift = 0;
 498     int width = 0, height = 0;
 499
 500     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
 501     priv_ctx = (struct gen9_hevc_encoder_context *)vme_context->private_enc_ctx;
 502     generic_state = (struct generic_enc_codec_state *)vme_context->generic_enc_state;
 503     priv_state = (struct gen9_hevc_encoder_state *)vme_context->private_enc_state;
 504
 505     if (priv_ctx->res_inited)
 506         return VA_STATUS_SUCCESS;
 507
 508     res_size = priv_state->mb_code_size;
 509     ALLOC_GPE_RESOURCE(res_mb_code_surface,
 510                        "Mb code surface", res_size);
 511
 512     res_size = priv_state->pic_state_size * generic_state->num_pak_passes;
 513     ALLOC_GPE_RESOURCE(res_brc_pic_states_write_buffer,
 514                        "Brc pic status write buffer",
 515                        res_size);
 516
 517     res_size = priv_state->pic_state_size * generic_state->num_pak_passes;
 518     ALLOC_GPE_RESOURCE(res_brc_pic_states_read_buffer,
 519                        "Brc pic status read buffer",
 520                        res_size);
 521
 522     res_size = GEN9_HEVC_ENC_BRC_HISTORY_BUFFER_SIZE;
 523     ALLOC_GPE_RESOURCE(res_brc_history_buffer,
 524                        "Brc history buffer",
 525                        res_size);
 526
 527     res_size = GEN9_HEVC_ENC_BRC_PAK_STATISTCS_SIZE;
 528     ALLOC_GPE_RESOURCE(res_brc_pak_statistic_buffer,
 529                        "Brc pak statistic buffer",
 530                        res_size);
 531
 532     res_size = 1024;
 533     ALLOC_GPE_RESOURCE(res_brc_input_buffer_for_enc_kernels,
 534                        "Brc input buffer for enc kernels buffer",
 535                        res_size);
 536
 537     width = ALIGN(priv_state->downscaled_width_4x_in_mb * 8, 64);
 538     height = ALIGN(priv_state->downscaled_height_4x_in_mb * 4, 8) * 2;
 539     ALLOC_GPE_2D_RESOURCE(res_brc_intra_dist_buffer,
 540                           "Brc intra distortion buffer",
 541                           width, height, width);
 542
 543     width = ALIGN(GEN9_HEVC_ENC_BRC_CONSTANT_SURFACE_WIDTH, 64);
 544     height = GEN9_HEVC_ENC_BRC_CONSTANT_SURFACE_HEIGHT;
 545     ALLOC_GPE_2D_RESOURCE(res_brc_constant_data_buffer,
 546                           "Brc constant data buffer",
 547                           width, height, width);
 548
 549     width = ALIGN((priv_state->frame_width_4x * 4 + 31) >> 4, 64);
 550     height = ALIGN((priv_state->frame_height_4x * 4 + 31) >> 5, 4);
 551     ALLOC_GPE_2D_RESOURCE(res_brc_mb_qp_buffer,
 552                           "Brc mb qp buffer",
 553                           width, height, width);
 554
 555     //HME scaling buffer allocation
 556     width = ALIGN(priv_state->downscaled_width_4x_in_mb * 8, 64);
 557     height = ALIGN(priv_state->downscaled_height_4x_in_mb * 4, 8);
 558     ALLOC_GPE_2D_RESOURCE(res_brc_me_dist_buffer,
 559                           "Brc me distortion buffer",
 560                           width, height, width);
 561
 562     if (generic_state->hme_supported) {
 563         width = ALIGN(priv_state->downscaled_width_4x_in_mb * 8, 64);
 564         height = ALIGN(priv_state->downscaled_height_4x_in_mb * 4 * 10, 8) * 2;
 565         ALLOC_GPE_2D_RESOURCE(s4x_memv_distortion_buffer,
 566                               "4x MEMV distortion buffer",
 567                               width, height, width);
 568
 569         width = ALIGN(priv_state->downscaled_width_4x_in_mb * 32, 64);
 570         height = priv_state->downscaled_height_4x_in_mb * 4 * 10;
 571         ALLOC_GPE_2D_RESOURCE(s4x_memv_data_buffer,
 572                               "4x MEMV data buffer",
 573                               width, height, width);
 574
 575         if (generic_state->b16xme_supported) {
 576             width = ALIGN(priv_state->downscaled_width_16x_in_mb * 32, 64);
 577             height = priv_state->downscaled_height_16x_in_mb * 2 * 4 * 10;
 578             ALLOC_GPE_2D_RESOURCE(s16x_memv_data_buffer,
 579                                   "16x MEMV data buffer",
 580                                   width, height, width);
 581
 582             if (generic_state->b32xme_supported) {
 583                 width = ALIGN(priv_state->downscaled_width_32x_in_mb * 32, 64);
 584                 height = priv_state->downscaled_height_32x_in_mb * 2 * 4 * 10;
 585                 ALLOC_GPE_2D_RESOURCE(s32x_memv_data_buffer,
 586                                       "32x MEMV data buffer",
 587                                       width, height, width);
 588             }
 589         }
 590     }
 591
 592     if (priv_state->flatness_check_supported) {
 593         width = ALIGN(priv_state->width_in_mb * 4, 64);
 594         height = priv_state->downscaled_height_4x_in_mb * 4;
 595         ALLOC_GPE_2D_RESOURCE(res_flatness_check_surface,
 596                               "Flatness check buffer",
 597                               width, height, width);
 598     }
 599
 600     if (priv_ctx->scaled_2x_surface_obj)
 601         i965_DestroySurfaces(priv_ctx->ctx, &priv_ctx->scaled_2x_surface_id, 1);
 602
 603     width = priv_state->frame_width_in_max_lcu >> 1;
 604     height = priv_state->frame_height_in_max_lcu >> 1;
 605     if (priv_state->bit_depth_luma_minus8) {
 606         width = ALIGN(width, 32);
 607         height = ALIGN(height, 32);
 608     }
 609
 610     i965_CreateSurfaces(ctx,
 611                         width,
 612                         height,
 613                         VA_RT_FORMAT_YUV420,
 614                         1,
 615                         &priv_ctx->scaled_2x_surface_id);
 616     priv_ctx->scaled_2x_surface_obj =
 617         SURFACE(priv_ctx->scaled_2x_surface_id);
 618
 619     if (!priv_ctx->scaled_2x_surface_obj)
 620         goto FAIL;
 621
 622     i965_check_alloc_surface_bo(ctx, priv_ctx->scaled_2x_surface_obj, 1,
 623                                 VA_FOURCC('N', 'V', '1', '2'), SUBSAMPLE_YUV420);
 624
 625     res_size = (priv_state->frame_width_in_max_lcu >> 5) *
 626                (priv_state->frame_height_in_max_lcu >> 5) *
 627                32;
 628     ALLOC_GPE_RESOURCE(res_32x32_pu_output_buffer,
 629                        "32x32 pu output buffer",
 630                        res_size);
 631
 632     width = priv_state->frame_width_in_max_lcu >> 3;
 633     height = priv_state->frame_height_in_max_lcu >> 5;
 634     ALLOC_GPE_2D_RESOURCE(res_slice_map_buffer,
 635                           "Slice map buffer",
 636                           width, height, width);
 637
 638     res_size = 8192 * 1024;
 639     ALLOC_GPE_RESOURCE(res_kernel_debug,
 640                        "kernel debug",
 641                        res_size);
 642
 643     width = priv_state->frame_width_in_max_lcu >> 3;
 644     height = priv_state->frame_height_in_max_lcu >> 5;
 645     ALLOC_GPE_2D_RESOURCE(res_simplest_intra_buffer,
 646                           "Simplest intra buffer",
 647                           width, height, width);
 648
 649     res_size = (priv_state->frame_width_in_max_lcu >> 4) *
 650                (priv_state->frame_height_in_max_lcu >> 4) * 8 * 4;
 651     ALLOC_GPE_RESOURCE(res_sad_16x16_pu_buffer,
 652                        "Sad 16x16 pu",
 653                        res_size);
 654
 655     res_size = (priv_state->frame_width_in_max_lcu >> 4) *
 656                (priv_state->frame_height_in_max_lcu >> 4) * 32;
 657     ALLOC_GPE_RESOURCE(res_vme_8x8_mode_buffer,
 658                        "Vme 8x8 mode",
 659                        res_size);
 660
 661     res_size = (priv_state->frame_width_in_max_lcu >> 3) *
 662                (priv_state->frame_height_in_max_lcu >> 3) * 32;
 663     ALLOC_GPE_RESOURCE(res_intra_mode_buffer,
 664                        "Intra mode",
 665                        res_size);
 666
 667     res_size = (priv_state->frame_width_in_max_lcu >> 4) *
 668                (priv_state->frame_height_in_max_lcu >> 4) * 16;
 669     ALLOC_GPE_RESOURCE(res_intra_distortion_buffer,
 670                        "Intra distortion",
 671                        res_size);
 672
 673     width = priv_state->frame_width_in_max_lcu >> 1;
 674     height = priv_state->frame_height_in_max_lcu >> 4;
 675     ALLOC_GPE_2D_RESOURCE(res_min_distortion_buffer,
 676                           "Min distortion buffer",
 677                           width, height, width);
 678
 679     res_size = priv_state->frame_width_in_max_lcu *
 680                priv_state->frame_height_in_max_lcu;
 681     ALLOC_GPE_RESOURCE(res_vme_uni_sic_buffer,
 682                        "Vme uni sic buffer",
 683                        res_size);
 684
 685     width = sizeof(gen9_hevc_mbenc_control_region);
 686     height = GEN9_HEVC_ENC_CONCURRENT_SURFACE_HEIGHT;
 687     ALLOC_GPE_2D_RESOURCE(res_con_corrent_thread_buffer,
 688                           "Con corrent thread buffer",
 689                           width, height, width);
 690
 691     res_size = priv_state->frame_width_in_max_lcu *
 692                priv_state->frame_height_in_max_lcu / 4;
 693     ALLOC_GPE_RESOURCE(res_mv_index_buffer,
 694                        "Mv index buffer",
 695                        res_size);
 696
 697     res_size = priv_state->frame_width_in_max_lcu *
 698                priv_state->frame_height_in_max_lcu / 2;
 699     ALLOC_GPE_RESOURCE(res_mvp_index_buffer,
 700                        "Mvp index buffer",
 701                        res_size);
 702
 703     width = ALIGN(priv_state->width_in_mb * 4, 64);
 704     height = ALIGN(priv_state->height_in_mb, 8);
 705     ALLOC_GPE_2D_RESOURCE(res_roi_buffer,
 706                           "ROI buffer",
 707                           width, height, width);
 708
 709     res_size = priv_state->width_in_mb *
 710                priv_state->height_in_mb * 52;
 711     ALLOC_GPE_RESOURCE(res_mb_statistics_buffer,
 712                        "MB statistics buffer",
 713                        res_size);
 714
 715     // PAK pipe buffer allocation
 716     size_shift = (priv_state->bit_depth_luma_minus8 ||
 717                   priv_state->bit_depth_chroma_minus8) ? 2 : 3;
 718
 719     res_size = ALIGN(priv_state->picture_width, 32) >> size_shift;
 720     ALLOC_GPE_RESOURCE(deblocking_filter_line_buffer,
 721                        "Deblocking filter line buffer",
 722                        res_size << 6);
 723     ALLOC_GPE_RESOURCE(deblocking_filter_tile_line_buffer,
 724                        "Deblocking filter tile line buffer",
 725                        res_size << 6);
 726
 727     res_size = ALIGN(priv_state->picture_height +
 728                      priv_state->height_in_lcu * 6, 32) >> size_shift;
 729     ALLOC_GPE_RESOURCE(deblocking_filter_tile_column_buffer,
 730                        "Deblocking filter tile column buffer",
 731                        res_size << 6);
 732
 733     res_size = (((priv_state->picture_width + 15) >> 4) * 188 +
 734                 priv_state->width_in_lcu * 9 + 1023) >> 9;
 735     ALLOC_GPE_RESOURCE(metadata_line_buffer,
 736                        "metadata line buffer",
 737                        res_size << 6);
 738
 739     res_size = (((priv_state->picture_width + 15) >> 4) * 172 +
 740                 priv_state->width_in_lcu * 9 + 1023) >> 9;
 741     ALLOC_GPE_RESOURCE(metadata_tile_line_buffer,
 742                        "metadata tile line buffer",
 743                        res_size << 6);
 744
 745     res_size = (((priv_state->picture_height + 15) >> 4) * 176 +
 746                 priv_state->height_in_lcu * 89 + 1023) >> 9;
 747     ALLOC_GPE_RESOURCE(metadata_tile_column_buffer,
 748                        "metadata tile column buffer",
 749                        res_size << 6);
 750
 751     res_size = ALIGN(((priv_state->picture_width >> 1) +
 752                       priv_state->width_in_lcu * 3), 16) >> size_shift;
 753     ALLOC_GPE_RESOURCE(sao_line_buffer,
 754                        "sao line buffer",
 755                        res_size << 6);
 756
 757     res_size = ALIGN(((priv_state->picture_width >> 1) +
 758                       priv_state->width_in_lcu * 6), 16) >> size_shift;
 759     ALLOC_GPE_RESOURCE(sao_tile_line_buffer,
 760                        "sao tile line buffer",
 761                        res_size << 6);
 762
 763     res_size = ALIGN(((priv_state->picture_height >> 1) +
 764                       priv_state->height_in_lcu * 6), 16) >> size_shift;
 765     ALLOC_GPE_RESOURCE(sao_tile_column_buffer,
 766                        "sao tile column buffer",
 767                        res_size << 6);
 768
 769     priv_ctx->res_inited = 1;
 770     return VA_STATUS_SUCCESS;
 771
 772 FAIL:
 773     gen9_hevc_enc_free_resources(vme_context);
 774     return VA_STATUS_ERROR_ALLOCATION_FAILED;
 775 }
 776
 777 #define VME_IMPLEMENTATION_START
 778
 779 static void
 780 gen9_hevc_set_gpe_1d_surface(VADriverContextP ctx,
 781                              struct gen9_hevc_encoder_context *priv_ctx,
 782                              struct i965_gpe_context *gpe_context,
 783                              enum GEN9_HEVC_ENC_SURFACE_TYPE surface_type,
 784                              int bti_idx,
 785                              int is_raw_buffer,
 786                              int size,
 787                              unsigned int offset,
 788                              struct i965_gpe_resource *gpe_buffer,
 789                              dri_bo *bo)
 790 {
 791     if (!gpe_buffer && !bo) {
 792         gpe_buffer = priv_ctx->gpe_surfaces[surface_type].gpe_resource;
 793         bo = priv_ctx->gpe_surfaces[surface_type].bo;
 794     }
 795
 796     if (gpe_buffer)
 797         i965_add_buffer_gpe_surface(ctx, gpe_context,
 798                                     gpe_buffer, is_raw_buffer,
 799                                     size == 0 ? gpe_buffer->size - offset : size,
 800                                     offset, bti_idx);
 801     else if (bo)
 802         gen9_add_dri_buffer_gpe_surface(ctx, gpe_context,
 803                                         bo, is_raw_buffer,
 804                                         size == 0 ? bo->size - offset : size,
 805                                         offset, bti_idx);
 806 }
 807
 808 static void
 809 gen9_hevc_set_gpe_2d_surface(VADriverContextP ctx,
 810                              struct gen9_hevc_encoder_context *priv_ctx,
 811                              struct i965_gpe_context *gpe_context,
 812                              enum GEN9_HEVC_ENC_SURFACE_TYPE surface_type,
 813                              int bti_idx,
 814                              int has_uv_surface,
 815                              int is_media_block_rw,
 816                              unsigned int format,
 817                              struct i965_gpe_resource *gpe_buffer,
 818                              struct object_surface *surface_object)
 819 {
 820     if (!gpe_buffer && !surface_object) {
 821         gpe_buffer = priv_ctx->gpe_surfaces[surface_type].gpe_resource;
 822         surface_object = priv_ctx->gpe_surfaces[surface_type].surface_object;
 823     }
 824
 825     if (gpe_buffer) {
 826         i965_add_buffer_2d_gpe_surface(ctx,
 827                                        gpe_context,
 828                                        gpe_buffer,
 829                                        is_media_block_rw,
 830                                        format,
 831                                        bti_idx);
 832     } else if (surface_object) {
 833         i965_add_2d_gpe_surface(ctx, gpe_context,
 834                                 surface_object,
 835                                 0, is_media_block_rw, format,
 836                                 bti_idx);
 837
 838         if (has_uv_surface)
 839             i965_add_2d_gpe_surface(ctx, gpe_context,
 840                                     surface_object,
 841                                     1, is_media_block_rw, format,
 842                                     bti_idx + 1);
 843     }
 844 }
 845
 846 static void
 847 gen9_hevc_set_gpe_adv_surface(VADriverContextP ctx,
 848                               struct gen9_hevc_encoder_context *priv_ctx,
 849                               struct i965_gpe_context *gpe_context,
 850                               enum GEN9_HEVC_ENC_SURFACE_TYPE surface_type,
 851                               int bti_idx,
 852                               struct object_surface *surface_object)
 853 {
 854     if (!surface_object)
 855         surface_object = priv_ctx->gpe_surfaces[surface_type].surface_object;
 856
 857     if (surface_object)
 858         i965_add_adv_gpe_surface(ctx, gpe_context,
 859                                  surface_object, bti_idx);
 860 }
 861
 862 static void
 863 gen9_hevc_add_gpe_surface(struct gen9_hevc_encoder_context *priv_ctx,
 864                           enum GEN9_HEVC_ENC_SURFACE_TYPE surface_type,
 865                           struct i965_gpe_resource *gpe_buffer,
 866                           struct object_surface *surface_object)
 867 {
 868     if (gpe_buffer && gpe_buffer->bo)
 869         priv_ctx->gpe_surfaces[surface_type].gpe_resource = gpe_buffer;
 870     else if (surface_object)
 871         priv_ctx->gpe_surfaces[surface_type].surface_object = surface_object;
 872 }
 873
 874 static void
 875 gen9_hevc_init_gpe_surfaces_table(VADriverContextP ctx,
 876                                   struct encode_state *encode_state,
 877                                   struct intel_encoder_context *encoder_context)
 878 {
 879     struct encoder_vme_mfc_context *vme_context = NULL;
 880     struct gen9_hevc_encoder_context *priv_ctx = NULL;
 881     struct gen9_hevc_surface_priv *surface_priv = NULL;
 882
 883     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
 884     priv_ctx = (struct gen9_hevc_encoder_context *)vme_context->private_enc_ctx;
 885
 886     if (encode_state->reconstructed_object->fourcc == VA_FOURCC_P010) {
 887         surface_priv = (struct gen9_hevc_surface_priv *)encode_state->reconstructed_object->private_data;
 888
 889         gen9_hevc_add_gpe_surface(priv_ctx, HEVC_ENC_SURFACE_RAW_Y, NULL,
 890                                   surface_priv->surface_obj_nv12);
 891         gen9_hevc_add_gpe_surface(priv_ctx, HEVC_ENC_SURFACE_RAW_Y_UV, NULL,
 892                                   surface_priv->surface_obj_nv12);
 893         gen9_hevc_add_gpe_surface(priv_ctx, HEVC_ENC_SURFACE_RAW_VME, NULL,
 894                                   surface_priv->surface_obj_nv12);
 895     } else {
 896         gen9_hevc_add_gpe_surface(priv_ctx, HEVC_ENC_SURFACE_RAW_Y, NULL,
 897                                   encode_state->input_yuv_object);
 898         gen9_hevc_add_gpe_surface(priv_ctx, HEVC_ENC_SURFACE_RAW_Y_UV, NULL,
 899                                   encode_state->input_yuv_object);
 900         gen9_hevc_add_gpe_surface(priv_ctx, HEVC_ENC_SURFACE_RAW_VME, NULL,
 901                                   encode_state->input_yuv_object);
 902     }
 903
 904     if (priv_ctx->scaled_2x_surface_obj) {
 905         gen9_hevc_add_gpe_surface(priv_ctx, HEVC_ENC_SURFACE_Y_2X, NULL,
 906                                   priv_ctx->scaled_2x_surface_obj);
 907         gen9_hevc_add_gpe_surface(priv_ctx, HEVC_ENC_SURFACE_Y_2X_VME, NULL,
 908                                   priv_ctx->scaled_2x_surface_obj);
 909     }
 910
 911     gen9_hevc_add_gpe_surface(priv_ctx, HEVC_ENC_SURFACE_BRC_HISTORY,
 912                               &priv_ctx->res_brc_history_buffer,
 913                               NULL);
 914     gen9_hevc_add_gpe_surface(priv_ctx, HEVC_ENC_SURFACE_BRC_PAST_PAK_INFO,
 915                               &priv_ctx->res_brc_pak_statistic_buffer,
 916                               NULL);
 917     gen9_hevc_add_gpe_surface(priv_ctx, HEVC_ENC_SURFACE_HCP_PAK,
 918                               &priv_ctx->res_mb_code_surface,
 919                               NULL);
 920     gen9_hevc_add_gpe_surface(priv_ctx, HEVC_ENC_SURFACE_CU_RECORD,
 921                               &priv_ctx->res_mb_code_surface,
 922                               NULL);
 923     gen9_hevc_add_gpe_surface(priv_ctx, HEVC_ENC_SURFACE_32x32_PU_OUTPUT,
 924                               &priv_ctx->res_32x32_pu_output_buffer,
 925                               NULL);
 926     gen9_hevc_add_gpe_surface(priv_ctx, HEVC_ENC_SURFACE_SLICE_MAP,
 927                               &priv_ctx->res_slice_map_buffer,
 928                               NULL);
 929     gen9_hevc_add_gpe_surface(priv_ctx, HEVC_ENC_SURFACE_BRC_INPUT,
 930                               &priv_ctx->res_brc_input_buffer_for_enc_kernels,
 931                               NULL);
 932     gen9_hevc_add_gpe_surface(priv_ctx, HEVC_ENC_SURFACE_LCU_QP,
 933                               &priv_ctx->res_brc_mb_qp_buffer,
 934                               NULL);
 935     gen9_hevc_add_gpe_surface(priv_ctx, HEVC_ENC_SURFACE_ROI,
 936                               &priv_ctx->res_roi_buffer,
 937                               NULL);
 938     gen9_hevc_add_gpe_surface(priv_ctx, HEVC_ENC_SURFACE_RAW_MBSTAT,
 939                               &priv_ctx->res_mb_statistics_buffer,
 940                               NULL);
 941     gen9_hevc_add_gpe_surface(priv_ctx, HEVC_ENC_SURFACE_BRC_DATA,
 942                               &priv_ctx->res_brc_constant_data_buffer,
 943                               NULL);
 944     gen9_hevc_add_gpe_surface(priv_ctx, HEVC_ENC_SURFACE_KERNEL_DEBUG,
 945                               &priv_ctx->res_kernel_debug,
 946                               NULL);
 947     gen9_hevc_add_gpe_surface(priv_ctx, HEVC_ENC_SURFACE_SIMPLIFIED_INTRA,
 948                               &priv_ctx->res_simplest_intra_buffer,
 949                               NULL);
 950     gen9_hevc_add_gpe_surface(priv_ctx, HEVC_ENC_SURFACE_HME_MVP,
 951                               &priv_ctx->s4x_memv_data_buffer,
 952                               NULL);
 953     gen9_hevc_add_gpe_surface(priv_ctx, HEVC_ENC_SURFACE_HME_DIST,
 954                               &priv_ctx->s4x_memv_distortion_buffer,
 955                               NULL);
 956     gen9_hevc_add_gpe_surface(priv_ctx, HEVC_ENC_SURFACE_16x16PU_SAD,
 957                               &priv_ctx->res_sad_16x16_pu_buffer,
 958                               NULL);
 959     gen9_hevc_add_gpe_surface(priv_ctx, HEVC_ENC_SURFACE_VME_8x8,
 960                               &priv_ctx->res_vme_8x8_mode_buffer,
 961                               NULL);
 962     gen9_hevc_add_gpe_surface(priv_ctx, HEVC_ENC_SURFACE_INTRA_MODE,
 963                               &priv_ctx->res_intra_mode_buffer,
 964                               NULL);
 965     gen9_hevc_add_gpe_surface(priv_ctx, HEVC_ENC_SURFACE_INTRA_DIST,
 966                               &priv_ctx->res_intra_distortion_buffer,
 967                               NULL);
 968     gen9_hevc_add_gpe_surface(priv_ctx, HEVC_ENC_SURFACE_MIN_DIST,
 969                               &priv_ctx->res_min_distortion_buffer,
 970                               NULL);
 971     gen9_hevc_add_gpe_surface(priv_ctx, HEVC_ENC_SURFACE_VME_UNI_SIC_DATA,
 972                               &priv_ctx->res_vme_uni_sic_buffer,
 973                               NULL);
 974     gen9_hevc_add_gpe_surface(priv_ctx, HEVC_ENC_SURFACE_CONCURRENT_THREAD,
 975                               &priv_ctx->res_con_corrent_thread_buffer,
 976                               NULL);
 977     gen9_hevc_add_gpe_surface(priv_ctx, HEVC_ENC_SURFACE_MB_MV_INDEX,
 978                               &priv_ctx->res_mv_index_buffer,
 979                               NULL);
 980     gen9_hevc_add_gpe_surface(priv_ctx, HEVC_ENC_SURFACE_MVP_INDEX,
 981                               &priv_ctx->res_mvp_index_buffer,
 982                               NULL);
 983 }
 984
 985 static VAStatus
 986 gen9_hevc_enc_check_parameters(VADriverContextP ctx,
 987                                struct encode_state *encode_state,
 988                                struct intel_encoder_context *encoder_context)
 989 {
 990     VAEncSequenceParameterBufferHEVC *seq_param = NULL;
 991     VAEncPictureParameterBufferHEVC *pic_param = NULL;
 992     VAEncSliceParameterBufferHEVC *slice_param = NULL;
 993     int i = 0, j = 0;
 994
 995     seq_param = (VAEncSequenceParameterBufferHEVC *)encode_state->seq_param_ext->buffer;
 996     pic_param = (VAEncPictureParameterBufferHEVC *)encode_state->pic_param_ext->buffer;
 997
 998     for (i = 0; i < encode_state->num_slice_params_ext; i++) {
 999         slice_param = (VAEncSliceParameterBufferHEVC *)encode_state->slice_params_ext[i]->buffer;
1000
1001         if (slice_param->slice_fields.bits.slice_temporal_mvp_enabled_flag &&
1002             slice_param->slice_fields.bits.collocated_from_l0_flag &&
1003             (pic_param->collocated_ref_pic_index == 0xff ||
1004              pic_param->collocated_ref_pic_index > GEN9_MAX_REF_SURFACES))
1005             slice_param->slice_fields.bits.slice_temporal_mvp_enabled_flag = 0;
1006
1007         if (slice_param->num_ref_idx_l0_active_minus1 > GEN9_HEVC_NUM_MAX_REF_L0 - 1 ||
1008             slice_param->num_ref_idx_l1_active_minus1 > GEN9_HEVC_NUM_MAX_REF_L1 - 1)
1009             return VA_STATUS_ERROR_ATTR_NOT_SUPPORTED;
1010     }
1011
1012     i = 1 << (seq_param->log2_diff_max_min_luma_coding_block_size +
1013               seq_param->log2_min_luma_coding_block_size_minus3 + 3);
1014     if (i < GEN9_HEVC_ENC_MIN_LCU_SIZE ||
1015         i > GEN9_HEVC_ENC_MAX_LCU_SIZE)
1016         return VA_STATUS_ERROR_ATTR_NOT_SUPPORTED;
1017
1018     //The TU max size in SPS must be the same as the CU max size in SPS
1019     i = seq_param->log2_min_transform_block_size_minus2 +
1020         seq_param->log2_diff_max_min_transform_block_size + 2;
1021     j = seq_param->log2_min_luma_coding_block_size_minus3 +
1022         seq_param->log2_diff_max_min_luma_coding_block_size + 3;
1023
1024     if (i != j)
1025         return VA_STATUS_ERROR_ATTR_NOT_SUPPORTED;
1026
1027     slice_param = (VAEncSliceParameterBufferHEVC *)encode_state->slice_params_ext[0]->buffer;
1028     i = pic_param->pic_init_qp + slice_param->slice_qp_delta;
1029     j = -seq_param->seq_fields.bits.bit_depth_luma_minus8 * 6;
1030     if (i < j || i > 51)
1031         return VA_STATUS_ERROR_INVALID_PARAMETER;
1032
1033     return VA_STATUS_SUCCESS;
1034 }
1035
1036 static void
1037 gen9_hevc_enc_init_seq_parameters(struct gen9_hevc_encoder_context *priv_ctx,
1038                                   struct generic_enc_codec_state *generic_state,
1039                                   struct gen9_hevc_encoder_state *priv_state,
1040                                   VAEncSequenceParameterBufferHEVC *seq_param)
1041 {
1042     int new = 0, m = 0, n = 0;
1043
1044     if (priv_state->picture_width != seq_param->pic_width_in_luma_samples ||
1045         priv_state->picture_height != seq_param->pic_height_in_luma_samples ||
1046         priv_state->bit_depth_luma_minus8 != seq_param->seq_fields.bits.bit_depth_luma_minus8 ||
1047         priv_state->bit_depth_chroma_minus8 != seq_param->seq_fields.bits.bit_depth_chroma_minus8)
1048         new = 1;
1049
1050     if (!new)
1051         return;
1052
1053     priv_state->bit_depth_luma_minus8 = seq_param->seq_fields.bits.bit_depth_luma_minus8;
1054     priv_state->bit_depth_chroma_minus8 = seq_param->seq_fields.bits.bit_depth_chroma_minus8;
1055     priv_state->cu_size = 1 << (seq_param->log2_min_luma_coding_block_size_minus3 + 3);
1056     priv_state->lcu_size = 1 << (seq_param->log2_diff_max_min_luma_coding_block_size +
1057                                  seq_param->log2_min_luma_coding_block_size_minus3 + 3);
1058     priv_state->picture_width = (seq_param->pic_width_in_luma_samples / priv_state->cu_size) * priv_state->cu_size;
1059     priv_state->picture_height = (seq_param->pic_height_in_luma_samples / priv_state->cu_size) * priv_state->cu_size;
1060     priv_state->width_in_lcu  = ALIGN(priv_state->picture_width, priv_state->lcu_size) / priv_state->lcu_size;
1061     priv_state->height_in_lcu = ALIGN(priv_state->picture_height, priv_state->lcu_size) / priv_state->lcu_size;
1062     priv_state->width_in_cu  = ALIGN(priv_state->picture_width, priv_state->cu_size) / priv_state->cu_size;
1063     priv_state->height_in_cu = ALIGN(priv_state->picture_height, priv_state->cu_size) / priv_state->cu_size;
1064     priv_state->width_in_mb  = ALIGN(priv_state->picture_width, 16) / 16;
1065     priv_state->height_in_mb = ALIGN(priv_state->picture_height, 16) / 16;
1066
1067     m = (priv_state->picture_width + GEN9_HEVC_ENC_MIN_LCU_SIZE - 1) / GEN9_HEVC_ENC_MIN_LCU_SIZE;
1068     n = (priv_state->picture_height + GEN9_HEVC_ENC_MIN_LCU_SIZE - 1) / GEN9_HEVC_ENC_MIN_LCU_SIZE;
1069     priv_state->mb_data_offset = ALIGN(m * n * priv_state->pak_obj_size, 0x1000);
1070
1071     m = ALIGN(priv_state->picture_width, GEN9_HEVC_ENC_MAX_LCU_SIZE) / 8;
1072     n = ALIGN(priv_state->picture_height, GEN9_HEVC_ENC_MAX_LCU_SIZE) / 8;
1073     priv_state->mb_code_size = priv_state->mb_data_offset +
1074                                ALIGN(m * n * priv_state->cu_record_size, 0x1000);
1075
1076     priv_state->frame_width_in_max_lcu = ALIGN(priv_state->picture_width, 32);
1077     priv_state->frame_height_in_max_lcu = ALIGN(priv_state->picture_height, 32);
1078     priv_state->frame_width_4x = ALIGN(priv_state->picture_width / 4, 16);
1079     priv_state->frame_height_4x = ALIGN(priv_state->picture_height / 4, 16);
1080     priv_state->frame_width_16x = ALIGN(priv_state->picture_width / 16, 16);
1081     priv_state->frame_height_16x = ALIGN(priv_state->picture_height / 16, 16);
1082     priv_state->frame_width_32x = ALIGN(priv_state->picture_width / 32, 16);
1083     priv_state->frame_height_32x = ALIGN(priv_state->picture_height / 32, 16);
1084
1085     priv_state->downscaled_width_4x_in_mb = priv_state->frame_width_4x / 16;
1086     if (priv_state->bit_depth_luma_minus8) {
1087         priv_state->downscaled_width_4x_in_mb = ALIGN(priv_state->downscaled_width_4x_in_mb * 16, 32) /
1088                                                 16;
1089         priv_state->frame_width_4x = priv_state->downscaled_width_4x_in_mb * 16;
1090     }
1091
1092     priv_state->downscaled_height_4x_in_mb = priv_state->frame_height_4x / 16;
1093     priv_state->downscaled_width_16x_in_mb = priv_state->frame_width_16x / 16;
1094     priv_state->downscaled_height_16x_in_mb = priv_state->frame_height_16x / 16;
1095     priv_state->downscaled_width_32x_in_mb = priv_state->frame_width_32x / 16;
1096     priv_state->downscaled_height_32x_in_mb = priv_state->frame_height_32x / 16;
1097     priv_state->flatness_check_enable = priv_state->flatness_check_supported;
1098     priv_state->widi_first_intra_refresh = 1;
1099
1100     generic_state->hme_supported = GEN9_HEVC_HME_SUPPORTED;
1101     generic_state->b16xme_supported = GEN9_HEVC_16XME_SUPPORTED;
1102     generic_state->b32xme_supported = GEN9_HEVC_32XME_SUPPORTED;
1103     if (generic_state->hme_supported &&
1104         (priv_state->frame_width_4x < GEN9_HEVC_VME_MIN_ALLOWED_SIZE ||
1105          priv_state->frame_height_4x < GEN9_HEVC_VME_MIN_ALLOWED_SIZE)) {
1106         generic_state->b16xme_supported = 0;
1107         generic_state->b32xme_supported = 0;
1108
1109         if (priv_state->frame_width_4x < GEN9_HEVC_VME_MIN_ALLOWED_SIZE) {
1110             priv_state->frame_width_4x = GEN9_HEVC_VME_MIN_ALLOWED_SIZE;
1111             priv_state->downscaled_width_4x_in_mb = priv_state->frame_width_4x / 16;
1112         }
1113
1114         if (priv_state->frame_height_4x < GEN9_HEVC_VME_MIN_ALLOWED_SIZE) {
1115             priv_state->frame_height_4x = GEN9_HEVC_VME_MIN_ALLOWED_SIZE;
1116             priv_state->downscaled_height_4x_in_mb = priv_state->frame_height_4x / 16;
1117         }
1118     } else if (generic_state->b16xme_supported &&
1119                (priv_state->frame_width_16x < GEN9_HEVC_VME_MIN_ALLOWED_SIZE ||
1120                 priv_state->frame_height_16x < GEN9_HEVC_VME_MIN_ALLOWED_SIZE)) {
1121         generic_state->b32xme_supported = 0;
1122
1123         if (priv_state->frame_width_16x < GEN9_HEVC_VME_MIN_ALLOWED_SIZE) {
1124             priv_state->frame_width_16x = GEN9_HEVC_VME_MIN_ALLOWED_SIZE;
1125             priv_state->downscaled_width_16x_in_mb = priv_state->frame_width_16x / 16;
1126         }
1127
1128         if (priv_state->frame_height_16x < GEN9_HEVC_VME_MIN_ALLOWED_SIZE) {
1129             priv_state->frame_height_16x = GEN9_HEVC_VME_MIN_ALLOWED_SIZE;
1130             priv_state->downscaled_height_16x_in_mb = priv_state->frame_height_16x / 16;
1131         }
1132     } else if (generic_state->b32xme_supported &&
1133                (priv_state->frame_width_32x < GEN9_HEVC_VME_MIN_ALLOWED_SIZE ||
1134                 priv_state->frame_height_32x < GEN9_HEVC_VME_MIN_ALLOWED_SIZE)) {
1135         if (priv_state->frame_width_32x < GEN9_HEVC_VME_MIN_ALLOWED_SIZE) {
1136             priv_state->frame_width_32x = GEN9_HEVC_VME_MIN_ALLOWED_SIZE;
1137             priv_state->downscaled_width_32x_in_mb = priv_state->frame_width_32x / 16;
1138         }
1139
1140         if (priv_state->frame_height_32x < GEN9_HEVC_VME_MIN_ALLOWED_SIZE) {
1141             priv_state->frame_height_32x = GEN9_HEVC_VME_MIN_ALLOWED_SIZE;
1142             priv_state->downscaled_height_32x_in_mb = priv_state->frame_height_32x / 16;
1143         }
1144     }
1145
1146     priv_state->gop_ref_dist = seq_param->ip_period;
1147     priv_state->gop_size = seq_param->intra_period;
1148     priv_state->frame_number = 0;
1149
1150     priv_ctx->res_inited = 0;
1151 }
1152
1153 static void
1154 gen9_hevc_enc_init_pic_parameters(struct generic_enc_codec_state *generic_state,
1155                                   struct gen9_hevc_encoder_state *priv_state,
1156                                   VAEncSequenceParameterBufferHEVC *seq_param,
1157                                   VAEncPictureParameterBufferHEVC *pic_param,
1158                                   VAEncSliceParameterBufferHEVC *slice_param)
1159 {
1160     unsigned int log2_max_coding_block_size = 0, raw_ctu_bits = 0;
1161
1162     priv_state->picture_coding_type = slice_param->slice_type;
1163
1164     priv_state->ctu_max_bitsize_allowed = pic_param->ctu_max_bitsize_allowed;
1165     log2_max_coding_block_size  = seq_param->log2_min_luma_coding_block_size_minus3 + 3 +
1166                                   seq_param->log2_diff_max_min_luma_coding_block_size;
1167     raw_ctu_bits = (1 << (2 * log2_max_coding_block_size + 3)) +
1168                    (1 << (2 * log2_max_coding_block_size + 2));
1169     raw_ctu_bits = (5 * raw_ctu_bits / 3);
1170
1171     if (priv_state->ctu_max_bitsize_allowed == 0 ||
1172         priv_state->ctu_max_bitsize_allowed > raw_ctu_bits)
1173         priv_state->ctu_max_bitsize_allowed = raw_ctu_bits;
1174 }
1175
1176 static void
1177 gen9_hevc_enc_init_slice_parameters(VADriverContextP ctx,
1178                                     struct encode_state *encode_state,
1179                                     struct intel_encoder_context *encoder_context)
1180 {
1181     struct encoder_vme_mfc_context *vme_context = NULL;
1182     struct gen9_hevc_encoder_state *priv_state = NULL;
1183     VAEncPictureParameterBufferHEVC *pic_param = NULL;
1184     VAEncSliceParameterBufferHEVC *slice_param = NULL;
1185     int i = 0, j = 0;
1186
1187     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
1188     priv_state = (struct gen9_hevc_encoder_state *)vme_context->private_enc_state;
1189     pic_param = (VAEncPictureParameterBufferHEVC *)encode_state->pic_param_ext->buffer;
1190
1191     priv_state->low_delay = 1;
1192     priv_state->arbitrary_num_mb_in_slice = 0;
1193
1194     for (i = 0; i < encode_state->num_slice_params_ext; i++) {
1195         slice_param = (VAEncSliceParameterBufferHEVC *)encode_state->slice_params_ext[i]->buffer;
1196
1197         if (slice_param->slice_type == HEVC_SLICE_B && priv_state->low_delay) {
1198             for (j = 0; j <= slice_param->num_ref_idx_l0_active_minus1; j++) {
1199                 if (pic_param->decoded_curr_pic.pic_order_cnt <
1200                     slice_param->ref_pic_list0[j].pic_order_cnt)
1201                     priv_state->low_delay = 0;
1202             }
1203
1204             for (j = 0; j <= slice_param->num_ref_idx_l1_active_minus1; j++) {
1205                 if (pic_param->decoded_curr_pic.pic_order_cnt <
1206                     slice_param->ref_pic_list1[j].pic_order_cnt)
1207                     priv_state->low_delay = 0;
1208             }
1209         }
1210
1211         if (!priv_state->arbitrary_num_mb_in_slice &&
1212             (slice_param->num_ctu_in_slice % priv_state->width_in_lcu))
1213             priv_state->arbitrary_num_mb_in_slice = 1;
1214     }
1215 }
1216
1217 static VAStatus
1218 gen9_hevc_enc_init_parameters(VADriverContextP ctx,
1219                               struct encode_state *encode_state,
1220                               struct intel_encoder_context *encoder_context)
1221 {
1222     struct i965_driver_data *i965 = i965_driver_data(ctx);
1223     struct encoder_vme_mfc_context *vme_context = NULL;
1224     struct generic_enc_codec_state *generic_state = NULL;
1225     struct gen9_hevc_encoder_context *priv_ctx = NULL;
1226     struct gen9_hevc_encoder_state *priv_state = NULL;
1227     VAEncPictureParameterBufferHEVC *pic_param = NULL;
1228     VAEncSequenceParameterBufferHEVC *seq_param = NULL;
1229     VAEncSliceParameterBufferHEVC *slice_param = NULL;
1230     struct object_buffer *obj_buffer = NULL;
1231     VAStatus va_status = VA_STATUS_SUCCESS;
1232
1233     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
1234     generic_state = (struct generic_enc_codec_state *)vme_context->generic_enc_state;
1235     priv_ctx = (struct gen9_hevc_encoder_context *)vme_context->private_enc_ctx;
1236     priv_state = (struct gen9_hevc_encoder_state *)vme_context->private_enc_state;
1237     pic_param = (VAEncPictureParameterBufferHEVC *)encode_state->pic_param_ext->buffer;
1238     seq_param = (VAEncSequenceParameterBufferHEVC *)encode_state->seq_param_ext->buffer;
1239     slice_param = (VAEncSliceParameterBufferHEVC *)encode_state->slice_params_ext[0]->buffer;
1240
1241     if (!pic_param || !seq_param || !slice_param)
1242         return VA_STATUS_ERROR_INVALID_PARAMETER;
1243
1244     va_status = gen9_hevc_enc_check_parameters(ctx, encode_state, encoder_context);
1245     if (va_status |= VA_STATUS_SUCCESS)
1246         return va_status;
1247
1248     gen9_hevc_enc_init_seq_parameters(priv_ctx, generic_state, priv_state, seq_param);
1249     gen9_hevc_enc_init_pic_parameters(generic_state, priv_state, seq_param, pic_param, slice_param);
1250     gen9_hevc_enc_init_slice_parameters(ctx, encode_state, encoder_context);
1251
1252     if (priv_state->picture_coding_type == HEVC_SLICE_I) {
1253         generic_state->hme_enabled = 0;
1254         generic_state->b16xme_enabled = 0;
1255         generic_state->b32xme_enabled = 0;
1256     } else {
1257         generic_state->hme_enabled = generic_state->hme_supported;
1258         generic_state->b16xme_enabled = generic_state->b16xme_supported;
1259         generic_state->b32xme_enabled = generic_state->b32xme_supported;
1260     }
1261
1262     obj_buffer = BUFFER(pic_param->coded_buf);
1263     if (!obj_buffer ||
1264         !obj_buffer->buffer_store ||
1265         !obj_buffer->buffer_store->bo) {
1266         va_status = VA_STATUS_ERROR_INVALID_PARAMETER;
1267         goto EXIT;
1268     }
1269     encode_state->coded_buf_object = obj_buffer;
1270     priv_state->status_buffer.bo = obj_buffer->buffer_store->bo;
1271
1272     va_status = gen9_hevc_ensure_surface(ctx, priv_state,
1273                                          encode_state->input_yuv_object, 0);
1274     if (va_status != VA_STATUS_SUCCESS)
1275         goto EXIT;
1276
1277     va_status = gen9_hevc_ensure_surface(ctx, priv_state,
1278                                          encode_state->reconstructed_object, 1);
1279     if (va_status != VA_STATUS_SUCCESS)
1280         goto EXIT;
1281
1282 #if 0
1283     if (encode_state->input_yuv_object->orig_width > priv_state->picture_width)
1284         encode_state->input_yuv_object->orig_width = priv_state->picture_width;
1285
1286     if (encode_state->input_yuv_object->orig_height > priv_state->picture_height)
1287         encode_state->input_yuv_object->orig_height = priv_state->picture_height;
1288
1289
1290     if (encode_state->reconstructed_object->orig_width > priv_state->picture_width)
1291         encode_state->reconstructed_object->orig_width = priv_state->picture_width;
1292
1293     if (encode_state->reconstructed_object->orig_height > priv_state->picture_height)
1294         encode_state->reconstructed_object->orig_height = priv_state->picture_height;
1295 #endif
1296
1297     va_status = gen9_hevc_init_surface_private(ctx, generic_state, priv_state,
1298                                                encode_state->reconstructed_object);
1299     if (va_status != VA_STATUS_SUCCESS)
1300         goto EXIT;
1301
1302     {
1303         struct gen9_hevc_surface_priv *surface_priv = NULL;
1304
1305         surface_priv = (struct gen9_hevc_surface_priv *)encode_state->reconstructed_object->private_data;
1306
1307         surface_priv->qp_value = pic_param->pic_init_qp + slice_param->slice_qp_delta;
1308     }
1309
1310     va_status = gen9_hevc_enc_alloc_resources(ctx, encode_state,
1311                                               encoder_context);
1312     if (va_status != VA_STATUS_SUCCESS)
1313         goto EXIT;
1314
1315     gen9_hevc_init_gpe_surfaces_table(ctx, encode_state,
1316                                       encoder_context);
1317
1318 EXIT:
1319     return va_status;
1320 }
1321
1322 // VME&BRC implementation
1323
1324 static void
1325 gen9_hevc_vme_init_gpe_context(VADriverContextP ctx,
1326                                struct i965_gpe_context *gpe_context,
1327                                unsigned int curbe_size,
1328                                unsigned int inline_data_size)
1329 {
1330     struct i965_driver_data *i965 = i965_driver_data(ctx);
1331
1332     gpe_context->curbe.length = curbe_size;
1333
1334     gpe_context->sampler.entry_size = 0;
1335     gpe_context->sampler.max_entries = 0;
1336
1337     gpe_context->idrt.entry_size = ALIGN(sizeof(struct gen8_interface_descriptor_data), 64);
1338     gpe_context->idrt.max_entries = 1;
1339
1340     gpe_context->surface_state_binding_table.max_entries = MAX_HEVC_KERNELS_ENCODER_SURFACES;
1341     gpe_context->surface_state_binding_table.binding_table_offset = 0;
1342     gpe_context->surface_state_binding_table.surface_state_offset = ALIGN(MAX_HEVC_KERNELS_ENCODER_SURFACES * 4, 64);
1343     gpe_context->surface_state_binding_table.length = ALIGN(MAX_HEVC_KERNELS_ENCODER_SURFACES * 4, 64) + ALIGN(MAX_HEVC_KERNELS_ENCODER_SURFACES * SURFACE_STATE_PADDED_SIZE_GEN9, 64);
1344
1345     if (i965->intel.eu_total > 0)
1346         gpe_context->vfe_state.max_num_threads = 6 * i965->intel.eu_total;
1347     else
1348         gpe_context->vfe_state.max_num_threads = 112;
1349
1350     gpe_context->vfe_state.curbe_allocation_size = MAX(1, ALIGN(gpe_context->curbe.length, 32) >> 5);
1351     gpe_context->vfe_state.urb_entry_size = MAX(1, ALIGN(inline_data_size, 32) >> 5);
1352     gpe_context->vfe_state.num_urb_entries = (MAX_HEVC_KERNELS_URB_SIZE -
1353                                               gpe_context->vfe_state.curbe_allocation_size -
1354                                               ((gpe_context->idrt.entry_size >> 5) *
1355                                                gpe_context->idrt.max_entries)) /
1356                                              gpe_context->vfe_state.urb_entry_size;
1357     gpe_context->vfe_state.num_urb_entries = CLAMP(gpe_context->vfe_state.num_urb_entries, 1, 64);
1358     gpe_context->vfe_state.gpgpu_mode = 0;
1359 }
1360
1361 static void
1362 gen9_hevc_vme_init_scoreboard(struct i965_gpe_context *gpe_context,
1363                               unsigned int mask,
1364                               unsigned int enable,
1365                               unsigned int type)
1366 {
1367     gpe_context->vfe_desc5.scoreboard0.mask = mask;
1368     gpe_context->vfe_desc5.scoreboard0.type = type;
1369     gpe_context->vfe_desc5.scoreboard0.enable = enable;
1370
1371     gpe_context->vfe_desc6.scoreboard1.delta_x0 = 0xF;
1372     gpe_context->vfe_desc6.scoreboard1.delta_y0 = 0x0;
1373
1374     gpe_context->vfe_desc6.scoreboard1.delta_x1 = 0x0;
1375     gpe_context->vfe_desc6.scoreboard1.delta_y1 = 0xF;
1376
1377     gpe_context->vfe_desc6.scoreboard1.delta_x2 = 0x1;
1378     gpe_context->vfe_desc6.scoreboard1.delta_y2 = 0xF;
1379
1380     gpe_context->vfe_desc6.scoreboard1.delta_x3 = 0xF;
1381     gpe_context->vfe_desc6.scoreboard1.delta_y3 = 0xF;
1382
1383     gpe_context->vfe_desc7.scoreboard2.delta_x4 = 0xF;
1384     gpe_context->vfe_desc7.scoreboard2.delta_y4 = 0x1;
1385
1386     gpe_context->vfe_desc7.scoreboard2.delta_x5 = 0x0;
1387     gpe_context->vfe_desc7.scoreboard2.delta_y5 = 0xE;
1388
1389     gpe_context->vfe_desc7.scoreboard2.delta_x6 = 0x1;
1390     gpe_context->vfe_desc7.scoreboard2.delta_y6 = 0xE;
1391
1392     gpe_context->vfe_desc7.scoreboard2.delta_x7 = 0xF;
1393     gpe_context->vfe_desc7.scoreboard2.delta_y7 = 0xE;
1394 }
1395
1396 static void
1397 gen9_hevc_vme_set_scoreboard_26z(struct i965_gpe_context *gpe_context,
1398                                  unsigned int mask,
1399                                  unsigned int enable,
1400                                  unsigned int type)
1401 {
1402     gpe_context->vfe_desc5.scoreboard0.mask = mask;
1403     gpe_context->vfe_desc5.scoreboard0.type = type;
1404     gpe_context->vfe_desc5.scoreboard0.enable = enable;
1405
1406     gpe_context->vfe_desc6.scoreboard1.delta_x0 = -1;
1407     gpe_context->vfe_desc6.scoreboard1.delta_y0 = 3;
1408
1409     gpe_context->vfe_desc6.scoreboard1.delta_x1 = -1;
1410     gpe_context->vfe_desc6.scoreboard1.delta_y1 = 1;
1411
1412     gpe_context->vfe_desc6.scoreboard1.delta_x2 = -1;
1413     gpe_context->vfe_desc6.scoreboard1.delta_y2 = -1;
1414
1415     gpe_context->vfe_desc6.scoreboard1.delta_x3 = 0;
1416     gpe_context->vfe_desc6.scoreboard1.delta_y3 = -1;
1417
1418     gpe_context->vfe_desc7.scoreboard2.delta_x4 = 0;
1419     gpe_context->vfe_desc7.scoreboard2.delta_y4 = -2;
1420
1421     gpe_context->vfe_desc7.scoreboard2.delta_x5 = 0;
1422     gpe_context->vfe_desc7.scoreboard2.delta_y5 = -3;
1423
1424     gpe_context->vfe_desc7.scoreboard2.delta_x6 = 1;
1425     gpe_context->vfe_desc7.scoreboard2.delta_y6 = -2;
1426
1427     gpe_context->vfe_desc7.scoreboard2.delta_x7 = 1;
1428     gpe_context->vfe_desc7.scoreboard2.delta_y7 = -3;
1429 }
1430
1431 static void
1432 gen9_hevc_vme_set_scoreboard_26(struct i965_gpe_context *gpe_context,
1433                                 unsigned int mask,
1434                                 unsigned int enable,
1435                                 unsigned int type)
1436 {
1437     gpe_context->vfe_desc5.scoreboard0.mask = mask;
1438     gpe_context->vfe_desc5.scoreboard0.type = type;
1439     gpe_context->vfe_desc5.scoreboard0.enable = enable;
1440
1441     gpe_context->vfe_desc6.scoreboard1.delta_x0 = -1;
1442     gpe_context->vfe_desc6.scoreboard1.delta_y0 = 0;
1443
1444     gpe_context->vfe_desc6.scoreboard1.delta_x1 = -1;
1445     gpe_context->vfe_desc6.scoreboard1.delta_y1 = -1;
1446
1447     gpe_context->vfe_desc6.scoreboard1.delta_x2 = 0;
1448     gpe_context->vfe_desc6.scoreboard1.delta_y2 = -1;
1449
1450     gpe_context->vfe_desc6.scoreboard1.delta_x3 = 1;
1451     gpe_context->vfe_desc6.scoreboard1.delta_y3 = -1;
1452 }
1453
1454 static void
1455 gen9_hevc_init_object_walker(struct hevc_enc_kernel_walker_parameter *hevc_walker_param,
1456                              struct gpe_media_object_walker_parameter *gpe_param)
1457 {
1458     memset(gpe_param, 0, sizeof(*gpe_param));
1459
1460     gpe_param->use_scoreboard = hevc_walker_param->use_scoreboard;
1461     gpe_param->block_resolution.x = hevc_walker_param->resolution_x;
1462     gpe_param->block_resolution.y = hevc_walker_param->resolution_y;
1463     gpe_param->global_resolution.x = hevc_walker_param->resolution_x;
1464     gpe_param->global_resolution.y = hevc_walker_param->resolution_y;
1465     gpe_param->global_outer_loop_stride.x = hevc_walker_param->resolution_x;
1466     gpe_param->global_outer_loop_stride.y = 0;
1467     gpe_param->global_inner_loop_unit.x = 0;
1468     gpe_param->global_inner_loop_unit.y = hevc_walker_param->resolution_y;
1469     gpe_param->local_loop_exec_count = 0xFFFF;
1470     gpe_param->global_loop_exec_count = 0xFFFF;
1471
1472     if (hevc_walker_param->no_dependency) {
1473         gpe_param->scoreboard_mask = 0;
1474         gpe_param->use_scoreboard = 0;
1475         gpe_param->local_outer_loop_stride.x = 0;
1476         gpe_param->local_outer_loop_stride.y = 1;
1477         gpe_param->local_inner_loop_unit.x = 1;
1478         gpe_param->local_inner_loop_unit.y = 0;
1479         gpe_param->local_end.x = hevc_walker_param->resolution_x - 1;
1480         gpe_param->local_end.y = 0;
1481     }
1482 }
1483
1484 static void
1485 gen9_hevc_init_object_walker_26z(struct gen9_hevc_encoder_state *priv_state,
1486                                  struct i965_gpe_context *gpe_context,
1487                                  struct gpe_media_object_walker_parameter *gpe_param,
1488                                  int split_count,
1489                                  unsigned int max_slight_height,
1490                                  int use_hw_scoreboard,
1491                                  int scoreboard_type)
1492 {
1493     int width = priv_state->width_in_mb;
1494     int height = max_slight_height * 2;
1495     int ts_width = ((width + 3) & 0xfffc) >> 1;
1496     int lcu_width = (width + 1) >> 1;
1497     int lcu_height = (height + 1) >> 1;
1498     int tmp1 = ((lcu_width + 1) >> 1) + ((lcu_width + ((lcu_height - 1) << 1)) + (2 * split_count - 1)) /
1499                (2 * split_count);
1500
1501     gpe_param->use_scoreboard  = use_hw_scoreboard;
1502     gpe_param->scoreboard_mask = 0xFF;
1503     gpe_param->global_resolution.x = ts_width;
1504     gpe_param->global_resolution.y = 4 * tmp1;
1505     gpe_param->global_start.x = 0;
1506     gpe_param->global_start.y = 0;
1507     gpe_param->global_outer_loop_stride.x = ts_width;
1508     gpe_param->global_outer_loop_stride.y = 0;
1509     gpe_param->global_inner_loop_unit.x = 0;
1510     gpe_param->global_inner_loop_unit.y = 4 * tmp1;
1511     gpe_param->block_resolution.x = ts_width;
1512     gpe_param->block_resolution.y = 4 * tmp1;
1513     gpe_param->local_start.x = ts_width;
1514     gpe_param->local_start.y = 0;
1515     gpe_param->local_end.x = 0;
1516     gpe_param->local_end.y = 0;
1517     gpe_param->local_outer_loop_stride.x = 1;
1518     gpe_param->local_outer_loop_stride.y = 0;
1519     gpe_param->local_inner_loop_unit.x = -2;
1520     gpe_param->local_inner_loop_unit.y = 4;
1521     gpe_param->middle_loop_extra_steps = 3;
1522     gpe_param->mid_loop_unit_x = 0;
1523     gpe_param->mid_loop_unit_y = 1;
1524     gpe_param->global_loop_exec_count = 0;
1525     gpe_param->local_loop_exec_count = ((lcu_width + (lcu_height - 1) * 2 + 2 * split_count - 1) /
1526                                         (2 * split_count)) * 2 -
1527                                        1;
1528
1529     gen9_hevc_vme_set_scoreboard_26z(gpe_context, 0xff, use_hw_scoreboard, scoreboard_type);
1530 }
1531
1532 static void
1533 gen9_hevc_init_object_walker_26(struct gen9_hevc_encoder_state *priv_state,
1534                                 struct i965_gpe_context *gpe_context,
1535                                 struct gpe_media_object_walker_parameter *gpe_param,
1536                                 int split_count,
1537                                 unsigned int max_slight_height,
1538                                 int use_hw_scoreboard,
1539                                 int scoreboard_type)
1540 {
1541     int width = priv_state->width_in_mb;
1542     int height = max_slight_height;
1543     int ts_width = (width + 1) & 0xfffe;
1544     int ts_height = (height + 1) & 0xfffe;
1545     int tmp1 = ((ts_width + 1) >> 1) +
1546                ((ts_width + ((ts_height - 1) << 1)) +
1547                 (2 * split_count - 1)) / (2 * split_count);
1548
1549     gpe_param->use_scoreboard = use_hw_scoreboard;
1550     gpe_param->scoreboard_mask = 0x0f;
1551     gpe_param->global_resolution.x = ts_width;
1552     gpe_param->global_resolution.y = tmp1;
1553     gpe_param->global_start.x = 0;
1554     gpe_param->global_start.y = 0;
1555     gpe_param->global_outer_loop_stride.x = ts_width;
1556     gpe_param->global_outer_loop_stride.y = 0;
1557     gpe_param->global_inner_loop_unit.x = 0;
1558     gpe_param->global_inner_loop_unit.y = tmp1;
1559     gpe_param->block_resolution.x = ts_width;
1560     gpe_param->block_resolution.y = tmp1;
1561     gpe_param->local_start.x = ts_width;
1562     gpe_param->local_start.y = 0;
1563     gpe_param->local_end.x = 0;
1564     gpe_param->local_end.y = 0;
1565     gpe_param->local_outer_loop_stride.x = 1;
1566     gpe_param->local_outer_loop_stride.y = 0;
1567     gpe_param->local_inner_loop_unit.x = -2;
1568     gpe_param->local_inner_loop_unit.y = 1;
1569     gpe_param->middle_loop_extra_steps = 0;
1570     gpe_param->mid_loop_unit_x = 0;
1571     gpe_param->mid_loop_unit_y = 0;
1572     gpe_param->global_loop_exec_count = 0;
1573     gpe_param->local_loop_exec_count = (width + (height - 1) * 2 + split_count - 1) /
1574                                        split_count;
1575
1576     gen9_hevc_vme_set_scoreboard_26(gpe_context, 0xff, use_hw_scoreboard, scoreboard_type);
1577 }
1578
1579 static void
1580 gen9_hevc_run_object_walker(VADriverContextP ctx,
1581                             struct intel_encoder_context *encoder_context,
1582                             struct i965_gpe_context *gpe_context,
1583                             struct gpe_media_object_walker_parameter *param,
1584                             int media_state)
1585 {
1586     struct intel_batchbuffer *batch = encoder_context->base.batch;
1587
1588     intel_batchbuffer_start_atomic(batch, 0x1000);
1589
1590     intel_batchbuffer_emit_mi_flush(batch);
1591
1592     gen9_gpe_pipeline_setup(ctx, gpe_context, batch);
1593     gen8_gpe_media_object_walker(ctx, gpe_context, batch, param);
1594     gen8_gpe_media_state_flush(ctx, gpe_context, batch);
1595     gen9_gpe_pipeline_end(ctx, gpe_context, batch);
1596
1597     intel_batchbuffer_end_atomic(batch);
1598
1599     intel_batchbuffer_flush(batch);
1600 }
1601
1602 static void
1603 gen9_hevc_run_object(VADriverContextP ctx,
1604                      struct intel_encoder_context *encoder_context,
1605                      struct i965_gpe_context *gpe_context,
1606                      struct gpe_media_object_parameter *param,
1607                      int media_state)
1608 {
1609     struct intel_batchbuffer *batch = encoder_context->base.batch;
1610
1611     intel_batchbuffer_start_atomic(batch, 0x1000);
1612
1613     intel_batchbuffer_emit_mi_flush(batch);
1614
1615     gen9_gpe_pipeline_setup(ctx, gpe_context, batch);
1616     gen8_gpe_media_object(ctx, gpe_context, batch, param);
1617     gen8_gpe_media_state_flush(ctx, gpe_context, batch);
1618
1619     gen9_gpe_pipeline_end(ctx, gpe_context, batch);
1620
1621     intel_batchbuffer_end_atomic(batch);
1622
1623     intel_batchbuffer_flush(batch);
1624 }
1625
1626 static void
1627 gen9_hevc_get_b_mbenc_default_curbe(enum HEVC_TU_MODE tu_mode,
1628                                     int slice_type,
1629                                     void **curbe_ptr,
1630                                     int *curbe_size)
1631 {
1632     if (tu_mode == HEVC_TU_BEST_SPEED) {
1633         if (slice_type == HEVC_SLICE_I) {
1634             *curbe_size = sizeof(HEVC_ENC_ENCB_TU7_I_CURBE_DATA);
1635             *curbe_ptr = (void *)HEVC_ENC_ENCB_TU7_I_CURBE_DATA;
1636         } else if (slice_type == HEVC_SLICE_P) {
1637             *curbe_size = sizeof(HEVC_ENC_ENCB_TU7_P_CURBE_DATA);
1638             *curbe_ptr = (void *)HEVC_ENC_ENCB_TU7_P_CURBE_DATA;
1639         } else {
1640             *curbe_size = sizeof(HEVC_ENC_ENCB_TU7_B_CURBE_DATA);
1641             *curbe_ptr = (void *)HEVC_ENC_ENCB_TU7_B_CURBE_DATA;
1642         }
1643     } else if (tu_mode == HEVC_TU_RT_SPEED) {
1644         if (slice_type == HEVC_SLICE_P) {
1645             *curbe_size = sizeof(HEVC_ENC_ENCB_TU4_P_CURBE_DATA);
1646             *curbe_ptr = (void *)HEVC_ENC_ENCB_TU4_P_CURBE_DATA;
1647         } else {
1648             *curbe_size = sizeof(HEVC_ENC_ENCB_TU4_B_CURBE_DATA);
1649             *curbe_ptr = (void *)HEVC_ENC_ENCB_TU4_B_CURBE_DATA;
1650         }
1651     } else {
1652         if (slice_type == HEVC_SLICE_P) {
1653             *curbe_size = sizeof(HEVC_ENC_ENCB_TU1_P_CURBE_DATA);
1654             *curbe_ptr = (void *)HEVC_ENC_ENCB_TU1_P_CURBE_DATA;
1655         } else {
1656             *curbe_size = sizeof(HEVC_ENC_ENCB_TU1_B_CURBE_DATA);
1657             *curbe_ptr = (void *)HEVC_ENC_ENCB_TU1_B_CURBE_DATA;
1658         }
1659     }
1660 }
1661
1662 // BRC start
1663
1664 static void
1665 gen9_hevc_configure_roi(struct encode_state *encode_state,
1666                         struct intel_encoder_context *encoder_context)
1667 {
1668     struct encoder_vme_mfc_context *vme_context = NULL;
1669     struct gen9_hevc_encoder_state *priv_state = NULL;
1670     int i = 0;
1671
1672     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
1673     priv_state = (struct gen9_hevc_encoder_state *)vme_context->private_enc_state;
1674
1675     priv_state->num_roi = MIN(encoder_context->brc.num_roi, I965_MAX_NUM_ROI_REGIONS);
1676     priv_state->roi_value_is_qp_delta = encoder_context->brc.roi_value_is_qp_delta;
1677
1678     for (i = 0; i < priv_state->num_roi; i++) {
1679         priv_state->roi[i].left = encoder_context->brc.roi[i].left >> 4;
1680         priv_state->roi[i].right = encoder_context->brc.roi[i].right >> 4;
1681         priv_state->roi[i].top = encoder_context->brc.roi[i].top >> 4;
1682         priv_state->roi[i].bottom = encoder_context->brc.roi[i].bottom >> 4;
1683         priv_state->roi[i].value = encoder_context->brc.roi[i].value;
1684     }
1685 }
1686
1687 static void
1688 gen9_hevc_brc_prepare(struct encode_state *encode_state,
1689                       struct intel_encoder_context *encoder_context)
1690 {
1691     struct encoder_vme_mfc_context *vme_context = NULL;
1692     struct generic_enc_codec_state *generic_state = NULL;
1693     struct gen9_hevc_encoder_state *priv_state = NULL;
1694     enum HEVC_BRC_METHOD brc_method = HEVC_BRC_CQP;
1695     int internal_tu_mode = encoder_context->quality_level;
1696     int brc_reset = 0;
1697
1698     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
1699     generic_state = (struct generic_enc_codec_state *)vme_context->generic_enc_state;
1700     priv_state = (struct gen9_hevc_encoder_state *)vme_context->private_enc_state;
1701
1702     if (encoder_context->rate_control_mode & VA_RC_CBR)
1703         brc_method = HEVC_BRC_CBR;
1704     else if (encoder_context->rate_control_mode & VA_RC_VBR)
1705         brc_method = HEVC_BRC_VBR;
1706     else if (encoder_context->rate_control_mode & VA_RC_VCM)
1707         brc_method = HEVC_BRC_VCM;
1708
1709     if (internal_tu_mode >= HEVC_TU_RT_SPEED ||
1710         internal_tu_mode == 0)
1711         internal_tu_mode = internal_tu_mode >= HEVC_TU_BEST_SPEED ?
1712                            HEVC_TU_BEST_SPEED : HEVC_TU_RT_SPEED;
1713     else
1714         internal_tu_mode = HEVC_TU_BEST_QUALITY;
1715
1716     brc_reset = priv_state->brc_method != brc_method ||
1717                 priv_state->tu_mode != internal_tu_mode;
1718
1719     if (!generic_state->brc_inited ||
1720         encoder_context->brc.need_reset ||
1721         brc_reset) {
1722         priv_state->tu_mode = internal_tu_mode;
1723         if (priv_state->tu_mode == HEVC_TU_BEST_QUALITY)
1724             priv_state->num_regions_in_slice = 1;
1725         else
1726             priv_state->num_regions_in_slice = 4;
1727
1728         if (internal_tu_mode == HEVC_TU_BEST_SPEED)
1729             priv_state->walking_pattern_26 = 1;
1730         else
1731             priv_state->walking_pattern_26 = 0;
1732
1733         if (brc_method == HEVC_BRC_CQP) {
1734             generic_state->brc_enabled = 0;
1735             generic_state->num_pak_passes = 1;
1736             priv_state->lcu_brc_enabled = 0;
1737         } else {
1738             generic_state->brc_enabled = 1;
1739             generic_state->num_pak_passes = 4;
1740
1741             if (brc_method == HEVC_BRC_VCM ||
1742                 encoder_context->brc.mb_rate_control[0] == 0)
1743                 priv_state->lcu_brc_enabled = (priv_state->tu_mode == HEVC_TU_BEST_QUALITY);
1744             else if (brc_method == HEVC_BRC_ICQ ||
1745                      encoder_context->brc.mb_rate_control[0] == 1)
1746                 priv_state->lcu_brc_enabled = 1;
1747             else
1748                 priv_state->lcu_brc_enabled = 0;
1749
1750             if (brc_method == HEVC_BRC_CBR) {
1751                 priv_state->target_bit_rate_in_kbs =
1752                     ALIGN(encoder_context->brc.bits_per_second[0], HEVC_BRC_KBPS) /
1753                     HEVC_BRC_KBPS;
1754                 priv_state->max_bit_rate_in_kbs = priv_state->target_bit_rate_in_kbs;
1755                 priv_state->min_bit_rate_in_kbs = priv_state->target_bit_rate_in_kbs;
1756             } else {
1757                 if (encoder_context->brc.target_percentage[0] > HEVC_BRC_MIN_TARGET_PERCENTAGE) {
1758                     priv_state->target_bit_rate_in_kbs =
1759                         ALIGN(encoder_context->brc.bits_per_second[0], HEVC_BRC_KBPS) /
1760                         HEVC_BRC_KBPS;
1761                     priv_state->max_bit_rate_in_kbs = priv_state->target_bit_rate_in_kbs;
1762                     priv_state->min_bit_rate_in_kbs = priv_state->target_bit_rate_in_kbs *
1763                                                       (2 * encoder_context->brc.target_percentage[0] - 100) /
1764                                                       100;
1765                     priv_state->target_bit_rate_in_kbs = priv_state->max_bit_rate_in_kbs *
1766                                                          encoder_context->brc.target_percentage[0] / 100;
1767
1768                     brc_reset = 1;
1769                 }
1770             }
1771
1772             if (encoder_context->brc.framerate[0].den)
1773                 priv_state->frames_per_100s = encoder_context->brc.framerate[0].num * 100 /
1774                                               encoder_context->brc.framerate[0].den;
1775
1776             priv_state->init_vbv_buffer_fullness_in_bit =
1777                 encoder_context->brc.hrd_initial_buffer_fullness;
1778             priv_state->vbv_buffer_size_in_bit =
1779                 encoder_context->brc.hrd_buffer_size;
1780         }
1781
1782         priv_state->brc_method = brc_method;
1783         generic_state->brc_need_reset = brc_reset;
1784         encoder_context->brc.need_reset = 0;
1785     }
1786
1787     gen9_hevc_configure_roi(encode_state, encoder_context);
1788 }
1789
1790 static void
1791 gen9_hevc_brc_init_rest_set_curbe(VADriverContextP ctx,
1792                                   struct encode_state *encode_state,
1793                                   struct intel_encoder_context *encoder_context,
1794                                   struct i965_gpe_context *gpe_context,
1795                                   int reset)
1796 {
1797     struct encoder_vme_mfc_context *vme_context = NULL;
1798     struct gen9_hevc_encoder_state *priv_state = NULL;
1799     gen9_hevc_brc_initreset_curbe_data *cmd = NULL;
1800     VAEncSequenceParameterBufferHEVC *seq_param = NULL;
1801     double input_bits_per_frame = 0;
1802     double bps_ratio = 0;
1803
1804     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
1805     priv_state = (struct gen9_hevc_encoder_state *)vme_context->private_enc_state;
1806     seq_param = (VAEncSequenceParameterBufferHEVC *)encode_state->seq_param_ext->buffer;
1807
1808     cmd = i965_gpe_context_map_curbe(gpe_context);
1809     if (!cmd)
1810         return;
1811
1812     memcpy((void *)cmd, GEN9_HEVC_BRCINIT_CURBE_DATA, sizeof(GEN9_HEVC_BRCINIT_CURBE_DATA));
1813
1814     cmd->dw0.profile_level_max_frame = gen9_hevc_get_profile_level_max_frame(seq_param,
1815                                                                              priv_state->user_max_frame_size,
1816                                                                              priv_state->frames_per_100s);
1817     cmd->dw1.init_buf_full = priv_state->init_vbv_buffer_fullness_in_bit;
1818     cmd->dw2.buf_size = priv_state->vbv_buffer_size_in_bit;
1819     cmd->dw3.targe_bit_rate = priv_state->target_bit_rate_in_kbs * HEVC_BRC_KBPS;
1820     cmd->dw4.maximum_bit_rate = priv_state->max_bit_rate_in_kbs * HEVC_BRC_KBPS;
1821
1822     cmd->dw9.frame_width = priv_state->picture_width;
1823     cmd->dw10.frame_height = priv_state->picture_height;
1824     cmd->dw12.number_slice = encode_state->num_slice_params_ext;
1825     cmd->dw6.frame_rate_m = priv_state->frames_per_100s;
1826     cmd->dw7.frame_rate_d = 100;
1827     cmd->dw8.brc_flag = 0;
1828     cmd->dw8.brc_flag |= (priv_state->lcu_brc_enabled) ? 0 : HEVC_BRCINIT_DISABLE_MBBRC;
1829     cmd->dw25.acqp_buffer = 1;
1830
1831     if (priv_state->brc_method == HEVC_BRC_CBR) {
1832         cmd->dw4.maximum_bit_rate = cmd->dw3.targe_bit_rate;
1833         cmd->dw8.brc_flag |= HEVC_BRCINIT_ISCBR;
1834     } else if (priv_state->brc_method == HEVC_BRC_VBR) {
1835         if (cmd->dw4.maximum_bit_rate < cmd->dw3.targe_bit_rate)
1836             cmd->dw4.maximum_bit_rate = cmd->dw3.targe_bit_rate * 2;
1837         cmd->dw8.brc_flag |= HEVC_BRCINIT_ISVBR;
1838     } else if (priv_state->brc_method == HEVC_BRC_AVBR) {
1839         cmd->dw4.maximum_bit_rate = priv_state->target_bit_rate_in_kbs * HEVC_BRC_KBPS;
1840         cmd->dw8.brc_flag |= HEVC_BRCINIT_ISAVBR;
1841     } else if (priv_state->brc_method == HEVC_BRC_ICQ) {
1842         cmd->dw25.acqp_buffer = priv_state->crf_quality_factor;
1843         cmd->dw8.brc_flag |= HEVC_BRCINIT_ISICQ;
1844     } else if (priv_state->brc_method == HEVC_BRC_VCM) {
1845         cmd->dw4.maximum_bit_rate = priv_state->target_bit_rate_in_kbs * HEVC_BRC_KBPS;
1846         cmd->dw8.brc_flag |= HEVC_BRCINIT_ISVCM;
1847     }
1848
1849     if (priv_state->num_b_in_gop[1] ||
1850         priv_state->num_b_in_gop[2]) {
1851         cmd->dw8.brc_param_a = priv_state->gop_size / priv_state->gop_ref_dist;
1852         cmd->dw9.brc_param_b = cmd->dw8.brc_param_a;
1853         cmd->dw13.brc_param_c = cmd->dw8.brc_param_a * 2;
1854         cmd->dw14.brc_param_d = priv_state->gop_size - cmd->dw8.brc_param_a -
1855                                 cmd->dw9.brc_param_b - cmd->dw13.brc_param_c;
1856
1857         if (!priv_state->num_b_in_gop[2])
1858             cmd->dw14.max_brc_level = 3;
1859         else
1860             cmd->dw14.max_brc_level = 4;
1861     } else {
1862         cmd->dw14.max_brc_level = 1;
1863         cmd->dw8.brc_param_a = priv_state->gop_ref_dist ? (priv_state->gop_size - 1) / priv_state->gop_ref_dist : 0;
1864         cmd->dw9.brc_param_b = priv_state->gop_size - 1 - cmd->dw8.brc_param_a;
1865     }
1866
1867     cmd->dw10.avbr_accuracy = GEN9_HEVC_AVBR_ACCURACY;
1868     cmd->dw11.avbr_convergence = GEN9_HEVC_AVBR_CONVERGENCE;
1869
1870     input_bits_per_frame = (double)(cmd->dw4.maximum_bit_rate) *
1871                            (double)(cmd->dw7.frame_rate_d) /
1872                            (double)(cmd->dw6.frame_rate_m);
1873     if (cmd->dw2.buf_size < (unsigned int)input_bits_per_frame * 4)
1874         cmd->dw2.buf_size = (unsigned int)input_bits_per_frame * 4;
1875
1876     if (cmd->dw1.init_buf_full == 0)
1877         cmd->dw1.init_buf_full = cmd->dw2.buf_size * 7 / 8;
1878
1879     if (cmd->dw1.init_buf_full < (unsigned int)(input_bits_per_frame * 2))
1880         cmd->dw1.init_buf_full = (unsigned int)(input_bits_per_frame * 2);
1881
1882     if (cmd->dw1.init_buf_full > cmd->dw2.buf_size)
1883         cmd->dw1.init_buf_full = cmd->dw2.buf_size;
1884
1885     if (priv_state->brc_method == HEVC_BRC_AVBR) {
1886         cmd->dw2.buf_size = priv_state->target_bit_rate_in_kbs * 2 * HEVC_BRC_KBPS;
1887         cmd->dw1.init_buf_full = (unsigned int)(cmd->dw2.buf_size * 3 / 4);
1888     }
1889
1890     bps_ratio = input_bits_per_frame / (cmd->dw2.buf_size / 30);
1891     bps_ratio = (bps_ratio < 0.1) ? 0.1 : (bps_ratio > 3.5) ? 3.5 : bps_ratio;
1892
1893     cmd->dw19.deviation_threshold0_pbframe = (unsigned int)(-50 * pow(0.90, bps_ratio));
1894     cmd->dw19.deviation_threshold1_pbframe = (unsigned int)(-50 * pow(0.66, bps_ratio));
1895     cmd->dw19.deviation_threshold2_pbframe = (unsigned int)(-50 * pow(0.46, bps_ratio));
1896     cmd->dw19.deviation_threshold3_pbframe = (unsigned int)(-50 * pow(0.3, bps_ratio));
1897
1898     cmd->dw20.deviation_threshold4_pbframe = (unsigned int)(50 *  pow(0.3, bps_ratio));
1899     cmd->dw20.deviation_threshold5_pbframe = (unsigned int)(50 * pow(0.46, bps_ratio));
1900     cmd->dw20.deviation_threshold6_pbframe = (unsigned int)(50 * pow(0.7,  bps_ratio));
1901     cmd->dw20.deviation_threshold7_pbframe = (unsigned int)(50 * pow(0.9,  bps_ratio));
1902
1903     cmd->dw21.deviation_threshold0_vbr_control = (unsigned int)(-50 * pow(0.9, bps_ratio));
1904     cmd->dw21.deviation_threshold1_vbr_control = (unsigned int)(-50 * pow(0.7, bps_ratio));
1905     cmd->dw21.deviation_threshold2_vbr_control = (unsigned int)(-50 * pow(0.5, bps_ratio));
1906     cmd->dw21.deviation_threshold3_vbr_control = (unsigned int)(-50 * pow(0.3, bps_ratio));
1907
1908     cmd->dw22.deviation_threshold4_vbr_control = (unsigned int)(100 * pow(0.4, bps_ratio));
1909     cmd->dw22.deviation_threshold5_vbr_control = (unsigned int)(100 * pow(0.5, bps_ratio));
1910     cmd->dw22.deviation_threshold6_vbr_control = (unsigned int)(100 * pow(0.75, bps_ratio));
1911     cmd->dw22.deviation_threshold7_vbr_control = (unsigned int)(100 * pow(0.9, bps_ratio));
1912
1913     cmd->dw23.deviation_threshold0_iframe = (unsigned int)(-50 * pow(0.8, bps_ratio));
1914     cmd->dw23.deviation_threshold1_iframe = (unsigned int)(-50 * pow(0.6, bps_ratio));
1915     cmd->dw23.deviation_threshold2_iframe = (unsigned int)(-50 * pow(0.34, bps_ratio));
1916     cmd->dw23.deviation_threshold3_iframe = (unsigned int)(-50 * pow(0.2, bps_ratio));
1917
1918     cmd->dw24.deviation_threshold4_iframe = (unsigned int)(50 * pow(0.2,  bps_ratio));
1919     cmd->dw24.deviation_threshold5_iframe = (unsigned int)(50 * pow(0.4,  bps_ratio));
1920     cmd->dw24.deviation_threshold6_iframe = (unsigned int)(50 * pow(0.66, bps_ratio));
1921     cmd->dw24.deviation_threshold7_iframe = (unsigned int)(50 * pow(0.9,  bps_ratio));
1922
1923     if (!reset)
1924         priv_state->brc_init_current_target_buf_full_in_bits = cmd->dw1.init_buf_full;
1925
1926     priv_state->brc_init_reset_buf_size_in_bits = (double)cmd->dw2.buf_size;
1927     priv_state->brc_init_reset_input_bits_per_frame = input_bits_per_frame;
1928
1929     i965_gpe_context_unmap_curbe(gpe_context);
1930 }
1931
1932 static void
1933 gen9_hevc_brc_init_rest_set_surfaces(VADriverContextP ctx,
1934                                      struct intel_encoder_context *encoder_context,
1935                                      struct i965_gpe_context *gpe_context)
1936 {
1937     struct encoder_vme_mfc_context *vme_context = NULL;
1938     struct gen9_hevc_encoder_context *priv_ctx = NULL;
1939     int bti_idx = 0;
1940
1941     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
1942     priv_ctx = (struct gen9_hevc_encoder_context *)vme_context->private_enc_ctx;
1943
1944     gen9_hevc_set_gpe_1d_surface(ctx, priv_ctx, gpe_context,
1945                                  HEVC_ENC_SURFACE_BRC_HISTORY, bti_idx++,
1946                                  0, 0, 0, NULL, NULL);
1947
1948     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
1949                                  HEVC_ENC_SURFACE_BRC_ME_DIST, bti_idx++,
1950                                  0, 1, I965_SURFACEFORMAT_R8_UNORM,
1951                                  &priv_ctx->res_brc_me_dist_buffer, NULL);
1952 }
1953
1954 static void
1955 gen9_hevc_brc_init_reset(VADriverContextP ctx,
1956                          struct encode_state *encode_state,
1957                          struct intel_encoder_context *encoder_context,
1958                          int reset)
1959 {
1960     struct encoder_vme_mfc_context *vme_context = NULL;
1961     struct gen9_hevc_encoder_context *priv_ctx = NULL;
1962
1963     struct i965_gpe_context *gpe_context = NULL;
1964     struct gpe_media_object_parameter param;
1965     int media_state = HEVC_ENC_MEDIA_STATE_BRC_INIT_RESET;
1966     int gpe_idx = reset ? HEVC_BRC_RESET_IDX : HEVC_BRC_INIT_IDX;
1967
1968     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
1969     priv_ctx = (struct gen9_hevc_encoder_context *)vme_context->private_enc_ctx;
1970     gpe_context = &priv_ctx->brc_context.gpe_contexts[gpe_idx];
1971
1972     gen8_gpe_context_init(ctx, gpe_context);
1973     gen9_gpe_reset_binding_table(ctx, gpe_context);
1974     gen9_hevc_brc_init_rest_set_curbe(ctx, encode_state, encoder_context, gpe_context,
1975                                       reset);
1976     gen9_hevc_brc_init_rest_set_surfaces(ctx, encoder_context, gpe_context);
1977     gen8_gpe_setup_interface_data(ctx, gpe_context);
1978
1979     memset(&param, 0, sizeof(param));
1980     gen9_hevc_run_object(ctx, encoder_context, gpe_context, &param,
1981                          media_state);
1982 }
1983
1984 static void
1985 gen9_hevc_brc_intra_dist_set_curbe(VADriverContextP ctx,
1986                                    struct encode_state *encode_state,
1987                                    struct intel_encoder_context *encoder_context,
1988                                    struct i965_gpe_context *gpe_context)
1989 {
1990     struct encoder_vme_mfc_context *vme_context = NULL;
1991     struct gen9_hevc_encoder_state *priv_state = NULL;
1992     gen9_hevc_brc_coarse_intra_curbe_data *cmd = NULL;
1993     int bti_idx = 0;
1994
1995     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
1996     priv_state = (struct gen9_hevc_encoder_state *)vme_context->private_enc_state;
1997
1998     cmd = i965_gpe_context_map_curbe(gpe_context);
1999     if (!cmd)
2000         return;
2001
2002     memset((void *)cmd, 0, sizeof(*cmd));
2003
2004     cmd->dw0.picture_width_in_luma_samples = priv_state->frame_width_4x;
2005     cmd->dw0.picture_height_in_luma_samples = priv_state->frame_height_4x;
2006
2007     cmd->dw1.inter_sad = 2;
2008     cmd->dw1.intra_sad = 2;
2009
2010     cmd->dw8.bti_src_y4 = bti_idx++;
2011     cmd->dw9.bti_intra_dist = bti_idx++;
2012     cmd->dw10.bti_vme_intra = bti_idx++;
2013
2014     i965_gpe_context_unmap_curbe(gpe_context);
2015 }
2016
2017 static void
2018 gen9_hevc_brc_intra_dist_set_surfaces(VADriverContextP ctx,
2019                                       struct encode_state *encode_state,
2020                                       struct intel_encoder_context *encoder_context,
2021                                       struct i965_gpe_context *gpe_context)
2022 {
2023     struct encoder_vme_mfc_context *vme_context = NULL;
2024     struct gen9_hevc_encoder_context *priv_ctx = NULL;
2025     struct object_surface *obj_surface;
2026     struct gen9_hevc_surface_priv *surface_priv;
2027     int bti_idx = 0;
2028
2029     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
2030     priv_ctx = (struct gen9_hevc_encoder_context *)vme_context->private_enc_ctx;
2031
2032     obj_surface = encode_state->reconstructed_object;
2033     surface_priv = (struct gen9_hevc_surface_priv *)(obj_surface->private_data);
2034     obj_surface = surface_priv->scaled_surface_obj[HEVC_SCALED_SURF_4X_ID];
2035
2036     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
2037                                  HEVC_ENC_SURFACE_Y_4X, bti_idx++,
2038                                  0, 1, I965_SURFACEFORMAT_R8_UNORM,
2039                                  NULL, obj_surface);
2040
2041     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
2042                                  HEVC_ENC_SURFACE_BRC_ME_DIST, bti_idx++,
2043                                  0, 1, I965_SURFACEFORMAT_R8_UNORM,
2044                                  &priv_ctx->res_brc_intra_dist_buffer, NULL);
2045
2046     gen9_hevc_set_gpe_adv_surface(ctx, priv_ctx, gpe_context,
2047                                   HEVC_ENC_SURFACE_Y_4X_VME, bti_idx++,
2048                                   obj_surface);
2049 }
2050
2051 static void
2052 gen9_hevc_brc_intra_dist(VADriverContextP ctx,
2053                          struct encode_state *encode_state,
2054                          struct intel_encoder_context *encoder_context)
2055 {
2056     struct encoder_vme_mfc_context *vme_context = NULL;
2057     struct gen9_hevc_encoder_context *priv_ctx = NULL;
2058     struct gen9_hevc_encoder_state *priv_state = NULL;
2059     struct i965_gpe_context *gpe_context = NULL;
2060     struct gpe_media_object_walker_parameter param;
2061     struct hevc_enc_kernel_walker_parameter hevc_walker_param;
2062     int media_state = HEVC_ENC_MEDIA_STATE_ENC_I_FRAME_DIST;
2063     int gpe_idx = HEVC_BRC_COARSE_INTRA_IDX;
2064
2065     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
2066     priv_ctx = (struct gen9_hevc_encoder_context *)vme_context->private_enc_ctx;
2067     priv_state = (struct gen9_hevc_encoder_state *)vme_context->private_enc_state;
2068     gpe_context = &priv_ctx->brc_context.gpe_contexts[gpe_idx];
2069
2070     gen8_gpe_context_init(ctx, gpe_context);
2071     gen9_gpe_reset_binding_table(ctx, gpe_context);
2072     gen9_hevc_brc_intra_dist_set_curbe(ctx, encode_state, encoder_context, gpe_context);
2073     gen9_hevc_brc_intra_dist_set_surfaces(ctx, encode_state, encoder_context, gpe_context);
2074     gen8_gpe_setup_interface_data(ctx, gpe_context);
2075
2076     memset((void *)&hevc_walker_param, 0, sizeof(hevc_walker_param));
2077     hevc_walker_param.resolution_x = priv_state->downscaled_width_4x_in_mb;
2078     hevc_walker_param.resolution_y = priv_state->downscaled_height_4x_in_mb;
2079     hevc_walker_param.no_dependency = 1;
2080     gen9_hevc_init_object_walker(&hevc_walker_param, &param);
2081     gen9_hevc_run_object_walker(ctx, encoder_context, gpe_context, &param,
2082                                 media_state);
2083 }
2084
2085 static GEN9_HEVC_BRC_UPDATE_FRAME_TYPE gen9_hevc_get_brc_frame_type(unsigned int pic_type,
2086                                                                     int low_delay)
2087 {
2088     if (pic_type == HEVC_SLICE_I)
2089         return HEVC_BRC_FTYPE_I;
2090     else if (pic_type == HEVC_SLICE_P)
2091         return HEVC_BRC_FTYPE_P_OR_LB;
2092     else
2093         return low_delay ? HEVC_BRC_FTYPE_P_OR_LB : HEVC_BRC_FTYPE_B;
2094 }
2095
2096 static void
2097 gen9_hevc_brc_update_set_roi_curbe(VADriverContextP ctx,
2098                                    struct encode_state *encode_state,
2099                                    struct intel_encoder_context *encoder_context,
2100                                    gen9_hevc_brc_udpate_curbe_data *cmd)
2101 {
2102     struct encoder_vme_mfc_context *vme_context = NULL;
2103     struct generic_enc_codec_state *generic_state = NULL;
2104     struct gen9_hevc_encoder_state *priv_state = NULL;
2105     struct intel_roi *roi_par = NULL;
2106     unsigned int roi_size = 0, roi_ratio = 0;
2107     int i = 0;
2108
2109     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
2110     generic_state = (struct generic_enc_codec_state *)vme_context->generic_enc_state;
2111     priv_state = (struct gen9_hevc_encoder_state *)vme_context->private_enc_state;
2112
2113     cmd->dw6.cqp_value = 0;
2114     cmd->dw6.roi_flag = 1 | (generic_state->brc_enabled << 1) |
2115                         (priv_state->video_surveillance_flag << 2);
2116
2117     for (i = 0; i < priv_state->num_roi; i++) {
2118         roi_par = &priv_state->roi[i];
2119         roi_size += abs(roi_par->right - roi_par->left) *
2120                     abs(roi_par->bottom - roi_par->top) * 256;
2121     }
2122
2123     if (roi_size)
2124         roi_ratio = MIN(2 * (priv_state->width_in_mb * priv_state->height_in_mb * 256 / roi_size - 1),
2125                         51);
2126
2127     cmd->dw6.roi_ratio = roi_ratio;
2128     cmd->dw7.frame_width_in_lcu = priv_state->frame_width_in_max_lcu;
2129
2130     if (!generic_state->brc_enabled) {
2131         VAEncPictureParameterBufferHEVC *pic_param = NULL;
2132         VAEncSliceParameterBufferHEVC *slice_param = NULL;
2133
2134         pic_param = (VAEncPictureParameterBufferHEVC *)encode_state->pic_param_ext->buffer;
2135         slice_param = (VAEncSliceParameterBufferHEVC *)encode_state->slice_params_ext[0]->buffer;
2136
2137         cmd->dw1.frame_number = priv_state->frame_number;
2138         cmd->dw6.cqp_value = pic_param->pic_init_qp + slice_param->slice_qp_delta;
2139         cmd->dw5.curr_frame_type = gen9_hevc_get_brc_frame_type(priv_state->picture_coding_type,
2140                                                                 priv_state->low_delay);
2141     }
2142 }
2143
2144 static void
2145 gen9_hevc_brc_update_lcu_based_set_roi_parameters(VADriverContextP ctx,
2146                                                   struct encode_state *encode_state,
2147                                                   struct intel_encoder_context *encoder_context,
2148                                                   struct i965_gpe_context *gpe_context)
2149 {
2150     struct encoder_vme_mfc_context *vme_context = NULL;
2151     struct gen9_hevc_encoder_context *priv_ctx = NULL;
2152     struct generic_enc_codec_state *generic_state = NULL;
2153     struct gen9_hevc_encoder_state *priv_state = NULL;
2154     struct intel_roi *roi_par = NULL;
2155     unsigned int width_in_mb_aligned = 0;
2156     unsigned int roi_level, qp_delta;
2157     unsigned int mb_num = 0;
2158     unsigned int *pdata = NULL;
2159     unsigned int out_data = 0;
2160     int i = 0;
2161
2162     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
2163     priv_ctx = (struct gen9_hevc_encoder_context *)vme_context->private_enc_ctx;
2164     generic_state = (struct generic_enc_codec_state *)vme_context->generic_enc_state;
2165     priv_state = (struct gen9_hevc_encoder_state *)vme_context->private_enc_state;
2166
2167     width_in_mb_aligned = ALIGN(priv_state->width_in_mb * 4, 64);
2168     mb_num = priv_state->width_in_mb * priv_state->height_in_mb;
2169
2170     pdata = i965_map_gpe_resource(&priv_ctx->res_roi_buffer);
2171     if (!pdata)
2172         return;
2173
2174     for (i = 0 ; i < mb_num; i++) {
2175         int cur_mb_y = i / priv_state->width_in_mb;
2176         int cur_mb_x = i - cur_mb_y * priv_state->width_in_mb;
2177         int roi_idx = 0;
2178
2179         out_data = 0;
2180
2181         for (roi_idx = (priv_state->num_roi - 1); roi_idx >= 0; roi_idx--) {
2182             roi_par = &priv_state->roi[roi_idx];
2183
2184             roi_level = qp_delta = 0;
2185             if (generic_state->brc_enabled && !priv_state->roi_value_is_qp_delta)
2186                 roi_level = roi_par->value * 5;
2187             else
2188                 qp_delta = roi_par->value;
2189
2190             if (roi_level == 0 && qp_delta == 0)
2191                 continue;
2192
2193             if ((cur_mb_x >= roi_par->left) &&
2194                 (cur_mb_x < roi_par->right) &&
2195                 (cur_mb_y >= roi_par->top) &&
2196                 (cur_mb_y < roi_par->bottom))
2197                 out_data = 15 | (((roi_level) & 0xFF) << 8) | ((qp_delta & 0xFF) << 16);
2198             else if ((cur_mb_x >= roi_par->left - 1) &&
2199                      (cur_mb_x < roi_par->right + 1) &&
2200                      (cur_mb_y >= roi_par->top - 1) &&
2201                      (cur_mb_y < roi_par->bottom + 1))
2202                 out_data = 14 | (((roi_level) & 0xFF) << 8) | ((qp_delta & 0xFF) << 16);
2203             else if ((cur_mb_x >= roi_par->left - 2) &&
2204                      (cur_mb_x < roi_par->right + 2) &&
2205                      (cur_mb_y >= roi_par->top - 2) &&
2206                      (cur_mb_y < roi_par->bottom + 2))
2207                 out_data = 13 | (((roi_level) & 0xFF) << 8) | ((qp_delta & 0xFF) << 16);
2208             else if ((cur_mb_x >= roi_par->left - 3) &&
2209                      (cur_mb_x < roi_par->right + 3) &&
2210                      (cur_mb_y >= roi_par->top - 3) &&
2211                      (cur_mb_y < roi_par->bottom + 3)) {
2212                 out_data = 12 | (((roi_level) & 0xFF) << 8) | ((qp_delta & 0xFF) << 16);
2213             }
2214         }
2215
2216         pdata[(cur_mb_y * (width_in_mb_aligned >> 2)) + cur_mb_x] = out_data;
2217     }
2218
2219     i965_unmap_gpe_resource(&priv_ctx->res_roi_buffer);
2220 }
2221
2222 static void
2223 gen9_hevc_brc_update_lcu_based_set_curbe(VADriverContextP ctx,
2224                                          struct encode_state *encode_state,
2225                                          struct intel_encoder_context *encoder_context,
2226                                          struct i965_gpe_context *gpe_context)
2227 {
2228     struct encoder_vme_mfc_context *vme_context = NULL;
2229     struct gen9_hevc_encoder_context *priv_ctx = NULL;
2230     struct generic_enc_codec_state *generic_state = NULL;
2231     struct gen9_hevc_encoder_state *priv_state = NULL;
2232     gen9_hevc_brc_udpate_curbe_data *cmd = NULL, *frame_cmd = NULL;
2233
2234     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
2235     priv_ctx = (struct gen9_hevc_encoder_context *)vme_context->private_enc_ctx;
2236     generic_state = (struct generic_enc_codec_state *)vme_context->generic_enc_state;
2237     priv_state = (struct gen9_hevc_encoder_state *)vme_context->private_enc_state;
2238
2239     cmd = i965_gpe_context_map_curbe(gpe_context);
2240     if (!cmd)
2241         return;
2242
2243     if (generic_state->brc_enabled) {
2244         frame_cmd = i965_gpe_context_map_curbe(&priv_ctx->brc_context.gpe_contexts[HEVC_BRC_FRAME_UPDATE_IDX]);
2245
2246         if (!frame_cmd)
2247             return;
2248
2249         memcpy((void *)cmd, (void *)frame_cmd, sizeof(*cmd));
2250
2251         i965_gpe_context_unmap_curbe(&priv_ctx->brc_context.gpe_contexts[HEVC_BRC_FRAME_UPDATE_IDX]);
2252     } else {
2253         memcpy((void *)cmd, GEN9_HEVC_BRCUPDATE_CURBE_DATA,
2254                sizeof(GEN9_HEVC_BRCUPDATE_CURBE_DATA));
2255     }
2256
2257     if (priv_state->num_roi)
2258         gen9_hevc_brc_update_set_roi_curbe(ctx, encode_state, encoder_context, cmd);
2259
2260     i965_gpe_context_unmap_curbe(gpe_context);
2261 }
2262
2263 static void
2264 gen9_hevc_brc_update_lcu_based_set_surfaces(VADriverContextP ctx,
2265                                             struct encode_state *encode_state,
2266                                             struct intel_encoder_context *encoder_context,
2267                                             struct i965_gpe_context *gpe_context)
2268 {
2269     struct encoder_vme_mfc_context *vme_context = NULL;
2270     struct gen9_hevc_encoder_context *priv_ctx = NULL;
2271     struct gen9_hevc_encoder_state *priv_state = NULL;
2272     int bti_idx = 0;
2273
2274     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
2275     priv_ctx = (struct gen9_hevc_encoder_context *)vme_context->private_enc_ctx;
2276     priv_state = (struct gen9_hevc_encoder_state *)vme_context->private_enc_state;
2277
2278     gen9_hevc_set_gpe_1d_surface(ctx, priv_ctx, gpe_context,
2279                                  HEVC_ENC_SURFACE_BRC_HISTORY, bti_idx++,
2280                                  0, 0, 0, NULL, NULL);
2281
2282     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
2283                                  HEVC_ENC_SURFACE_BRC_ME_DIST, bti_idx++,
2284                                  0, 1, I965_SURFACEFORMAT_R8_UNORM,
2285                                  priv_state->picture_coding_type == HEVC_SLICE_I ?
2286                                  &priv_ctx->res_brc_intra_dist_buffer :
2287                                  &priv_ctx->res_brc_me_dist_buffer,
2288                                  NULL);
2289
2290     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
2291                                  HEVC_ENC_SURFACE_BRC_ME_DIST, bti_idx++,
2292                                  0, 1, I965_SURFACEFORMAT_R8_UNORM,
2293                                  &priv_ctx->res_brc_intra_dist_buffer, NULL);
2294
2295     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
2296                                  HEVC_ENC_SURFACE_HME_MVP, bti_idx++,
2297                                  0, 1, I965_SURFACEFORMAT_R8_UNORM,
2298                                  NULL, NULL);
2299
2300     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
2301                                  HEVC_ENC_SURFACE_LCU_QP, bti_idx++,
2302                                  0, 1, I965_SURFACEFORMAT_R8_UNORM,
2303                                  NULL, NULL);
2304
2305     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
2306                                  HEVC_ENC_SURFACE_ROI, bti_idx++,
2307                                  0, 1, I965_SURFACEFORMAT_R8_UNORM,
2308                                  NULL, NULL);
2309 }
2310
2311 static void
2312 gen9_hevc_brc_update_lcu_based(VADriverContextP ctx,
2313                                struct encode_state *encode_state,
2314                                struct intel_encoder_context *encoder_context)
2315 {
2316     struct encoder_vme_mfc_context *vme_context = NULL;
2317     struct gen9_hevc_encoder_context *priv_ctx = NULL;
2318     struct gen9_hevc_encoder_state *priv_state = NULL;
2319     struct i965_gpe_context *gpe_context = NULL;
2320     struct gpe_media_object_walker_parameter param;
2321     struct hevc_enc_kernel_walker_parameter hevc_walker_param;
2322     int media_state = HEVC_ENC_MEDIA_STATE_HEVC_BRC_LCU_UPDATE;
2323     int gpe_idx = HEVC_BRC_LCU_UPDATE_IDX;
2324
2325     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
2326     priv_ctx = (struct gen9_hevc_encoder_context *)vme_context->private_enc_ctx;
2327     priv_state = (struct gen9_hevc_encoder_state *)vme_context->private_enc_state;
2328     gpe_context = &priv_ctx->brc_context.gpe_contexts[gpe_idx];
2329
2330     gen8_gpe_context_init(ctx, gpe_context);
2331     gen9_gpe_reset_binding_table(ctx, gpe_context);
2332
2333     if (priv_state->num_roi)
2334         gen9_hevc_brc_update_lcu_based_set_roi_parameters(ctx, encode_state, encoder_context,
2335                                                           gpe_context);
2336
2337     gen9_hevc_brc_update_lcu_based_set_curbe(ctx, encode_state, encoder_context, gpe_context);
2338     gen9_hevc_brc_update_lcu_based_set_surfaces(ctx, encode_state, encoder_context, gpe_context);
2339     gen8_gpe_setup_interface_data(ctx, gpe_context);
2340
2341     memset((void *)&hevc_walker_param, 0, sizeof(hevc_walker_param));
2342     hevc_walker_param.resolution_x = ALIGN(priv_state->picture_width, 128) >> 7;
2343     hevc_walker_param.resolution_y = ALIGN(priv_state->picture_height, 128) >> 7;
2344     hevc_walker_param.no_dependency = 1;
2345     gen9_hevc_init_object_walker(&hevc_walker_param, &param);
2346     gen9_hevc_run_object_walker(ctx, encoder_context, gpe_context, &param,
2347                                 media_state);
2348 }
2349
2350 static void
2351 gen9_hevc_add_pic_state(VADriverContextP ctx,
2352                         struct encode_state *encode_state,
2353                         struct intel_encoder_context *encoder_context,
2354                         struct i965_gpe_resource *pic_state_ptr,
2355                         int pic_state_offset,
2356                         int brc_update)
2357 {
2358     struct i965_driver_data *i965 = i965_driver_data(ctx);
2359     struct encoder_vme_mfc_context *pak_context = NULL;
2360     struct gen9_hevc_encoder_state *priv_state = NULL;
2361     VAEncPictureParameterBufferHEVC *pic_param = NULL;
2362     VAEncSequenceParameterBufferHEVC *seq_param = NULL;
2363     unsigned int tmp_data[31], *cmd_ptr = NULL;
2364     int cmd_size = 0;
2365
2366     pak_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
2367     priv_state = (struct gen9_hevc_encoder_state *)pak_context->private_enc_state;
2368
2369     pic_param = (VAEncPictureParameterBufferHEVC *)encode_state->pic_param_ext->buffer;
2370     seq_param = (VAEncSequenceParameterBufferHEVC *)encode_state->seq_param_ext->buffer;
2371
2372     cmd_ptr = tmp_data;
2373     cmd_size = (IS_KBL(i965->intel.device_info) || IS_GLK(i965->intel.device_info)) ? 31 : 19;
2374     memset((void *)tmp_data, 0, 4 * cmd_size);
2375
2376     if (IS_KBL(i965->intel.device_info) || IS_GLK(i965->intel.device_info))
2377         *cmd_ptr++ = HCP_PIC_STATE | (31 - 2);
2378     else
2379         *cmd_ptr++ = HCP_PIC_STATE | (19 - 2);
2380
2381     *cmd_ptr++ = (priv_state->height_in_cu - 1) << 16 |
2382                  0 << 14 |
2383                  (priv_state->width_in_cu - 1);
2384     *cmd_ptr++ = (seq_param->log2_min_transform_block_size_minus2 +
2385                   seq_param->log2_diff_max_min_transform_block_size) << 6 |
2386                  seq_param->log2_min_transform_block_size_minus2 << 4 |
2387                  (seq_param->log2_min_luma_coding_block_size_minus3 +
2388                   seq_param->log2_diff_max_min_luma_coding_block_size) << 2 |
2389                  seq_param->log2_min_luma_coding_block_size_minus3;
2390     *cmd_ptr++ = 0;
2391     *cmd_ptr++ = ((IS_KBL(i965->intel.device_info) || IS_GLK(i965->intel.device_info)) ? 1 : 0) << 27 |
2392                  seq_param->seq_fields.bits.strong_intra_smoothing_enabled_flag << 26 |
2393                  pic_param->pic_fields.bits.transquant_bypass_enabled_flag << 25 |
2394                  ((IS_KBL(i965->intel.device_info) || IS_GLK(i965->intel.device_info)) ? 0 : priv_state->ctu_max_bitsize_allowed > 0) << 24 |
2395                  seq_param->seq_fields.bits.amp_enabled_flag << 23 |
2396                  pic_param->pic_fields.bits.transform_skip_enabled_flag << 22 |
2397                  0 << 21 |
2398                  0 << 20 |
2399                  pic_param->pic_fields.bits.weighted_pred_flag << 19 |
2400                  pic_param->pic_fields.bits.weighted_bipred_flag << 18 |
2401                  0 << 17 |
2402                  pic_param->pic_fields.bits.entropy_coding_sync_enabled_flag << 16 |
2403                  0 << 15 |
2404                  pic_param->pic_fields.bits.sign_data_hiding_enabled_flag << 13 |
2405                  pic_param->log2_parallel_merge_level_minus2 << 10 |
2406                  pic_param->pic_fields.bits.constrained_intra_pred_flag << 9 |
2407                  seq_param->seq_fields.bits.pcm_loop_filter_disabled_flag << 8 |
2408                  (pic_param->diff_cu_qp_delta_depth & 0x03) << 6 |
2409                  pic_param->pic_fields.bits.cu_qp_delta_enabled_flag << 5 |
2410                  0 << 4 |
2411                  seq_param->seq_fields.bits.sample_adaptive_offset_enabled_flag << 3 |
2412                  0;
2413     *cmd_ptr++ = seq_param->seq_fields.bits.bit_depth_luma_minus8 << 27 |
2414                  seq_param->seq_fields.bits.bit_depth_chroma_minus8 << 24 |
2415                  ((IS_KBL(i965->intel.device_info) || IS_GLK(i965->intel.device_info)) ? 0 : 7) << 20 |
2416                  ((IS_KBL(i965->intel.device_info) || IS_GLK(i965->intel.device_info)) ? 0 : 7) << 16 |
2417                  seq_param->max_transform_hierarchy_depth_inter << 13 |
2418                  seq_param->max_transform_hierarchy_depth_intra << 10 |
2419                  (pic_param->pps_cr_qp_offset & 0x1f) << 5 |
2420                  (pic_param->pps_cb_qp_offset & 0x1f);
2421
2422     *cmd_ptr++ = 0 << 29 |
2423                  priv_state->ctu_max_bitsize_allowed;
2424     if (brc_update)
2425         *(cmd_ptr - 1) |= 0 << 31 |
2426                           1 << 26 |
2427                           1 << 25 |
2428                           0 << 24 |
2429                           (pic_state_offset ? 1 : 0) << 16;
2430
2431     *cmd_ptr++ = 0 << 31 |
2432                  0;
2433     *cmd_ptr++ = 0 << 31 |
2434                  0;
2435     *cmd_ptr++ = 0 << 16 |
2436                  0;
2437     *cmd_ptr++ = 0;
2438     *cmd_ptr++ = 0;
2439     *cmd_ptr++ = 0;
2440     *cmd_ptr++ = 0;
2441     *cmd_ptr++ = 0;
2442     *cmd_ptr++ = 0;
2443     *cmd_ptr++ = 0;
2444     *cmd_ptr++ = 0;
2445     *cmd_ptr++ = 0 << 30 |
2446                  0;
2447
2448     if (IS_KBL(i965->intel.device_info) || IS_GLK(i965->intel.device_info)) {
2449         int i = 0;
2450
2451         for (i = 0; i < 12; i++)
2452             *cmd_ptr++ = 0;
2453     }
2454
2455     if (pic_state_ptr) {
2456         char *pdata = i965_map_gpe_resource(pic_state_ptr);
2457
2458         if (!pdata)
2459             return;
2460
2461         memcpy(pdata + pic_state_offset, tmp_data, cmd_size * 4);
2462
2463         pdata += pic_state_offset + cmd_size * 4;
2464
2465         *(unsigned int *)pdata++ = MI_BATCH_BUFFER_END;
2466
2467         i965_unmap_gpe_resource(pic_state_ptr);
2468     } else {
2469         struct intel_batchbuffer *batch = encoder_context->base.batch;
2470
2471         BEGIN_BCS_BATCH(batch, cmd_size);
2472
2473         intel_batchbuffer_data(batch, tmp_data, cmd_size * 4);
2474
2475         ADVANCE_BCS_BATCH(batch);
2476     }
2477 }
2478
2479 static void
2480 gen9_hevc_brc_update_set_pic_states(VADriverContextP ctx,
2481                                     struct encode_state *encode_state,
2482                                     struct intel_encoder_context *encoder_context)
2483 {
2484     struct encoder_vme_mfc_context *vme_context = NULL;
2485     struct gen9_hevc_encoder_context *priv_ctx = NULL;
2486     struct generic_enc_codec_state *generic_state = NULL;
2487     struct gen9_hevc_encoder_state *priv_state = NULL;
2488     int i = 0, offset = 0;
2489
2490     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
2491     priv_ctx = (struct gen9_hevc_encoder_context *)vme_context->private_enc_ctx;
2492     generic_state = (struct generic_enc_codec_state *)vme_context->generic_enc_state;
2493     priv_state = (struct gen9_hevc_encoder_state *)vme_context->private_enc_state;
2494
2495     for (i = 0; i < generic_state->num_pak_passes; i++) {
2496         gen9_hevc_add_pic_state(ctx, encode_state, encoder_context,
2497                                 &priv_ctx->res_brc_pic_states_read_buffer,
2498                                 offset, 1);
2499
2500         offset += priv_state->pic_state_size;
2501     }
2502 }
2503
2504 static void
2505 gen9_hevc_brc_update_set_constant(VADriverContextP ctx,
2506                                   struct encode_state *encode_state,
2507                                   struct intel_encoder_context *encoder_context)
2508 {
2509     struct encoder_vme_mfc_context *vme_context = NULL;
2510     struct gen9_hevc_encoder_context *priv_ctx = NULL;
2511     struct gen9_hevc_encoder_state *priv_state = NULL;
2512     unsigned int width, height, size;
2513     unsigned char *pdata = NULL;
2514     int idx = 0;
2515
2516     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
2517     priv_ctx = (struct gen9_hevc_encoder_context *)vme_context->private_enc_ctx;
2518     priv_state = (struct gen9_hevc_encoder_state *)vme_context->private_enc_state;
2519
2520     pdata = i965_map_gpe_resource(&priv_ctx->res_brc_constant_data_buffer);
2521     if (!pdata)
2522         return;
2523
2524     width = ALIGN(GEN9_HEVC_ENC_BRC_CONSTANT_SURFACE_WIDTH, 64);
2525     height = GEN9_HEVC_ENC_BRC_CONSTANT_SURFACE_HEIGHT;
2526     size = width * height;
2527     memset((void *)pdata, 0, size);
2528
2529     memcpy((void *)pdata, GEN9_HEVC_BRCUPDATE_QP_ADJUST, GEN9_HEVC_BRCUPDATE_QP_ADJUST_SIZE);
2530     pdata += GEN9_HEVC_BRCUPDATE_QP_ADJUST_SIZE;
2531
2532     if (priv_state->picture_coding_type == HEVC_SLICE_I)
2533         memset((void *)pdata, 0, GEN9_HEVC_ENC_SKIP_VAL_SIZE);
2534     else {
2535         gen9_hevc_mbenc_b_mb_enc_curbe_data *curbe_cmd = NULL;
2536         int curbe_size = 0;
2537
2538         gen9_hevc_get_b_mbenc_default_curbe(priv_state->tu_mode,
2539                                             priv_state->picture_coding_type,
2540                                             (void **)&curbe_cmd,
2541                                             &curbe_size);
2542
2543         idx = curbe_cmd->dw3.block_based_skip_enable ? 1 : 0;
2544         memcpy((void *)pdata, GEN9_HEVC_ENC_SKIP_THREAD[idx], sizeof(GEN9_HEVC_ENC_SKIP_THREAD[idx]));
2545     }
2546     pdata += GEN9_HEVC_ENC_SKIP_VAL_SIZE;
2547
2548     memcpy((void *)pdata, GEN9_HEVC_ENC_BRC_LAMBDA_HAAR, sizeof(GEN9_HEVC_ENC_BRC_LAMBDA_HAAR));
2549     pdata += GEN9_HEVC_ENC_BRC_LAMBDA_TABLE_SIZE;
2550
2551     idx = (priv_state->picture_coding_type == HEVC_SLICE_I) ? 0 :
2552           (priv_state->picture_coding_type == HEVC_SLICE_P) ? 1 : 2;
2553     memcpy((void *)pdata, GEN9_HEVC_ENC_BRC_MVCOST_HAAR[idx], sizeof(GEN9_HEVC_ENC_BRC_MVCOST_HAAR[idx]));
2554
2555     i965_unmap_gpe_resource(&priv_ctx->res_brc_constant_data_buffer);
2556 }
2557
2558 static
2559 unsigned int gen9_hevc_get_start_code_offset(unsigned char *ptr,
2560                                              unsigned int size)
2561 {
2562     unsigned int count = 0;
2563
2564     while (count < size && *ptr != 0x01) {
2565         if (*ptr != 0)
2566             break;
2567
2568         count++;
2569         ptr++;
2570     }
2571
2572     return count + 1;
2573 }
2574
2575 static
2576 unsigned int gen9_hevc_get_emulation_num(unsigned char *ptr,
2577                                          unsigned int size)
2578 {
2579     unsigned int emulation_num = 0;
2580     unsigned int header_offset = 0;
2581     unsigned int zero_count = 0;
2582     int i = 0;
2583
2584     header_offset = gen9_hevc_get_start_code_offset(ptr, size);
2585     ptr += header_offset;
2586
2587     for (i = 0 ; i < (size - header_offset); i++, ptr++) {
2588         if (zero_count == 2 && !(*ptr & 0xFC)) {
2589             zero_count = 0;
2590             emulation_num++;
2591         }
2592
2593         if (*ptr == 0x00)
2594             zero_count++;
2595         else
2596             zero_count = 0;
2597     }
2598
2599     return emulation_num;
2600 }
2601
2602 #define HEVC_ENC_START_CODE_NAL_OFFSET                  (2)
2603
2604 static unsigned int
2605 gen9_hevc_get_pic_header_size(struct encode_state *encode_state)
2606 {
2607     VAEncPackedHeaderParameterBuffer *param = NULL;
2608     unsigned int header_begin = 0;
2609     unsigned int accum_size = 0;
2610     unsigned char *header_data = NULL;
2611     unsigned int length_in_bytes = 0;
2612     int packed_type = 0;
2613     int idx = 0, count = 0, idx_offset = 0;
2614     int i = 0, slice_idx = 0, start_index = 0;
2615
2616     for (i = 0; i < 4; i++) {
2617         idx_offset = 0;
2618         switch (i) {
2619         case 0:
2620             packed_type = VAEncPackedHeaderHEVC_VPS;
2621             break;
2622         case 1:
2623             packed_type = VAEncPackedHeaderHEVC_VPS;
2624             idx_offset = 1;
2625             break;
2626         case 2:
2627             packed_type = VAEncPackedHeaderHEVC_PPS;
2628             break;
2629         case 3:
2630             packed_type = VAEncPackedHeaderHEVC_SEI;
2631             break;
2632         default:
2633             break;
2634         }
2635
2636         idx = va_enc_packed_type_to_idx(packed_type) + idx_offset;
2637         if (encode_state->packed_header_data[idx]) {
2638             param = (VAEncPackedHeaderParameterBuffer *)encode_state->packed_header_param[idx]->buffer;
2639             header_data = (unsigned char *)encode_state->packed_header_data[idx]->buffer;
2640             length_in_bytes = (param->bit_length + 7) / 8;
2641
2642             header_begin = gen9_hevc_get_start_code_offset(header_data, length_in_bytes) +
2643                            HEVC_ENC_START_CODE_NAL_OFFSET;
2644
2645             accum_size += length_in_bytes;
2646             if (!param->has_emulation_bytes)
2647                 accum_size += gen9_hevc_get_emulation_num(header_data,
2648                                                           length_in_bytes);
2649         }
2650     }
2651
2652     for (slice_idx = 0; slice_idx < encode_state->num_slice_params_ext; slice_idx++) {
2653         count = encode_state->slice_rawdata_count[slice_idx];
2654         start_index = encode_state->slice_rawdata_index[slice_idx] &
2655                       SLICE_PACKED_DATA_INDEX_MASK;
2656
2657         for (i = 0; i < count; i++) {
2658             param = (VAEncPackedHeaderParameterBuffer *)
2659                     (encode_state->packed_header_params_ext[start_index + i]->buffer);
2660
2661             if (param->type == VAEncPackedHeaderSlice)
2662                 continue;
2663
2664             header_data = (unsigned char *)encode_state->packed_header_data[start_index]->buffer;
2665             length_in_bytes = (param->bit_length + 7) / 8;
2666
2667             accum_size += length_in_bytes;
2668             if (!param->has_emulation_bytes)
2669                 accum_size += gen9_hevc_get_emulation_num(header_data,
2670                                                           length_in_bytes);
2671         }
2672     }
2673
2674     header_begin = MIN(header_begin, accum_size);
2675
2676     return ((accum_size - header_begin) * 8);
2677 }
2678
2679 static void
2680 gen9_hevc_brc_update_set_curbe(VADriverContextP ctx,
2681                                struct encode_state *encode_state,
2682                                struct intel_encoder_context *encoder_context,
2683                                struct i965_gpe_context *gpe_context)
2684 {
2685     struct encoder_vme_mfc_context *vme_context = NULL;
2686     struct generic_enc_codec_state *generic_state = NULL;
2687     struct gen9_hevc_encoder_state *priv_state = NULL;
2688     gen9_hevc_brc_udpate_curbe_data *cmd = NULL;
2689
2690     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
2691     generic_state = (struct generic_enc_codec_state *)vme_context->generic_enc_state;
2692     priv_state = (struct gen9_hevc_encoder_state *)vme_context->private_enc_state;
2693
2694     cmd = i965_gpe_context_map_curbe(gpe_context);
2695     if (!cmd)
2696         return;
2697
2698     memcpy((void *)cmd, GEN9_HEVC_BRCUPDATE_CURBE_DATA,
2699            sizeof(GEN9_HEVC_BRCUPDATE_CURBE_DATA));
2700
2701     cmd->dw5.target_size_flag = 0;
2702     if (priv_state->brc_init_current_target_buf_full_in_bits >
2703         (double)priv_state->brc_init_reset_buf_size_in_bits) {
2704         priv_state->brc_init_current_target_buf_full_in_bits -=
2705             (double)priv_state->brc_init_reset_buf_size_in_bits;
2706         cmd->dw5.target_size_flag = 1;
2707     }
2708
2709     if (priv_state->num_skip_frames) {
2710         cmd->dw6.num_skipped_frames = priv_state->num_skip_frames;
2711         cmd->dw15.size_of_skipped_frames = priv_state->size_skip_frames;
2712
2713         priv_state->brc_init_current_target_buf_full_in_bits +=
2714             priv_state->brc_init_reset_input_bits_per_frame * priv_state->num_skip_frames;
2715     }
2716
2717     cmd->dw0.target_size = (unsigned int)priv_state->brc_init_current_target_buf_full_in_bits;
2718     cmd->dw1.frame_number = priv_state->frame_number;
2719     cmd->dw2.picture_header_size = gen9_hevc_get_pic_header_size(encode_state);
2720
2721     cmd->dw5.brc_flag = 0;
2722     cmd->dw5.curr_frame_type = gen9_hevc_get_brc_frame_type(priv_state->picture_coding_type,
2723                                                             priv_state->low_delay);
2724
2725     cmd->dw5.max_num_paks = generic_state->num_pak_passes;
2726     cmd->dw14.parallel_mode = priv_state->parallel_brc;
2727
2728     priv_state->brc_init_current_target_buf_full_in_bits +=
2729         priv_state->brc_init_reset_input_bits_per_frame;
2730
2731     if (priv_state->brc_method == HEVC_BRC_AVBR) {
2732         cmd->dw3.start_gadj_frame0 = (unsigned int)((10 * GEN9_HEVC_AVBR_CONVERGENCE) / (double)150);
2733         cmd->dw3.start_gadj_frame1 = (unsigned int)((50 * GEN9_HEVC_AVBR_CONVERGENCE) / (double)150);
2734         cmd->dw4.start_gadj_frame2 = (unsigned int)((100 * GEN9_HEVC_AVBR_CONVERGENCE) / (double)150);
2735         cmd->dw4.start_gadj_frame3 = (unsigned int)((150 * GEN9_HEVC_AVBR_CONVERGENCE) / (double)150);
2736         cmd->dw11.g_rate_ratio_threshold0 = (unsigned int)((100 - (GEN9_HEVC_AVBR_ACCURACY / (double)30) * (100 - 40)));
2737         cmd->dw11.g_rate_ratio_threshold1 = (unsigned int)((100 - (GEN9_HEVC_AVBR_ACCURACY / (double)30) * (100 - 75)));
2738         cmd->dw12.g_rate_ratio_threshold2 = (unsigned int)((100 - (GEN9_HEVC_AVBR_ACCURACY / (double)30) * (100 - 97)));
2739         cmd->dw12.g_rate_ratio_threshold3 = (unsigned int)((100 + (GEN9_HEVC_AVBR_ACCURACY / (double)30) * (103 - 100)));
2740         cmd->dw12.g_rate_ratio_threshold4 = (unsigned int)((100 + (GEN9_HEVC_AVBR_ACCURACY / (double)30) * (125 - 100)));
2741         cmd->dw12.g_rate_ratio_threshold5 = (unsigned int)((100 + (GEN9_HEVC_AVBR_ACCURACY / (double)30) * (160 - 100)));
2742     } else {
2743         cmd->dw3.start_gadj_frame0 = 10;
2744         cmd->dw3.start_gadj_frame1 = 50;
2745         cmd->dw4.start_gadj_frame2 = 100;
2746         cmd->dw4.start_gadj_frame3 = 150;
2747         cmd->dw11.g_rate_ratio_threshold0 = 40;
2748         cmd->dw11.g_rate_ratio_threshold1 = 75;
2749         cmd->dw12.g_rate_ratio_threshold2 = 97;
2750         cmd->dw12.g_rate_ratio_threshold3 = 103;
2751         cmd->dw12.g_rate_ratio_threshold4 = 125;
2752         cmd->dw12.g_rate_ratio_threshold5 = 160;
2753     }
2754
2755     i965_gpe_context_unmap_curbe(gpe_context);
2756 }
2757
2758 static void
2759 gen9_hevc_brc_update_set_surfaces(VADriverContextP ctx,
2760                                   struct intel_encoder_context *encoder_context,
2761                                   struct i965_gpe_context *gpe_context)
2762 {
2763     struct encoder_vme_mfc_context *vme_context = NULL;
2764     struct gen9_hevc_encoder_context *priv_ctx = NULL;
2765     struct gen9_hevc_encoder_state *priv_state = NULL;
2766     int bti_idx = 0;
2767
2768     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
2769     priv_ctx = (struct gen9_hevc_encoder_context *)vme_context->private_enc_ctx;
2770     priv_state = (struct gen9_hevc_encoder_state *)vme_context->private_enc_state;
2771
2772     gen9_hevc_set_gpe_1d_surface(ctx, priv_ctx, gpe_context,
2773                                  HEVC_ENC_SURFACE_BRC_HISTORY, bti_idx++,
2774                                  0, 0, 0, NULL, NULL);
2775
2776     gen9_hevc_set_gpe_1d_surface(ctx, priv_ctx, gpe_context,
2777                                  HEVC_ENC_SURFACE_BRC_PAST_PAK_INFO, bti_idx++,
2778                                  0, 0, 0, NULL, NULL);
2779
2780     gen9_hevc_set_gpe_1d_surface(ctx, priv_ctx, gpe_context,
2781                                  HEVC_ENC_SURFACE_BRC_HCP_PIC_STATE, bti_idx++,
2782                                  0, 0, 0, &priv_ctx->res_brc_pic_states_read_buffer,
2783                                  NULL);
2784
2785     gen9_hevc_set_gpe_1d_surface(ctx, priv_ctx, gpe_context,
2786                                  HEVC_ENC_SURFACE_BRC_HCP_PIC_STATE, bti_idx++,
2787                                  0, 0, 0, &priv_ctx->res_brc_pic_states_write_buffer,
2788                                  NULL);
2789
2790     gen9_hevc_set_gpe_1d_surface(ctx, priv_ctx, gpe_context,
2791                                  HEVC_ENC_SURFACE_BRC_INPUT, bti_idx++,
2792                                  0, 0, 0, NULL, NULL);
2793
2794     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
2795                                  HEVC_ENC_SURFACE_BRC_ME_DIST, bti_idx++,
2796                                  0, 1, I965_SURFACEFORMAT_R8_UNORM,
2797                                  priv_state->picture_coding_type == HEVC_SLICE_I ?
2798                                  &priv_ctx->res_brc_intra_dist_buffer :
2799                                  &priv_ctx->res_brc_me_dist_buffer,
2800                                  NULL);
2801
2802     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
2803                                  HEVC_ENC_SURFACE_BRC_DATA, bti_idx++,
2804                                  0, 1, I965_SURFACEFORMAT_R8_UNORM,
2805                                  NULL, NULL);
2806 }
2807
2808 static void
2809 gen9_hevc_brc_update(VADriverContextP ctx,
2810                      struct encode_state *encode_state,
2811                      struct intel_encoder_context *encoder_context)
2812 {
2813     struct encoder_vme_mfc_context *vme_context = NULL;
2814     struct gen9_hevc_encoder_context *priv_ctx = NULL;
2815     struct gen9_hevc_encoder_state *priv_state = NULL;
2816     struct i965_gpe_context *gpe_context = NULL;
2817     struct gpe_media_object_parameter param;
2818     int media_state = HEVC_ENC_MEDIA_STATE_BRC_UPDATE;
2819     int gpe_idx = HEVC_BRC_FRAME_UPDATE_IDX;
2820
2821     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
2822     priv_ctx = (struct gen9_hevc_encoder_context *)vme_context->private_enc_ctx;
2823     priv_state = (struct gen9_hevc_encoder_state *)vme_context->private_enc_state;
2824     gpe_context = &priv_ctx->brc_context.gpe_contexts[gpe_idx];
2825
2826     gen8_gpe_context_init(ctx, gpe_context);
2827     gen9_gpe_reset_binding_table(ctx, gpe_context);
2828
2829     gen9_hevc_brc_update_set_pic_states(ctx, encode_state, encoder_context);
2830     gen9_hevc_brc_update_set_constant(ctx, encode_state, encoder_context);
2831     gen9_hevc_brc_update_set_curbe(ctx, encode_state, encoder_context, gpe_context);
2832     gen9_hevc_brc_update_set_surfaces(ctx, encoder_context, gpe_context);
2833
2834     gen8_gpe_setup_interface_data(ctx, gpe_context);
2835
2836     memset(&param, 0, sizeof(param));
2837     gen9_hevc_run_object(ctx, encoder_context, gpe_context, &param,
2838                          media_state);
2839
2840     if (priv_state->lcu_brc_enabled ||
2841         priv_state->num_roi)
2842         gen9_hevc_brc_update_lcu_based(ctx, encode_state, encoder_context);
2843 }
2844
2845 // Depth converstion for 10bits
2846
2847 static void
2848 gen9_hevc_frame_depth_conversion_set_curbe(VADriverContextP ctx,
2849                                            struct encode_state *encode_state,
2850                                            struct intel_encoder_context *encoder_context,
2851                                            struct i965_gpe_context *gpe_context,
2852                                            GEN9_HEVC_DOWNSCALE_STAGE scale_stage)
2853 {
2854     struct encoder_vme_mfc_context *vme_context = NULL;
2855     struct gen9_hevc_encoder_state *priv_state = NULL;
2856     gen95_hevc_mbenc_ds_combined_curbe_data *cmd = NULL;
2857     int bti_idx = 0;
2858
2859     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
2860     priv_state = (struct gen9_hevc_encoder_state *)vme_context->private_enc_state;
2861
2862     cmd = i965_gpe_context_map_curbe(gpe_context);
2863     if (!cmd)
2864         return;
2865
2866     memset((void *)cmd, 0, sizeof(*cmd));
2867
2868     cmd->dw0.pak_bitdepth_chroma = 10;
2869     cmd->dw0.pak_bitdepth_luma = 10;
2870     cmd->dw0.enc_bitdepth_chroma = 8;
2871     cmd->dw0.enc_bitdepth_luma = 8;
2872     cmd->dw0.rounding_value = 1;
2873     cmd->dw1.pic_format = 0;
2874     cmd->dw1.pic_convert_flag = 1;
2875     cmd->dw1.pic_down_scale = scale_stage;
2876     cmd->dw1.pic_mb_stat_output_cntrl = 0;
2877     cmd->dw2.orig_pic_width = priv_state->picture_width;
2878     cmd->dw2.orig_pic_height = priv_state->picture_height;
2879
2880     cmd->dw3.bti_surface_p010 = bti_idx++;
2881     bti_idx++;
2882     cmd->dw4.bti_surface_nv12 = bti_idx++;
2883     bti_idx++;
2884     cmd->dw5.bti_src_y_4xdownscaled = bti_idx++;
2885     cmd->dw6.bti_surf_mbstate = bti_idx++;
2886     cmd->dw7.bit_src_y_2xdownscaled = bti_idx++;
2887
2888     i965_gpe_context_unmap_curbe(gpe_context);
2889 }
2890
2891 static void
2892 gen9_hevc_frame_depth_conversion_set_surfaces(VADriverContextP ctx,
2893                                               struct encode_state *encode_state,
2894                                               struct intel_encoder_context *encoder_context,
2895                                               struct i965_gpe_context *gpe_context,
2896                                               struct object_surface *src_surface,
2897                                               struct object_surface *dst_surface)
2898 {
2899     struct encoder_vme_mfc_context *vme_context = NULL;
2900     struct gen9_hevc_encoder_context *priv_ctx = NULL;
2901     struct gen9_hevc_surface_priv *surface_priv = NULL;
2902     int bti_idx = 0;
2903
2904     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
2905     priv_ctx = (struct gen9_hevc_encoder_context *)vme_context->private_enc_ctx;
2906     surface_priv = (struct gen9_hevc_surface_priv *)dst_surface->private_data;
2907
2908     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
2909                                  HEVC_ENC_SURFACE_RAW_10bit_Y_UV, bti_idx++,
2910                                  1, 1, I965_SURFACEFORMAT_R8_UNORM,
2911                                  NULL, src_surface);
2912     bti_idx++;
2913
2914     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
2915                                  HEVC_ENC_SURFACE_RAW_FC_8bit_Y_UV, bti_idx++,
2916                                  1, 1, I965_SURFACEFORMAT_R8_UNORM,
2917                                  NULL,
2918                                  surface_priv->surface_obj_nv12);
2919     bti_idx++;
2920
2921     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
2922                                  HEVC_ENC_SURFACE_Y_4X, bti_idx++,
2923                                  0, 1, I965_SURFACEFORMAT_R32_UNORM,
2924                                  NULL,
2925                                  surface_priv->scaled_surface_obj[HEVC_SCALED_SURF_4X_ID]);
2926
2927     gen9_hevc_set_gpe_1d_surface(ctx, priv_ctx, gpe_context,
2928                                  HEVC_ENC_SURFACE_RAW_MBSTAT, bti_idx++,
2929                                  0, 0, 0, NULL, NULL);
2930
2931     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
2932                                  HEVC_ENC_SURFACE_Y_2X, bti_idx++,
2933                                  0, 1, I965_SURFACEFORMAT_R32_UNORM,
2934                                  NULL, NULL);
2935 }
2936
2937 static void
2938 gen9_hevc_frame_depth_conversion(VADriverContextP ctx,
2939                                  struct encode_state *encode_state,
2940                                  struct intel_encoder_context *encoder_context,
2941                                  struct object_surface *src_surface,
2942                                  struct object_surface *dst_surface,
2943                                  GEN9_HEVC_DOWNSCALE_STAGE scale_stage)
2944 {
2945     struct encoder_vme_mfc_context *vme_context = NULL;
2946     struct gen9_hevc_encoder_context *priv_ctx = NULL;
2947     struct gen9_hevc_encoder_state *priv_state = NULL;
2948     struct gpe_media_object_walker_parameter param;
2949     struct hevc_enc_kernel_walker_parameter hevc_walker_param;
2950     struct i965_gpe_context *gpe_context = NULL;
2951     int media_state = HEVC_ENC_MEDIA_STATE_DS_COMBINED;
2952
2953     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
2954     priv_ctx = (struct gen9_hevc_encoder_context *)vme_context->private_enc_ctx;
2955     priv_state = (struct gen9_hevc_encoder_state *)vme_context->private_enc_state;
2956
2957     gpe_context = &priv_ctx->mbenc_context.gpe_contexts[HEVC_MBENC_DS_COMBINED_IDX];
2958
2959     gen8_gpe_context_init(ctx, gpe_context);
2960     gen9_gpe_reset_binding_table(ctx, gpe_context);
2961     gen9_hevc_frame_depth_conversion_set_curbe(ctx, encode_state, encoder_context, gpe_context,
2962                                                scale_stage);
2963     gen9_hevc_frame_depth_conversion_set_surfaces(ctx, encode_state, encoder_context, gpe_context,
2964                                                   src_surface, dst_surface);
2965     gen8_gpe_setup_interface_data(ctx, gpe_context);
2966
2967     memset((void *)&hevc_walker_param, 0, sizeof(hevc_walker_param));
2968     hevc_walker_param.resolution_x = ALIGN(priv_state->picture_width >> 2, 32) >> 3;
2969     hevc_walker_param.resolution_y = ALIGN(priv_state->picture_height >> 2, 32) >> 3;
2970     hevc_walker_param.no_dependency = 1;
2971     gen9_hevc_init_object_walker(&hevc_walker_param, &param);
2972     gen9_hevc_run_object_walker(ctx, encoder_context, gpe_context, &param,
2973                                 media_state);
2974 }
2975
2976 static void
2977 gen9_hevc_ref_frame_depth_conversion(VADriverContextP ctx,
2978                                      struct encode_state *encode_state,
2979                                      struct intel_encoder_context *encoder_context)
2980 {
2981     struct i965_driver_data *i965 = i965_driver_data(ctx);
2982     struct gen9_hevc_surface_priv *surface_priv = NULL;
2983     VAEncSliceParameterBufferHEVC *slice_param = NULL;
2984     struct object_surface *obj_surface = NULL;
2985     int i = 0;
2986
2987     slice_param = (VAEncSliceParameterBufferHEVC *)encode_state->slice_params_ext[0]->buffer;
2988
2989     for (i = 0; i < slice_param->num_ref_idx_l0_active_minus1 + 1; i++) {
2990         obj_surface = SURFACE(slice_param->ref_pic_list0[i].picture_id);
2991         if (obj_surface) {
2992             surface_priv = (struct gen9_hevc_surface_priv *)obj_surface->private_data;
2993
2994             if (!surface_priv->surface_nv12_valid) {
2995                 gen9_hevc_frame_depth_conversion(ctx, encode_state, encoder_context,
2996                                                  obj_surface, obj_surface,
2997                                                  HEVC_ENC_DS_DISABLED);
2998
2999                 surface_priv->surface_reff = surface_priv->surface_obj_nv12;
3000                 surface_priv->surface_nv12_valid = 1;
3001             }
3002         }
3003     }
3004
3005     for (i = 0; i < slice_param->num_ref_idx_l1_active_minus1 + 1; i++) {
3006         obj_surface = SURFACE(slice_param->ref_pic_list1[i].picture_id);
3007         if (obj_surface) {
3008             surface_priv = (struct gen9_hevc_surface_priv *)obj_surface->private_data;
3009
3010             if (!surface_priv->surface_nv12_valid) {
3011                 gen9_hevc_frame_depth_conversion(ctx, encode_state, encoder_context,
3012                                                  obj_surface, obj_surface,
3013                                                  HEVC_ENC_DS_DISABLED);
3014
3015                 surface_priv->surface_reff = surface_priv->surface_obj_nv12;
3016                 surface_priv->surface_nv12_valid = 1;
3017             }
3018         }
3019     }
3020 }
3021
3022 // Scaling implementation
3023
3024 static void
3025 gen9_hevc_scaling_set_curbe_2x(struct i965_gpe_context *gpe_context,
3026                                struct gen9_hevc_scaling_parameter *scaling_param)
3027 {
3028     gen9_hevc_scaling2x_curbe_data *cmd;
3029
3030     cmd = i965_gpe_context_map_curbe(gpe_context);
3031     if (!cmd)
3032         return;
3033
3034     memset((void *)cmd, 0, sizeof(*cmd));
3035
3036     cmd->dw0.input_picture_width  = scaling_param->input_frame_width;
3037     cmd->dw0.input_picture_height = scaling_param->input_frame_height;
3038
3039     cmd->dw8.input_y_bti = GEN9_HEVC_SCALING_FRAME_SRC_Y_INDEX;
3040     cmd->dw9.output_y_bti = GEN9_HEVC_SCALING_FRAME_DST_Y_INDEX;
3041
3042     i965_gpe_context_unmap_curbe(gpe_context);
3043 }
3044
3045 static void
3046 gen9_hevc_scaling_set_curbe_4x(struct i965_gpe_context *gpe_context,
3047                                struct gen9_hevc_scaling_parameter *scaling_param)
3048 {
3049     gen9_hevc_scaling4x_curbe_data *cmd;
3050
3051     cmd = i965_gpe_context_map_curbe(gpe_context);
3052     if (!cmd)
3053         return;
3054
3055     memset((void *)cmd, 0, sizeof(*cmd));
3056
3057     cmd->dw0.input_picture_width  = scaling_param->input_frame_width;
3058     cmd->dw0.input_picture_height = scaling_param->input_frame_height;
3059
3060     cmd->dw1.input_y_bti = GEN9_HEVC_SCALING_FRAME_SRC_Y_INDEX;
3061     cmd->dw2.output_y_bti = GEN9_HEVC_SCALING_FRAME_DST_Y_INDEX;
3062
3063     cmd->dw5.flatness_threshold = 0;
3064     cmd->dw6.enable_mb_flatness_check = scaling_param->enable_mb_flatness_check;
3065     cmd->dw7.enable_mb_variance_output = scaling_param->enable_mb_variance_output;
3066     cmd->dw8.enable_mb_pixel_average_output = scaling_param->enable_mb_pixel_average_output;
3067
3068     if (cmd->dw6.enable_mb_flatness_check ||
3069         cmd->dw7.enable_mb_variance_output ||
3070         cmd->dw8.enable_mb_pixel_average_output)
3071         cmd->dw10.mbv_proc_stat_bti = GEN9_HEVC_SCALING_FRAME_MBVPROCSTATS_DST_INDEX;
3072
3073     i965_gpe_context_unmap_curbe(gpe_context);
3074 }
3075
3076 static void
3077 gen9_hevc_scaling_set_curbe(struct gen9_hevc_scaling_parameter *scaling_param,
3078                             struct i965_gpe_context *gpe_context)
3079 {
3080     if (scaling_param->use_32x_scaling)
3081         gen9_hevc_scaling_set_curbe_2x(gpe_context, scaling_param);
3082     else
3083         gen9_hevc_scaling_set_curbe_4x(gpe_context, scaling_param);
3084 }
3085
3086 static void
3087 gen9_hevc_scaling_set_surfaces(VADriverContextP ctx,
3088                                struct encode_state *encode_state,
3089                                struct intel_encoder_context *encoder_context,
3090                                struct gen9_hevc_scaling_parameter *scaling_param,
3091                                struct i965_gpe_context *gpe_context)
3092 {
3093     unsigned int surface_format;
3094
3095     if (scaling_param->scaling_out_use_32unorm_surf_fmt)
3096         surface_format = I965_SURFACEFORMAT_R32_UNORM;
3097     else if (scaling_param->scaling_out_use_16unorm_surf_fmt)
3098         surface_format = I965_SURFACEFORMAT_R16_UNORM;
3099     else
3100         surface_format = I965_SURFACEFORMAT_R8_UNORM;
3101
3102     i965_add_2d_gpe_surface(ctx, gpe_context,
3103                             scaling_param->input_surface,
3104                             0, 1, surface_format,
3105                             GEN9_HEVC_SCALING_FRAME_SRC_Y_INDEX);
3106
3107     i965_add_2d_gpe_surface(ctx, gpe_context,
3108                             scaling_param->output_surface,
3109                             0, 1, surface_format,
3110                             GEN9_HEVC_SCALING_FRAME_DST_Y_INDEX);
3111
3112     if ((scaling_param->enable_mb_flatness_check ||
3113          scaling_param->enable_mb_variance_output ||
3114          scaling_param->enable_mb_pixel_average_output) &&
3115         scaling_param->use_4x_scaling) {
3116         i965_add_buffer_2d_gpe_surface(ctx, gpe_context,
3117                                        scaling_param->pres_mbv_proc_stat_buffer,
3118                                        1,
3119                                        I965_SURFACEFORMAT_R8_UNORM,
3120                                        GEN9_HEVC_SCALING_FRAME_MBVPROCSTATS_DST_INDEX);
3121     }
3122 }
3123
3124 static void
3125 gen9_hevc_kernel_scaling(VADriverContextP ctx,
3126                          struct encode_state *encode_state,
3127                          struct intel_encoder_context *encoder_context,
3128                          enum HEVC_HME_TYPE hme_type)
3129 {
3130     struct encoder_vme_mfc_context *vme_context = NULL;
3131     struct gen9_hevc_encoder_context *priv_ctx = NULL;
3132     struct gen9_hevc_encoder_state *priv_state = NULL;
3133     struct gen9_hevc_surface_priv *surface_priv = NULL;
3134     struct gen9_hevc_scaling_parameter scaling_param;
3135     struct gpe_media_object_walker_parameter param;
3136     unsigned int downscaled_width_in_mb, downscaled_height_in_mb;
3137     struct hevc_enc_kernel_walker_parameter hevc_walker_param;
3138     struct i965_gpe_context *gpe_context = NULL;
3139     int gpe_idx = 0, media_state = 0;;
3140
3141     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
3142     priv_ctx = (struct gen9_hevc_encoder_context *)vme_context->private_enc_ctx;
3143     priv_state = (struct gen9_hevc_encoder_state *)vme_context->private_enc_state;
3144
3145     surface_priv = (struct gen9_hevc_surface_priv *)encode_state->reconstructed_object->private_data;
3146
3147     memset((void *)&scaling_param, 0, sizeof(scaling_param));
3148     switch (hme_type) {
3149     case HEVC_HME_4X:
3150         media_state = HEVC_ENC_MEDIA_STATE_4X_SCALING;
3151         gpe_idx = HEVC_ENC_SCALING_4X;
3152         downscaled_width_in_mb = priv_state->downscaled_width_4x_in_mb;
3153         downscaled_height_in_mb = priv_state->downscaled_height_4x_in_mb;
3154
3155         scaling_param.input_surface = encode_state->input_yuv_object;
3156         scaling_param.input_frame_width = priv_state->picture_width;
3157         scaling_param.input_frame_height = priv_state->picture_height;
3158
3159         scaling_param.output_surface = surface_priv->scaled_surface_obj[HEVC_SCALED_SURF_4X_ID];
3160         scaling_param.output_frame_width = priv_state->frame_width_4x;
3161         scaling_param.output_frame_height = priv_state->frame_height_4x;
3162
3163         scaling_param.enable_mb_flatness_check = priv_state->flatness_check_enable;
3164         scaling_param.enable_mb_variance_output = 0;
3165         scaling_param.enable_mb_pixel_average_output = 0;
3166         scaling_param.pres_mbv_proc_stat_buffer = &(priv_ctx->res_flatness_check_surface);
3167
3168         scaling_param.blk8x8_stat_enabled = 0;
3169         scaling_param.use_4x_scaling  = 1;
3170         scaling_param.use_16x_scaling = 0;
3171         scaling_param.use_32x_scaling = 0;
3172         break;
3173     case HEVC_HME_16X:
3174         media_state = HEVC_ENC_MEDIA_STATE_16X_SCALING;
3175         gpe_idx = HEVC_ENC_SCALING_16X;
3176         downscaled_width_in_mb = priv_state->downscaled_width_16x_in_mb;
3177         downscaled_height_in_mb = priv_state->downscaled_height_16x_in_mb;
3178
3179         scaling_param.input_surface = surface_priv->scaled_surface_obj[HEVC_SCALED_SURF_4X_ID];
3180         scaling_param.input_frame_width = priv_state->frame_width_4x;
3181         scaling_param.input_frame_height = priv_state->frame_height_4x;
3182
3183         scaling_param.output_surface = surface_priv->scaled_surface_obj[HEVC_SCALED_SURF_16X_ID];
3184         scaling_param.output_frame_width = priv_state->frame_width_16x;
3185         scaling_param.output_frame_height = priv_state->frame_height_16x;
3186
3187         scaling_param.enable_mb_flatness_check = 0;
3188         scaling_param.enable_mb_variance_output = 0;
3189         scaling_param.enable_mb_pixel_average_output = 0;
3190
3191         scaling_param.blk8x8_stat_enabled = 0;
3192         scaling_param.use_4x_scaling  = 0;
3193         scaling_param.use_16x_scaling = 1;
3194         scaling_param.use_32x_scaling = 0;
3195         break;
3196     case HEVC_HME_32X:
3197         media_state = HEVC_ENC_MEDIA_STATE_32X_SCALING;
3198         gpe_idx = HEVC_ENC_SCALING_32X;
3199         downscaled_width_in_mb = priv_state->downscaled_width_32x_in_mb;
3200         downscaled_height_in_mb = priv_state->downscaled_height_32x_in_mb;
3201
3202         scaling_param.input_surface = surface_priv->scaled_surface_obj[HEVC_SCALED_SURF_16X_ID];
3203         scaling_param.input_frame_width = priv_state->frame_width_16x;
3204         scaling_param.input_frame_height = priv_state->frame_height_16x;
3205
3206         scaling_param.output_surface = surface_priv->scaled_surface_obj[HEVC_SCALED_SURF_32X_ID];
3207         scaling_param.output_frame_width = priv_state->frame_width_32x;
3208         scaling_param.output_frame_height = priv_state->frame_height_32x;
3209
3210         scaling_param.enable_mb_flatness_check = 0;
3211         scaling_param.enable_mb_variance_output = 0;
3212         scaling_param.enable_mb_pixel_average_output = 0;
3213
3214         scaling_param.blk8x8_stat_enabled = 0;
3215         scaling_param.use_4x_scaling  = 0;
3216         scaling_param.use_16x_scaling = 0;
3217         scaling_param.use_32x_scaling = 1;
3218         break;
3219     default:
3220         return;
3221     }
3222
3223     gpe_context = &priv_ctx->scaling_context.gpe_contexts[gpe_idx];
3224     gen8_gpe_context_init(ctx, gpe_context);
3225     gen9_gpe_reset_binding_table(ctx, gpe_context);
3226     gen9_hevc_scaling_set_curbe(&scaling_param, gpe_context);
3227
3228     if (hme_type == HEVC_HME_32X) {
3229         scaling_param.scaling_out_use_16unorm_surf_fmt = 1;
3230         scaling_param.scaling_out_use_32unorm_surf_fmt = 0;
3231     } else {
3232         scaling_param.scaling_out_use_16unorm_surf_fmt = 0;
3233         scaling_param.scaling_out_use_32unorm_surf_fmt = 1;
3234     }
3235
3236     gen9_hevc_scaling_set_surfaces(ctx, encode_state, encoder_context, &scaling_param,
3237                                    gpe_context);
3238     gen8_gpe_setup_interface_data(ctx, gpe_context);
3239
3240     memset((void *)&hevc_walker_param, 0, sizeof(hevc_walker_param));
3241     if (hme_type == HEVC_HME_32X) {
3242         hevc_walker_param.resolution_x = downscaled_width_in_mb;
3243         hevc_walker_param.resolution_y = downscaled_height_in_mb;
3244     } else {
3245         hevc_walker_param.resolution_x = downscaled_width_in_mb * 2;
3246         hevc_walker_param.resolution_y = downscaled_height_in_mb * 2;
3247     }
3248     hevc_walker_param.no_dependency = 1;
3249     gen9_hevc_init_object_walker(&hevc_walker_param, &param);
3250     gen9_hevc_run_object_walker(ctx, encoder_context, gpe_context, &param,
3251                                 media_state);
3252 }
3253
3254 static void
3255 gen9_hevc_hme_scaling(VADriverContextP ctx,
3256                       struct encode_state *encode_state,
3257                       struct intel_encoder_context *encoder_context)
3258 {
3259     struct encoder_vme_mfc_context *vme_context = NULL;
3260     struct generic_enc_codec_state *generic_state = NULL;
3261     struct gen9_hevc_encoder_state *priv_state = NULL;
3262
3263     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
3264     generic_state = (struct generic_enc_codec_state *)vme_context->generic_enc_state;
3265     priv_state = (struct gen9_hevc_encoder_state *)vme_context->private_enc_state;
3266
3267     if (priv_state->bit_depth_luma_minus8)
3268         gen9_hevc_frame_depth_conversion(ctx, encode_state, encoder_context,
3269                                          encode_state->input_yuv_object,
3270                                          encode_state->reconstructed_object,
3271                                          HEVC_ENC_2xDS_4xDS_STAGE);
3272     else
3273         gen9_hevc_kernel_scaling(ctx, encode_state, encoder_context, HEVC_HME_4X);
3274
3275     if (generic_state->b16xme_supported) {
3276         gen9_hevc_kernel_scaling(ctx, encode_state, encoder_context, HEVC_HME_16X);
3277
3278         if (generic_state->b32xme_supported)
3279             gen9_hevc_kernel_scaling(ctx, encode_state, encoder_context, HEVC_HME_32X);
3280     }
3281 }
3282
3283 // ME implementation
3284
3285 static void
3286 gen9_hevc_me_set_curbe(VADriverContextP ctx,
3287                        struct encode_state *encode_state,
3288                        struct intel_encoder_context *encoder_context,
3289                        enum HEVC_HME_TYPE hme_type,
3290                        struct i965_gpe_context *gpe_context)
3291 {
3292     struct encoder_vme_mfc_context *vme_context = NULL;
3293     struct generic_enc_codec_state *generic_state = NULL;
3294     struct gen9_hevc_encoder_state *priv_state = NULL;
3295     gen9_hevc_me_curbe_data *cmd = NULL;
3296     VAEncPictureParameterBufferHEVC *pic_param = NULL;
3297     VAEncSliceParameterBufferHEVC *slice_param = NULL;
3298     unsigned int use_mv_from_prev_step = 0;
3299     unsigned int write_distortions = 0;
3300     unsigned int slice_qp = 0;
3301     unsigned int me_method = 0;
3302     unsigned int mv_shift_factor = 0, prev_mv_read_pos_factor = 0;
3303     unsigned int downscaled_width_in_mb, downscaled_height_in_mb;
3304
3305     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
3306     generic_state = (struct generic_enc_codec_state *)vme_context->generic_enc_state;
3307     priv_state = (struct gen9_hevc_encoder_state *)vme_context->private_enc_state;
3308     pic_param = (VAEncPictureParameterBufferHEVC *)encode_state->pic_param_ext->buffer;
3309     slice_param = (VAEncSliceParameterBufferHEVC *)encode_state->slice_params_ext[0]->buffer;
3310
3311     switch (hme_type) {
3312     case HEVC_HME_4X :
3313         use_mv_from_prev_step = (generic_state->b16xme_enabled) ? 1 : 0;;
3314         write_distortions = 1;
3315         mv_shift_factor = 2;
3316         prev_mv_read_pos_factor = 0;
3317         downscaled_width_in_mb = ALIGN(priv_state->picture_width / 4, 16) / 16;
3318         downscaled_height_in_mb = ALIGN(priv_state->picture_height / 4, 16) / 16;
3319         break;
3320     case HEVC_HME_16X :
3321         use_mv_from_prev_step = (generic_state->b32xme_enabled) ? 1 : 0;
3322         write_distortions = 0;
3323         mv_shift_factor = 2;
3324         prev_mv_read_pos_factor = 1;
3325         downscaled_width_in_mb = ALIGN(priv_state->picture_width / 16, 16) / 16;
3326         downscaled_height_in_mb = ALIGN(priv_state->picture_height / 16, 16) / 16;
3327         break;
3328     case HEVC_HME_32X :
3329         use_mv_from_prev_step = 0;
3330         write_distortions = 0;
3331         mv_shift_factor = 1;
3332         prev_mv_read_pos_factor = 0;
3333         downscaled_width_in_mb = ALIGN(priv_state->picture_width / 32, 16) / 16;
3334         downscaled_height_in_mb = ALIGN(priv_state->picture_height / 32, 16) / 16;
3335         break;
3336     default:
3337         return;
3338     }
3339
3340     me_method = GEN9_HEVC_ME_METHOD[priv_state->tu_mode];
3341     slice_qp = pic_param->pic_init_qp + slice_param->slice_qp_delta;
3342
3343     cmd = i965_gpe_context_map_curbe(gpe_context);
3344     if (!cmd)
3345         return;
3346
3347     memcpy((void *)cmd, GEN9_HEVC_ME_INIT_CURBE_DATA, sizeof(gen9_hevc_me_curbe_data));
3348
3349     cmd->dw3.sub_pel_mode = 3;
3350     cmd->dw4.picture_height_minus1 = downscaled_height_in_mb - 1;
3351     cmd->dw4.picture_width = downscaled_width_in_mb;
3352     cmd->dw5.qp_prime_y = slice_qp;
3353     cmd->dw6.use_mv_from_prev_step = use_mv_from_prev_step;
3354     cmd->dw6.write_distortions = write_distortions;
3355     cmd->dw6.super_combine_dist = GEN9_HEVC_SUPER_COMBINE_DIST[priv_state->tu_mode];
3356     cmd->dw6.max_vmvr = 512;
3357
3358     if (priv_state->picture_coding_type != HEVC_SLICE_I) {
3359         cmd->dw13.num_ref_idx_l0_minus1 = slice_param->num_ref_idx_l0_active_minus1;
3360         if (priv_state->picture_coding_type == HEVC_SLICE_B) {
3361             cmd->dw1.bi_weight = 32;
3362             cmd->dw13.num_ref_idx_l1_minus1 = slice_param->num_ref_idx_l1_active_minus1;
3363         }
3364     }
3365
3366     cmd->dw15.prev_mv_read_pos_factor = prev_mv_read_pos_factor;
3367     cmd->dw15.mv_shift_factor = mv_shift_factor;
3368
3369     memcpy(&cmd->dw16, table_enc_search_path[GEN9_HEVC_ENC_MEMETHOD_TABLE][me_method], 14 * sizeof(unsigned int));
3370
3371     cmd->dw32._4x_memv_output_data_surf_index = GEN9_HEVC_ME_MV_DATA_SURFACE_INDEX;
3372     cmd->dw33._16x_32x_memv_input_data_surf_index = (hme_type == HEVC_HME_32X) ?
3373                                                     GEN9_HEVC_ME_32X_MV_DATA_SURFACE_INDEX : GEN9_HEVC_ME_16X_MV_DATA_SURFACE_INDEX;
3374     cmd->dw34._4x_me_output_dist_surf_index = GEN9_HEVC_ME_DISTORTION_SURFACE_INDEX;
3375     cmd->dw35._4x_me_output_brc_dist_surf_index = GEN9_HEVC_ME_BRC_DISTORTION_INDEX;
3376     cmd->dw36.vme_fwd_inter_pred_surf_index = GEN9_HEVC_ME_CURR_FOR_FWD_REF_INDEX;
3377     cmd->dw37.vme_bdw_inter_pred_surf_index = GEN9_HEVC_ME_CURR_FOR_BWD_REF_INDEX;
3378
3379     i965_gpe_context_unmap_curbe(gpe_context);
3380 }
3381
3382 static void
3383 gen9_hevc_me_set_surfaces(VADriverContextP ctx,
3384                           struct encode_state *encode_state,
3385                           struct intel_encoder_context *encoder_context,
3386                           enum HEVC_HME_TYPE hme_type,
3387                           struct i965_gpe_context *gpe_context)
3388 {
3389     struct i965_driver_data *i965 = i965_driver_data(ctx);
3390     struct encoder_vme_mfc_context *vme_context = NULL;
3391     struct gen9_hevc_encoder_context *priv_ctx = NULL;
3392     struct generic_enc_codec_state *generic_state = NULL;
3393     VAEncSliceParameterBufferHEVC *slice_param = NULL;
3394     struct gen9_hevc_surface_priv *surface_priv = NULL;
3395     struct object_surface *obj_surface = NULL;
3396     int scaled_surf_id = VA_INVALID_SURFACE, surface_id = VA_INVALID_SURFACE;
3397     int i = 0;
3398
3399     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
3400     priv_ctx = (struct gen9_hevc_encoder_context *)vme_context->private_enc_ctx;
3401     generic_state = (struct generic_enc_codec_state *)vme_context->generic_enc_state;
3402     slice_param = (VAEncSliceParameterBufferHEVC *)encode_state->slice_params_ext[0]->buffer;
3403
3404     switch (hme_type) {
3405     case HEVC_HME_4X:
3406         scaled_surf_id = HEVC_SCALED_SURF_4X_ID;
3407         i965_add_buffer_2d_gpe_surface(ctx, gpe_context,
3408                                        &priv_ctx->s4x_memv_data_buffer,
3409                                        1,
3410                                        I965_SURFACEFORMAT_R8_UNORM,
3411                                        GEN9_HEVC_ME_MV_DATA_SURFACE_INDEX);
3412
3413         if (generic_state->b16xme_enabled)
3414             i965_add_buffer_2d_gpe_surface(ctx, gpe_context,
3415                                            &priv_ctx->s16x_memv_data_buffer,
3416                                            1,
3417                                            I965_SURFACEFORMAT_R8_UNORM,
3418                                            GEN9_HEVC_ME_16X_MV_DATA_SURFACE_INDEX);
3419
3420         i965_add_buffer_2d_gpe_surface(ctx, gpe_context,
3421                                        &priv_ctx->res_brc_me_dist_buffer,
3422                                        1,
3423                                        I965_SURFACEFORMAT_R8_UNORM,
3424                                        GEN9_HEVC_ME_BRC_DISTORTION_INDEX);
3425
3426         i965_add_buffer_2d_gpe_surface(ctx, gpe_context,
3427                                        &priv_ctx->s4x_memv_distortion_buffer,
3428                                        1,
3429                                        I965_SURFACEFORMAT_R8_UNORM,
3430                                        GEN9_HEVC_ME_DISTORTION_SURFACE_INDEX);
3431         break;
3432     case HEVC_HME_16X:
3433         scaled_surf_id = HEVC_SCALED_SURF_16X_ID;
3434
3435         i965_add_buffer_2d_gpe_surface(ctx, gpe_context,
3436                                        &priv_ctx->s16x_memv_data_buffer,
3437                                        1,
3438                                        I965_SURFACEFORMAT_R8_UNORM,
3439                                        GEN9_HEVC_ME_MV_DATA_SURFACE_INDEX);
3440
3441         if (generic_state->b32xme_enabled)
3442             i965_add_buffer_2d_gpe_surface(ctx, gpe_context,
3443                                            &priv_ctx->s32x_memv_data_buffer,
3444                                            1,
3445                                            I965_SURFACEFORMAT_R8_UNORM,
3446                                            GEN9_HEVC_ME_32X_MV_DATA_SURFACE_INDEX);
3447         else
3448             i965_add_buffer_2d_gpe_surface(ctx, gpe_context,
3449                                            &priv_ctx->s16x_memv_data_buffer,
3450                                            1,
3451                                            I965_SURFACEFORMAT_R8_UNORM,
3452                                            GEN9_HEVC_ME_16X_MV_DATA_SURFACE_INDEX);
3453         break;
3454     case HEVC_HME_32X:
3455         scaled_surf_id = HEVC_SCALED_SURF_32X_ID;
3456         i965_add_buffer_2d_gpe_surface(ctx, gpe_context,
3457                                        &priv_ctx->s32x_memv_data_buffer,
3458                                        1,
3459                                        I965_SURFACEFORMAT_R8_UNORM,
3460                                        GEN9_HEVC_ME_MV_DATA_SURFACE_INDEX);
3461         break;
3462     default:
3463         return;
3464     }
3465
3466     obj_surface = encode_state->reconstructed_object;
3467     surface_priv = (struct gen9_hevc_surface_priv *)obj_surface->private_data;
3468     i965_add_adv_gpe_surface(ctx, gpe_context,
3469                              surface_priv->scaled_surface_obj[scaled_surf_id],
3470                              GEN9_HEVC_ME_CURR_FOR_FWD_REF_INDEX);
3471
3472     for (i = 0; i < slice_param->num_ref_idx_l0_active_minus1 + 1; i++) {
3473         surface_id = slice_param->ref_pic_list0[i].picture_id;
3474         obj_surface = SURFACE(surface_id);
3475         if (!obj_surface || !obj_surface->private_data)
3476             break;
3477
3478         surface_priv = (struct gen9_hevc_surface_priv *)obj_surface->private_data;
3479         i965_add_adv_gpe_surface(ctx, gpe_context,
3480                                  surface_priv->scaled_surface_obj[scaled_surf_id],
3481                                  GEN9_HEVC_ME_CURR_FOR_FWD_REF_INDEX + i * 2 + 1);
3482     }
3483
3484     obj_surface = encode_state->reconstructed_object;
3485     surface_priv = (struct gen9_hevc_surface_priv *)obj_surface->private_data;
3486     i965_add_adv_gpe_surface(ctx, gpe_context,
3487                              surface_priv->scaled_surface_obj[scaled_surf_id],
3488                              GEN9_HEVC_ME_CURR_FOR_BWD_REF_INDEX);
3489
3490     for (i = 0; i < slice_param->num_ref_idx_l1_active_minus1 + 1; i++) {
3491         surface_id = slice_param->ref_pic_list1[i].picture_id;
3492         obj_surface = SURFACE(surface_id);
3493         if (!obj_surface || !obj_surface->private_data)
3494             break;
3495
3496         surface_priv = (struct gen9_hevc_surface_priv *)obj_surface->private_data;
3497         i965_add_adv_gpe_surface(ctx, gpe_context,
3498                                  surface_priv->scaled_surface_obj[scaled_surf_id],
3499                                  GEN9_HEVC_ME_CURR_FOR_BWD_REF_INDEX + i * 2 + 1);
3500     }
3501 }
3502
3503 static void
3504 gen9_hevc_kernel_me(VADriverContextP ctx,
3505                     struct encode_state *encode_state,
3506                     struct intel_encoder_context *encoder_context,
3507                     enum HEVC_HME_TYPE hme_type)
3508 {
3509     struct encoder_vme_mfc_context *vme_context = NULL;
3510     struct gen9_hevc_encoder_context *priv_ctx = NULL;
3511     struct gen9_hevc_encoder_state *priv_state = NULL;
3512     struct gpe_media_object_walker_parameter param;
3513     struct hevc_enc_kernel_walker_parameter hevc_walker_param;
3514     unsigned int downscaled_width_in_mb, downscaled_height_in_mb;
3515     struct i965_gpe_context *gpe_context;
3516     int media_state = 0;
3517
3518     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
3519     priv_ctx = (struct gen9_hevc_encoder_context *)vme_context->private_enc_ctx;
3520     priv_state = (struct gen9_hevc_encoder_state *)vme_context->private_enc_state;
3521
3522     switch (hme_type) {
3523     case HEVC_HME_4X:
3524         media_state = HEVC_ENC_MEDIA_STATE_4X_ME;
3525         downscaled_width_in_mb = priv_state->downscaled_width_4x_in_mb;
3526         downscaled_height_in_mb = priv_state->downscaled_height_4x_in_mb;
3527         break;
3528     case HEVC_HME_16X:
3529         media_state = HEVC_ENC_MEDIA_STATE_16X_ME;
3530         downscaled_width_in_mb = priv_state->downscaled_width_16x_in_mb;
3531         downscaled_height_in_mb = priv_state->downscaled_height_16x_in_mb;
3532         break;
3533     case HEVC_HME_32X:
3534         media_state = HEVC_ENC_MEDIA_STATE_32X_ME;
3535         downscaled_width_in_mb = priv_state->downscaled_width_32x_in_mb;
3536         downscaled_height_in_mb = priv_state->downscaled_height_32x_in_mb;
3537         break;
3538     default:
3539         return;
3540     }
3541
3542     if (priv_state->picture_coding_type == HEVC_SLICE_P)
3543         gpe_context = &priv_ctx->me_context.gpe_context[hme_type][HEVC_ENC_ME_P];
3544     else
3545         gpe_context = &priv_ctx->me_context.gpe_context[hme_type][HEVC_ENC_ME_B];
3546
3547     gen8_gpe_context_init(ctx, gpe_context);
3548     gen9_gpe_reset_binding_table(ctx, gpe_context);
3549     gen9_hevc_me_set_curbe(ctx, encode_state, encoder_context, hme_type, gpe_context);
3550     gen9_hevc_me_set_surfaces(ctx, encode_state, encoder_context, hme_type, gpe_context);
3551     gen8_gpe_setup_interface_data(ctx, gpe_context);
3552
3553     memset((void *)&hevc_walker_param, 0, sizeof(hevc_walker_param));
3554     hevc_walker_param.resolution_x = downscaled_width_in_mb;
3555     hevc_walker_param.resolution_y = downscaled_height_in_mb;
3556     hevc_walker_param.no_dependency = 1;
3557     gen9_hevc_init_object_walker(&hevc_walker_param, &param);
3558
3559     gen9_hevc_run_object_walker(ctx, encoder_context, gpe_context, &param,
3560                                 media_state);
3561 }
3562
3563 static void
3564 gen9_hevc_hme_encode_me(VADriverContextP ctx,
3565                         struct encode_state *encode_state,
3566                         struct intel_encoder_context *encoder_context)
3567 {
3568     struct encoder_vme_mfc_context *vme_context = NULL;
3569     struct generic_enc_codec_state *generic_state = NULL;
3570
3571     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
3572     generic_state = (struct generic_enc_codec_state *)vme_context->generic_enc_state;
3573
3574     if (generic_state->b16xme_enabled) {
3575         if (generic_state->b32xme_enabled)
3576             gen9_hevc_kernel_me(ctx, encode_state, encoder_context, HEVC_HME_32X);
3577
3578         gen9_hevc_kernel_me(ctx, encode_state, encoder_context, HEVC_HME_16X);
3579     }
3580
3581     gen9_hevc_kernel_me(ctx, encode_state, encoder_context, HEVC_HME_4X);
3582 }
3583
3584 // MBENC kernels setting start
3585
3586 static unsigned char
3587 map_44_lut_value(unsigned int value,
3588                  unsigned char max)
3589 {
3590     unsigned int max_cost = 0;
3591     int data = 0;
3592     unsigned char ret = 0;
3593
3594     if (value == 0)
3595         return 0;
3596
3597     max_cost = ((max & 15) << (max >> 4));
3598     if (value >= max_cost)
3599         return max;
3600
3601     data = (int)(log((double)value) / log(2.)) - 3;
3602     if (data < 0)
3603         data = 0;
3604
3605     ret = (unsigned char)((data << 4) +
3606                           (int)((value + (data == 0 ? 0 : (1 << (data - 1)))) >> data));
3607     ret = (ret & 0xf) == 0 ? (ret | 8) : ret;
3608
3609     return ret;
3610 }
3611
3612 static void
3613 gen9_hevc_mbenc_set_costs(struct gen9_hevc_encoder_context *priv_ctx,
3614                           int slice_type,
3615                           int intra_trans_type,
3616                           unsigned int slice_qp,
3617                           unsigned char *mode_cost,
3618                           unsigned char *mv_cost,
3619                           unsigned char *mode_cost_sp,
3620                           unsigned int *simplest_intra_inter_threshold)
3621 {
3622     float had_bias = intra_trans_type == HEVC_ENC_INTRA_TRANS_HADAMARD ?
3623                      1.67f : 2.0f;
3624     double lambda_md, lambda_me;
3625
3626     lambda_md = priv_ctx->lambda_md_table[slice_type][slice_qp];
3627     lambda_me = priv_ctx->lambda_md_table[slice_type][slice_qp];
3628
3629     if (mode_cost) {
3630         mode_cost[0] = map_44_lut_value((unsigned int)(lambda_md * GEN9_HEVC_ENC_Mode_COST[slice_type][0] * had_bias), 0x6f);
3631         mode_cost[1] = map_44_lut_value((unsigned int)(lambda_md * GEN9_HEVC_ENC_Mode_COST[slice_type][1] * had_bias), 0x8f);
3632         mode_cost[2] = map_44_lut_value((unsigned int)(lambda_md * GEN9_HEVC_ENC_Mode_COST[slice_type][2] * had_bias), 0x8f);
3633         mode_cost[3] = map_44_lut_value((unsigned int)(lambda_md * GEN9_HEVC_ENC_Mode_COST[slice_type][3] * had_bias), 0x8f);
3634         mode_cost[4] = map_44_lut_value((unsigned int)(lambda_md * GEN9_HEVC_ENC_Mode_COST[slice_type][4] * had_bias), 0x8f);
3635         mode_cost[5] = map_44_lut_value((unsigned int)(lambda_md * GEN9_HEVC_ENC_Mode_COST[slice_type][5] * had_bias), 0x6f);
3636         mode_cost[6] = map_44_lut_value((unsigned int)(lambda_md * GEN9_HEVC_ENC_Mode_COST[slice_type][6] * had_bias), 0x6f);
3637         mode_cost[7] = map_44_lut_value((unsigned int)(lambda_md * GEN9_HEVC_ENC_Mode_COST[slice_type][7] * had_bias), 0x6f);
3638         mode_cost[8] = map_44_lut_value((unsigned int)(lambda_md * GEN9_HEVC_ENC_Mode_COST[slice_type][8] * had_bias), 0x8f);
3639         mode_cost[9] = map_44_lut_value((unsigned int)(lambda_md * GEN9_HEVC_ENC_Mode_COST[slice_type][9] * had_bias), 0x6f);
3640         mode_cost[10] = map_44_lut_value((unsigned int)(lambda_md * GEN9_HEVC_ENC_Mode_COST[slice_type][10] * had_bias), 0x6f);
3641         mode_cost[11] = map_44_lut_value((unsigned int)(lambda_md * GEN9_HEVC_ENC_Mode_COST[slice_type][11] * had_bias), 0x6f);
3642     }
3643
3644     if (mv_cost) {
3645         mv_cost[0] = map_44_lut_value((unsigned int)(lambda_me * GEN9_HEVC_ENC_MV_COST[slice_type][0] * had_bias), 0x6f);
3646         mv_cost[1] = map_44_lut_value((unsigned int)(lambda_me * GEN9_HEVC_ENC_MV_COST[slice_type][1] * had_bias), 0x6f);
3647         mv_cost[2] = map_44_lut_value((unsigned int)(lambda_me * GEN9_HEVC_ENC_MV_COST[slice_type][2] * had_bias), 0x6f);
3648         mv_cost[3] = map_44_lut_value((unsigned int)(lambda_me * GEN9_HEVC_ENC_MV_COST[slice_type][3] * had_bias), 0x6f);
3649         mv_cost[4] = map_44_lut_value((unsigned int)(lambda_me * GEN9_HEVC_ENC_MV_COST[slice_type][4] * had_bias), 0x6f);
3650         mv_cost[5] = map_44_lut_value((unsigned int)(lambda_me * GEN9_HEVC_ENC_MV_COST[slice_type][5] * had_bias), 0x6f);
3651         mv_cost[6] = map_44_lut_value((unsigned int)(lambda_me * GEN9_HEVC_ENC_MV_COST[slice_type][6] * had_bias), 0x6f);
3652         mv_cost[7] = map_44_lut_value((unsigned int)(lambda_me * GEN9_HEVC_ENC_MV_COST[slice_type][7] * had_bias), 0x6f);
3653     }
3654
3655     if (mode_cost_sp)
3656         *mode_cost_sp = map_44_lut_value((unsigned int)(lambda_md * 45 * had_bias), 0x8f);
3657
3658     if (simplest_intra_inter_threshold) {
3659         lambda_md *= had_bias;
3660         *simplest_intra_inter_threshold = 0;
3661         if (GEN9_HEVC_ENC_Mode_COST[slice_type][1] < GEN9_HEVC_ENC_Mode_COST[slice_type][3])
3662             *simplest_intra_inter_threshold = (unsigned int)(lambda_md *
3663                                                              (GEN9_HEVC_ENC_Mode_COST[slice_type][3] - GEN9_HEVC_ENC_Mode_COST[slice_type][1]) + 0.5);
3664     }
3665 }
3666
3667 static void
3668 gen9_hevc_set_lambda_tables(struct gen9_hevc_encoder_context *priv_ctx,
3669                             int slice_type,
3670                             int intra_trans_type)
3671 {
3672     if (slice_type != HEVC_SLICE_I) {
3673         if (priv_ctx->lambda_table_inited)
3674             return;
3675
3676         memcpy((void *)&priv_ctx->lambda_me_table[slice_type], &GEN9_HEVC_ENC_QP_LAMBDA_ME[slice_type],
3677                sizeof(GEN9_HEVC_ENC_QP_LAMBDA_ME[slice_type]));
3678         memcpy((void *)&priv_ctx->lambda_md_table[slice_type], &GEN9_HEVC_ENC_QP_LAMBDA_ME[slice_type],
3679                sizeof(GEN9_HEVC_ENC_QP_LAMBDA_ME[slice_type]));
3680     } else if (intra_trans_type != priv_ctx->lambda_intra_trans_type ||
3681                !priv_ctx->lambda_table_inited) {
3682         double temp = 0.0;
3683         double lambda = 0.0;
3684         int qp = 0;
3685
3686         for (qp = 0; qp < 52; qp++) {
3687             temp = (double)qp - 12;
3688             lambda = 0.85 * pow(2.0, temp / 3.0);
3689
3690             if ((intra_trans_type != HEVC_ENC_INTRA_TRANS_HAAR) &&
3691                 (intra_trans_type != HEVC_ENC_INTRA_TRANS_HADAMARD))
3692                 lambda *= 0.95;
3693
3694             priv_ctx->lambda_md_table[slice_type][qp] =
3695                 priv_ctx->lambda_me_table[slice_type][qp] = sqrt(lambda);
3696         }
3697
3698         priv_ctx->lambda_intra_trans_type = intra_trans_type;
3699     }
3700 }
3701
3702 static void
3703 gen9_hevc_lambda_tables_init(struct gen9_hevc_encoder_context *priv_ctx)
3704 {
3705     gen9_hevc_set_lambda_tables(priv_ctx, HEVC_SLICE_B, HEVC_ENC_INTRA_TRANS_HAAR);
3706     gen9_hevc_set_lambda_tables(priv_ctx, HEVC_SLICE_P, HEVC_ENC_INTRA_TRANS_HAAR);
3707     gen9_hevc_set_lambda_tables(priv_ctx, HEVC_SLICE_I, HEVC_ENC_INTRA_TRANS_HAAR);
3708
3709     priv_ctx->lambda_table_inited = 1;
3710 }
3711
3712 static void
3713 gen9_hevc_8x8_b_pak_set_curbe(VADriverContextP ctx,
3714                               struct encode_state *encode_state,
3715                               struct intel_encoder_context *encoder_context,
3716                               struct i965_gpe_context *gpe_context)
3717 {
3718     struct i965_driver_data *i965 = i965_driver_data(ctx);
3719     struct encoder_vme_mfc_context *vme_context = NULL;
3720     struct generic_enc_codec_state *generic_state = NULL;
3721     struct gen9_hevc_encoder_state *priv_state = NULL;
3722     gen9_hevc_mbenc_b_pak_curbe_data *cmd = NULL;
3723     VAEncPictureParameterBufferHEVC *pic_param = NULL;
3724     VAEncSliceParameterBufferHEVC *slice_param = NULL;
3725     unsigned int slice_qp = 0;
3726     int bti_idx = 0;
3727
3728     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
3729     generic_state = (struct generic_enc_codec_state *)vme_context->generic_enc_state;
3730     priv_state = (struct gen9_hevc_encoder_state *)vme_context->private_enc_state;
3731     pic_param = (VAEncPictureParameterBufferHEVC *)encode_state->pic_param_ext->buffer;
3732     slice_param = (VAEncSliceParameterBufferHEVC *)encode_state->slice_params_ext[0]->buffer;
3733
3734     slice_qp = pic_param->pic_init_qp + slice_param->slice_qp_delta;
3735     cmd = i965_gpe_context_map_curbe(gpe_context);
3736     if (!cmd)
3737         return;
3738
3739     memset((void *)cmd, 0, sizeof(*cmd));
3740
3741     cmd->dw0.frame_width = ALIGN(priv_state->picture_width, 16);
3742     cmd->dw0.frame_height = ALIGN(priv_state->picture_height, 16);
3743     cmd->dw1.max_vmvr = 511 * 4;
3744     cmd->dw1.qp = slice_qp;
3745     cmd->dw2.brc_enable = generic_state->brc_enabled;
3746     cmd->dw2.lcu_brc_enable = priv_state->lcu_brc_enabled;
3747     cmd->dw2.screen_content = !!pic_param->pic_fields.bits.screen_content_flag;
3748     cmd->dw2.slice_type = priv_state->picture_coding_type;
3749     cmd->dw2.roi_enable = (priv_state->num_roi > 0);
3750     cmd->dw2.fast_surveillance_flag = priv_state->picture_coding_type == HEVC_SLICE_I ?
3751                                       0 : priv_state->video_surveillance_flag;
3752     cmd->dw2.kbl_control_flag = (IS_KBL(i965->intel.device_info) || IS_GLK(i965->intel.device_info));
3753     cmd->dw2.enable_rolling_intra = priv_state->rolling_intra_refresh;
3754     cmd->dw2.simplest_intra_enable = (priv_state->tu_mode == HEVC_TU_BEST_SPEED);
3755     cmd->dw3.widi_intra_refresh_qp_delta = priv_state->widi_intra_refresh_qp_delta;
3756     cmd->dw3.widi_intra_refresh_mb_num = priv_state->widi_intra_insertion_location;
3757     cmd->dw3.widi_intra_refresh_unit_in_mb = priv_state->widi_intra_insertion_size;
3758
3759     cmd->dw16.bti_cu_record = bti_idx++;
3760     cmd->dw17.bti_pak_obj = bti_idx++;
3761     cmd->dw18.bti_slice_map = bti_idx++;
3762     cmd->dw19.bti_brc_input = bti_idx++;
3763     cmd->dw20.bti_lcu_qp = bti_idx++;
3764     cmd->dw21.bti_brc_data = bti_idx++;
3765     cmd->dw22.bti_mb_data = bti_idx++;
3766     cmd->dw23.bti_mvp_surface = bti_idx++;
3767     cmd->dw24.bti_debug = bti_idx++;
3768
3769     i965_gpe_context_unmap_curbe(gpe_context);
3770 }
3771
3772 static void
3773 gen9_hevc_8x8_b_pak_set_surfaces(VADriverContextP ctx,
3774                                  struct encode_state *encode_state,
3775                                  struct intel_encoder_context *encoder_context,
3776                                  struct i965_gpe_context *gpe_context)
3777 {
3778     struct encoder_vme_mfc_context *vme_context = NULL;
3779     struct gen9_hevc_encoder_context *priv_ctx = NULL;
3780     struct gen9_hevc_encoder_state *priv_state = NULL;
3781     int bti_idx = 0;
3782     int size = 0;
3783
3784     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
3785     priv_ctx = (struct gen9_hevc_encoder_context *)vme_context->private_enc_ctx;
3786     priv_state = (struct gen9_hevc_encoder_state *)vme_context->private_enc_state;
3787
3788     size = priv_state->width_in_cu * priv_state->height_in_cu *
3789            priv_state->cu_record_size;
3790     gen9_hevc_set_gpe_1d_surface(ctx, priv_ctx, gpe_context,
3791                                  HEVC_ENC_SURFACE_CU_RECORD, bti_idx++,
3792                                  0, size, priv_state->mb_data_offset, NULL, NULL);
3793
3794     size = priv_state->mb_data_offset;
3795     gen9_hevc_set_gpe_1d_surface(ctx, priv_ctx, gpe_context,
3796                                  HEVC_ENC_SURFACE_HCP_PAK, bti_idx++,
3797                                  0, size, 0, NULL, NULL);
3798
3799     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
3800                                  HEVC_ENC_SURFACE_SLICE_MAP, bti_idx++,
3801                                  0, 1, I965_SURFACEFORMAT_R8_UNORM,
3802                                  NULL, NULL);
3803
3804     gen9_hevc_set_gpe_1d_surface(ctx, priv_ctx, gpe_context,
3805                                  HEVC_ENC_SURFACE_BRC_INPUT, bti_idx++,
3806                                  0, 0, 0, NULL, NULL);
3807
3808     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
3809                                  HEVC_ENC_SURFACE_LCU_QP, bti_idx++,
3810                                  0, 1, I965_SURFACEFORMAT_R8_UNORM,
3811                                  NULL, NULL);
3812
3813     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
3814                                  HEVC_ENC_SURFACE_BRC_DATA, bti_idx++,
3815                                  0, 1, I965_SURFACEFORMAT_R8_UNORM,
3816                                  NULL, NULL);
3817
3818     gen9_hevc_set_gpe_1d_surface(ctx, priv_ctx, gpe_context,
3819                                  HEVC_ENC_SURFACE_MB_MV_INDEX, bti_idx++,
3820                                  0, 0, 0, NULL, NULL);
3821
3822     gen9_hevc_set_gpe_1d_surface(ctx, priv_ctx, gpe_context,
3823                                  HEVC_ENC_SURFACE_MVP_INDEX, bti_idx++,
3824                                  0, 0, 0, NULL, NULL);
3825
3826     gen9_hevc_set_gpe_1d_surface(ctx, priv_ctx, gpe_context,
3827                                  HEVC_ENC_SURFACE_KERNEL_DEBUG, bti_idx++,
3828                                  0, 0, 0, NULL, NULL);
3829 }
3830
3831 static void
3832 gen9_hevc_8x8_b_pak(VADriverContextP ctx,
3833                     struct encode_state *encode_state,
3834                     struct intel_encoder_context *encoder_context)
3835 {
3836     struct encoder_vme_mfc_context *vme_context = NULL;
3837     struct gen9_hevc_encoder_context *priv_ctx = NULL;
3838     struct gen9_hevc_encoder_state *priv_state = NULL;
3839     struct gpe_media_object_walker_parameter param;
3840     struct hevc_enc_kernel_walker_parameter hevc_walker_param;
3841     struct i965_gpe_context *gpe_context = NULL;
3842     int media_state = HEVC_ENC_MEDIA_STATE_HEVC_B_PAK;
3843
3844     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
3845     priv_ctx = (struct gen9_hevc_encoder_context *)vme_context->private_enc_ctx;
3846     priv_state = (struct gen9_hevc_encoder_state *)vme_context->private_enc_state;
3847
3848     gpe_context = &priv_ctx->mbenc_context.gpe_contexts[HEVC_MBENC_BPAK_IDX];
3849
3850     gen8_gpe_context_init(ctx, gpe_context);
3851     gen9_gpe_reset_binding_table(ctx, gpe_context);
3852     gen9_hevc_8x8_b_pak_set_curbe(ctx, encode_state, encoder_context, gpe_context);
3853     gen9_hevc_8x8_b_pak_set_surfaces(ctx, encode_state, encoder_context, gpe_context);
3854     gen8_gpe_setup_interface_data(ctx, gpe_context);
3855
3856     memset((void *)&hevc_walker_param, 0, sizeof(hevc_walker_param));
3857     hevc_walker_param.resolution_x = ALIGN(priv_state->picture_width, 32) >> 5;
3858     hevc_walker_param.resolution_y = ALIGN(priv_state->picture_height, 32) >> 5;
3859     hevc_walker_param.no_dependency = 1;
3860     gen9_hevc_init_object_walker(&hevc_walker_param, &param);
3861     gen9_hevc_run_object_walker(ctx, encoder_context, gpe_context, &param,
3862                                 media_state);
3863 }
3864
3865 static const unsigned char ftq_25i[27] = {
3866     0, 0, 0, 0,
3867     1, 3, 6, 8, 11,
3868     13, 16, 19, 22, 26,
3869     30, 34, 39, 44, 50,
3870     56, 62, 69, 77, 85,
3871     94, 104, 115
3872 };
3873
3874 static void
3875 gen9_hevc_set_forward_coeff_thd(unsigned char *pcoeff,
3876                                 int qp)
3877 {
3878     int idx = (qp + 1) >> 1;
3879
3880     memset((void *)pcoeff, ftq_25i[idx], 7);
3881 }
3882
3883 static int
3884 gen9_hevc_get_qp_from_ref_list(VADriverContextP ctx,
3885                                VAEncSliceParameterBufferHEVC *slice_param,
3886                                int list_idx,
3887                                int ref_frame_idx)
3888 {
3889     struct i965_driver_data *i965 = i965_driver_data(ctx);
3890     struct gen9_hevc_surface_priv *hevc_priv_surface = NULL;
3891     struct object_surface *obj_surface = NULL;
3892     VASurfaceID surface_id;
3893
3894     if (list_idx == 0) {
3895         if (ref_frame_idx < slice_param->num_ref_idx_l0_active_minus1 + 1)
3896             surface_id = slice_param->ref_pic_list0[ref_frame_idx].picture_id;
3897         else
3898             goto FAIL;
3899     } else {
3900         if (ref_frame_idx < slice_param->num_ref_idx_l1_active_minus1 + 1)
3901             surface_id = slice_param->ref_pic_list1[ref_frame_idx].picture_id;
3902         else
3903             goto FAIL;
3904     }
3905
3906     obj_surface = SURFACE(surface_id);
3907     if (obj_surface && obj_surface->private_data) {
3908         hevc_priv_surface = obj_surface->private_data;
3909         return hevc_priv_surface->qp_value;
3910     }
3911
3912 FAIL:
3913     return 0;
3914 }
3915
3916 static short
3917 gen9_hevc_get_poc_diff_from_ref_list(VAEncPictureParameterBufferHEVC *pic_param,
3918                                      VAEncSliceParameterBufferHEVC *slice_param,
3919                                      int list_idx,
3920                                      int ref_frame_idx)
3921 {
3922     short poc_diff = 0;
3923
3924     if (list_idx == 0 &&
3925         ref_frame_idx < slice_param->num_ref_idx_l0_active_minus1 + 1)
3926         poc_diff = pic_param->decoded_curr_pic.pic_order_cnt -
3927                    slice_param->ref_pic_list0[ref_frame_idx].pic_order_cnt;
3928     else if (list_idx == 1 &&
3929              ref_frame_idx < slice_param->num_ref_idx_l1_active_minus1 + 1)
3930         poc_diff = pic_param->decoded_curr_pic.pic_order_cnt -
3931                    slice_param->ref_pic_list1[ref_frame_idx].pic_order_cnt;
3932
3933     return CLAMP(-128, 127, poc_diff);
3934 }
3935
3936 static void
3937 gen9_hevc_set_control_region(VADriverContextP ctx,
3938                              struct encode_state *encode_state,
3939                              struct intel_encoder_context *encoder_context,
3940                              struct i965_gpe_context *gpe_context,
3941                              struct gen9_hevc_walking_pattern_parameter *param)
3942 {
3943     struct encoder_vme_mfc_context *vme_context = NULL;
3944     struct gen9_hevc_encoder_context *priv_ctx = NULL;
3945     struct gen9_hevc_encoder_state *priv_state = NULL;
3946     VAEncSliceParameterBufferHEVC *slice_param = NULL;
3947     gen9_hevc_mbenc_control_region *p_region = NULL;
3948     unsigned int slice, num_regions, height, num_slices, num_units_in_region;
3949     unsigned int frame_width_in_units, frame_height_in_units;
3950     unsigned short region_start_table[64];
3951     unsigned int offset_to_the_region_start[16];
3952     unsigned short temp_data[32][32];
3953     int is_arbitrary_slices = 0;
3954     int slice_start_y[I965_MAX_NUM_SLICE + 1];
3955     int max_height;
3956     int k = 0, i = 0;
3957
3958     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
3959     priv_ctx = (struct gen9_hevc_encoder_context *)vme_context->private_enc_ctx;
3960     priv_state = (struct gen9_hevc_encoder_state *)vme_context->private_enc_state;
3961
3962     memset(slice_start_y, 0, sizeof(int) * (I965_MAX_NUM_SLICE + 1));
3963     memset(region_start_table, 0, sizeof(region_start_table));
3964     memset(temp_data, 0, sizeof(temp_data));
3965     memset(offset_to_the_region_start, 0, sizeof(offset_to_the_region_start));
3966
3967     if (priv_state->num_regions_in_slice < 1)
3968         priv_state->num_regions_in_slice = 1;
3969
3970     if (priv_state->num_regions_in_slice > 16)
3971         priv_state->num_regions_in_slice = 16;
3972
3973     if (priv_state->walking_pattern_26) {
3974         frame_width_in_units = ALIGN(priv_state->picture_width, 16) / 16;
3975         frame_height_in_units = ALIGN(priv_state->picture_height, 16) / 16;
3976     } else {
3977         frame_width_in_units = ALIGN(priv_state->picture_width, 32) / 32;
3978         frame_height_in_units = ALIGN(priv_state->picture_height, 32) / 32;
3979     }
3980
3981     for (slice = 0; slice < encode_state->num_slice_params_ext; slice++) {
3982         slice_param = (VAEncSliceParameterBufferHEVC *)encode_state->slice_params_ext[slice]->buffer;
3983         if (slice_param->slice_segment_address %
3984             ALIGN(priv_state->picture_width, 32)) {
3985             is_arbitrary_slices = 1;
3986         } else {
3987             slice_start_y[slice] = slice_param->slice_segment_address /
3988                                    ALIGN(priv_state->picture_width, 32);
3989
3990             if (priv_state->walking_pattern_26) {
3991                 slice_start_y[slice] *= 2;
3992             }
3993         }
3994     }
3995
3996     slice_start_y[encode_state->num_slice_params_ext] = frame_height_in_units;
3997
3998     region_start_table[0] = 0;
3999     region_start_table[GEN9_HEVC_ENC_REGION_START_Y_OFFSET + 0] = 0;
4000     num_regions = 1;
4001
4002     if (is_arbitrary_slices) {
4003         height = frame_height_in_units;
4004         num_slices = 1;
4005         max_height = height;
4006         if (priv_state->num_regions_in_slice > 1) {
4007             num_units_in_region =
4008                 (frame_width_in_units + 2 * (frame_height_in_units - 1) + priv_state->num_regions_in_slice - 1) / priv_state->num_regions_in_slice;
4009
4010             num_regions = priv_state->num_regions_in_slice;
4011
4012             for (i = 1; i < priv_state->num_regions_in_slice; i++) {
4013                 unsigned int front = i * num_units_in_region;
4014
4015                 if (front < frame_width_in_units)
4016                     region_start_table[i] = (unsigned int)front;
4017                 else if (((front - frame_width_in_units + 1) & 1) == 0)
4018                     region_start_table[i] = (unsigned int)frame_width_in_units - 1;
4019                 else
4020                     region_start_table[i] = (unsigned int)frame_width_in_units - 2;
4021
4022                 region_start_table[GEN9_HEVC_ENC_REGION_START_Y_OFFSET + i] = (unsigned int)((front - region_start_table[i]) >> 1);
4023             }
4024         }
4025     } else {
4026         int start_y;
4027         int slice_is_merged = 0;
4028
4029         max_height = 0;
4030         num_slices = encode_state->num_slice_params_ext;
4031
4032         for (slice = 0; slice < num_slices; slice++) {
4033             int sliceHeight = slice_start_y[slice + 1] - slice_start_y[slice];
4034
4035             if (sliceHeight > max_height)
4036                 max_height = sliceHeight;
4037         }
4038
4039         while (!slice_is_merged) {
4040             int newNumSlices = 1;
4041
4042             start_y = 0;
4043
4044             for (slice = 1; slice < num_slices; slice++) {
4045                 if ((slice_start_y[slice + 1] - start_y) <= max_height)
4046                     slice_start_y[slice] = -1;
4047                 else
4048                     start_y = slice_start_y[slice];
4049             }
4050
4051             for (slice = 1; slice < num_slices; slice++) {
4052                 if (slice_start_y[slice] > 0) {
4053                     slice_start_y[newNumSlices] = slice_start_y[slice];
4054                     newNumSlices++;
4055                 }
4056             }
4057
4058             num_slices = newNumSlices;
4059             slice_start_y[num_slices] = frame_height_in_units;
4060
4061             if (num_slices * priv_state->num_regions_in_slice <= GEN9_HEVC_MEDIA_WALKER_MAX_COLORS)
4062                 slice_is_merged = 1;
4063             else {
4064                 int num = 1;
4065
4066                 max_height = frame_height_in_units;
4067
4068                 for (slice = 0; slice < num_slices - 1; slice++) {
4069                     if ((slice_start_y[slice + 2] - slice_start_y[slice]) <= max_height) {
4070                         max_height = slice_start_y[slice + 2] - slice_start_y[slice];
4071                         num = slice + 1;
4072                     }
4073                 }
4074
4075                 for (slice = num; slice < num_slices; slice++)
4076                     slice_start_y[slice] = slice_start_y[slice + 1];
4077
4078                 num_slices--;
4079             }
4080         }
4081
4082         num_units_in_region = (frame_width_in_units + 2 * (max_height - 1) + priv_state->num_regions_in_slice - 1) /
4083                               priv_state->num_regions_in_slice;
4084         num_regions = num_slices * priv_state->num_regions_in_slice;
4085
4086         for (slice = 0; slice < num_slices; slice++) {
4087             region_start_table[slice * priv_state->num_regions_in_slice] = 0;
4088             region_start_table[GEN9_HEVC_ENC_REGION_START_Y_OFFSET + (slice * priv_state->num_regions_in_slice)] = (unsigned int)slice_start_y[slice];
4089
4090             for (i = 1; i < priv_state->num_regions_in_slice; i++) {
4091                 int front = i * num_units_in_region;
4092
4093                 if ((unsigned int)front < frame_width_in_units)
4094                     region_start_table[slice * priv_state->num_regions_in_slice + i] = (unsigned int)front;
4095                 else if (((front - frame_width_in_units + 1) & 1) == 0)
4096                     region_start_table[slice * priv_state->num_regions_in_slice + i] = (unsigned int)frame_width_in_units - 1;
4097                 else
4098                     region_start_table[slice * priv_state->num_regions_in_slice + i] = (unsigned int)frame_width_in_units - 2;
4099
4100                 region_start_table[GEN9_HEVC_ENC_REGION_START_Y_OFFSET + (slice * priv_state->num_regions_in_slice + i)] = (unsigned int)slice_start_y[slice] +
4101                                                                                                                            ((front - region_start_table[i]) >> 1);
4102             }
4103         }
4104         height = max_height;
4105     }
4106
4107     for (k = 0; k < num_slices; k++) {
4108         int i;
4109         int nearest_reg = 0;
4110         int min_delta = priv_state->picture_height;
4111
4112         if (priv_state->walking_pattern_26) {
4113             int cur_lcu_pel_y = region_start_table[GEN9_HEVC_ENC_REGION_START_Y_OFFSET + (k * priv_state->num_regions_in_slice)] << 4;
4114             int ts_width = priv_state->picture_width >> 4;
4115             int ts_height = height;
4116             int offset_y = -((ts_width + 1) >> 1);
4117             int offset_delta = ((ts_width + ((ts_height - 1) << 1)) + (priv_state->num_regions_in_slice - 1)) / (priv_state->num_regions_in_slice);
4118
4119             for (i = 0; i < (int)num_regions; i++) {
4120                 if (region_start_table[i] == 0) {
4121                     int delta = cur_lcu_pel_y - (region_start_table[GEN9_HEVC_ENC_REGION_START_Y_OFFSET + i] << 4);
4122
4123                     if (delta >= 0) {
4124                         if (delta < min_delta) {
4125                             min_delta = delta;
4126                             nearest_reg = i;
4127                         }
4128                     }
4129                 }
4130
4131                 offset_to_the_region_start[k] = 2 * region_start_table[GEN9_HEVC_ENC_REGION_START_Y_OFFSET + nearest_reg];
4132             }
4133
4134             for (i = 0; i < priv_state->num_regions_in_slice; i++) {
4135                 int tmp_y = region_start_table[GEN9_HEVC_ENC_REGION_START_Y_OFFSET + (nearest_reg + priv_state->num_regions_in_slice)];
4136
4137                 temp_data[k * priv_state->num_regions_in_slice + i][0] = region_start_table[nearest_reg + i];
4138                 temp_data[k * priv_state->num_regions_in_slice + i][1] = region_start_table[GEN9_HEVC_ENC_REGION_START_Y_OFFSET + (nearest_reg + i)];
4139                 temp_data[k * priv_state->num_regions_in_slice + i][2] = region_start_table[GEN9_HEVC_ENC_REGION_START_Y_OFFSET + nearest_reg];
4140                 temp_data[k * priv_state->num_regions_in_slice + i][3] = (unsigned int)((tmp_y != 0) ? tmp_y : (priv_state->picture_height) >> 4);
4141                 temp_data[k * priv_state->num_regions_in_slice + i][4] = offset_to_the_region_start[k] & 0x0FFFF;
4142                 temp_data[k * priv_state->num_regions_in_slice + i][5] = 0;
4143                 temp_data[k * priv_state->num_regions_in_slice + i][6] = 0;
4144                 temp_data[k * priv_state->num_regions_in_slice + i][7] = (unsigned int)(offset_y + region_start_table[GEN9_HEVC_ENC_REGION_START_Y_OFFSET + nearest_reg] + ((i * offset_delta) >> 1));
4145             }
4146         } else {
4147             int cur_lcu_pel_y = region_start_table[GEN9_HEVC_ENC_REGION_START_Y_OFFSET +
4148                                                    (k * priv_state->num_regions_in_slice)] << 5;
4149             int ts_width = (priv_state->picture_width + 16) >> 5;
4150             int ts_height = height;
4151             int offset_y = -4 * ((ts_width + 1) >> 1);
4152             int offset_delta = ((ts_width + ((ts_height - 1) << 1)) + (priv_state->num_regions_in_slice - 1)) / (priv_state->num_regions_in_slice);
4153
4154             for (i = 0; i < (int)num_regions; i++) {
4155                 if (region_start_table[i] == 0) {
4156                     int delta = cur_lcu_pel_y - (region_start_table[GEN9_HEVC_ENC_REGION_START_Y_OFFSET + i] << 5);
4157
4158                     if (delta >= 0) {
4159                         if (delta < min_delta) {
4160                             min_delta = delta;
4161                             nearest_reg = i;
4162                         }
4163                     }
4164                 }
4165
4166                 offset_to_the_region_start[k] = 2 * region_start_table[GEN9_HEVC_ENC_REGION_START_Y_OFFSET + nearest_reg];
4167             }
4168
4169             for (i = 0; i < priv_state->num_regions_in_slice; i++) {
4170                 int tmp_y = 2 * region_start_table[GEN9_HEVC_ENC_REGION_START_Y_OFFSET + (nearest_reg + priv_state->num_regions_in_slice)];
4171
4172                 temp_data[k * priv_state->num_regions_in_slice + i][0] = region_start_table[nearest_reg + i];
4173                 temp_data[k * priv_state->num_regions_in_slice + i][1] = 2 * region_start_table[GEN9_HEVC_ENC_REGION_START_Y_OFFSET + (nearest_reg + i)];
4174                 temp_data[k * priv_state->num_regions_in_slice + i][2] = 2 * region_start_table[GEN9_HEVC_ENC_REGION_START_Y_OFFSET + nearest_reg];
4175                 temp_data[k * priv_state->num_regions_in_slice + i][3] = (unsigned int)((tmp_y != 0) ? tmp_y : (priv_state->picture_height) >> 4);
4176                 temp_data[k * priv_state->num_regions_in_slice + i][4] = offset_to_the_region_start[k] & 0x0FFFF;
4177                 temp_data[k * priv_state->num_regions_in_slice + i][5] = 0;
4178                 temp_data[k * priv_state->num_regions_in_slice + i][6] = 0;
4179                 temp_data[k * priv_state->num_regions_in_slice + i][7] = (unsigned int)(offset_y + 4 * region_start_table[GEN9_HEVC_ENC_REGION_START_Y_OFFSET + nearest_reg] + (4 * ((i * offset_delta) >> 1)));
4180             }
4181         }
4182     }
4183
4184     if (priv_state->walking_pattern_26)
4185         gen9_hevc_init_object_walker_26(priv_state, gpe_context, &param->gpe_param,
4186                                         priv_state->num_regions_in_slice, max_height,
4187                                         priv_state->use_hw_scoreboard,
4188                                         priv_state->use_hw_non_stalling_scoreborad);
4189     else
4190         gen9_hevc_init_object_walker_26z(priv_state, gpe_context, &param->gpe_param,
4191                                          priv_state->num_regions_in_slice, max_height,
4192                                          priv_state->use_hw_scoreboard,
4193                                          priv_state->use_hw_non_stalling_scoreborad);
4194
4195     p_region = (gen9_hevc_mbenc_control_region *)i965_map_gpe_resource(&priv_ctx->res_con_corrent_thread_buffer);
4196     if (!p_region)
4197         return;
4198
4199     memset((void *)p_region, 0, sizeof(*p_region) * GEN9_HEVC_ENC_CONCURRENT_SURFACE_HEIGHT);
4200
4201     for (i = 0; i < 1024 ; i += 64)
4202         memcpy(((unsigned char *)p_region) + i, (unsigned char *)temp_data[i / 64], 32);
4203
4204     param->max_height_in_region = priv_state->walking_pattern_26 ? max_height : max_height * 2;;
4205     param->num_region = num_regions;
4206     param->num_units_in_region = (frame_width_in_units + 2 * (max_height - 1) + priv_state->num_regions_in_slice - 1) /
4207                                  priv_state->num_regions_in_slice;
4208
4209     i965_unmap_gpe_resource(&priv_ctx->res_con_corrent_thread_buffer);
4210 }
4211
4212 static const char hevc_qpc_table[22] = {
4213     29, 30, 31, 32, 32, 33, 34, 34, 35, 35, 36, 36, 37, 37, 37, 38, 38, 38, 39, 39, 39, 39
4214 };
4215
4216 static void
4217 gen9_hevc_8x8_b_mbenc_set_curbe(VADriverContextP ctx,
4218                                 struct encode_state *encode_state,
4219                                 struct intel_encoder_context *encoder_context,
4220                                 struct i965_gpe_context *gpe_context,
4221                                 struct gen9_hevc_walking_pattern_parameter *param)
4222 {
4223     struct encoder_vme_mfc_context *vme_context = NULL;
4224     struct gen9_hevc_encoder_context *priv_ctx = NULL;
4225     struct generic_enc_codec_state *generic_state = NULL;
4226     struct gen9_hevc_encoder_state *priv_state = NULL;
4227     gen9_hevc_mbenc_b_mb_enc_curbe_data *cmd = NULL;
4228     VAEncSequenceParameterBufferHEVC *seq_param = NULL;
4229     VAEncPictureParameterBufferHEVC *pic_param = NULL;
4230     VAEncSliceParameterBufferHEVC *slice_param = NULL;
4231     unsigned int slice_qp = 0, slice_type = HEVC_SLICE_I;
4232     unsigned char mode_cost[12], mv_cost[8], mode_cost_sp;
4233     unsigned char forward_trans_thd[7];
4234     unsigned int simplest_intra_inter_threshold;
4235     int transform_8x8_mode_flag = 1;
4236     int qp_bd_offset_c, q_pi, qp_c;
4237     void *default_curbe_ptr = NULL;
4238     int default_curbe_size = 0;
4239     int max_sp_len = 57;
4240     int bti_idx = 0;
4241
4242     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
4243     priv_ctx = (struct gen9_hevc_encoder_context *)vme_context->private_enc_ctx;
4244     generic_state = (struct generic_enc_codec_state *)vme_context->generic_enc_state;
4245     priv_state = (struct gen9_hevc_encoder_state *)vme_context->private_enc_state;
4246     seq_param = (VAEncSequenceParameterBufferHEVC *)encode_state->seq_param_ext->buffer;
4247     pic_param = (VAEncPictureParameterBufferHEVC *)encode_state->pic_param_ext->buffer;
4248     slice_param = (VAEncSliceParameterBufferHEVC *)encode_state->slice_params_ext[0]->buffer;
4249
4250     slice_qp = pic_param->pic_init_qp + slice_param->slice_qp_delta;
4251     slice_type = priv_state->picture_coding_type;
4252
4253     if (priv_state->tu_mode == HEVC_TU_BEST_SPEED) {
4254         gen9_hevc_set_lambda_tables(priv_ctx, slice_type,
4255                                     HEVC_ENC_INTRA_TRANS_HAAR);
4256
4257         max_sp_len = 25;
4258     }
4259
4260     gen9_hevc_mbenc_set_costs(priv_ctx, slice_type, HEVC_ENC_INTRA_TRANS_REGULAR, slice_qp,
4261                               mode_cost, mv_cost, &mode_cost_sp,
4262                               &simplest_intra_inter_threshold);
4263
4264     gen9_hevc_set_forward_coeff_thd(forward_trans_thd, slice_qp);
4265
4266     gen9_hevc_get_b_mbenc_default_curbe(priv_state->tu_mode,
4267                                         slice_type, &default_curbe_ptr,
4268                                         &default_curbe_size);
4269
4270     cmd = i965_gpe_context_map_curbe(gpe_context);
4271     if (!cmd)
4272         return;
4273
4274     memcpy((void *)cmd, default_curbe_ptr, default_curbe_size);
4275
4276     cmd->dw0.adaptive_en = 1;
4277     cmd->dw0.t_8x8_flag_for_inter_en = transform_8x8_mode_flag;
4278     cmd->dw2.pic_width = priv_state->width_in_mb;
4279     cmd->dw2.len_sp = max_sp_len;
4280     cmd->dw3.src_access = 0;
4281     cmd->dw3.ref_access = 0;
4282     cmd->dw3.ft_enable = (HEVC_ENC_FTQ_BASED_SKIP[priv_state->tu_mode] >> 1) & 0x01;
4283     cmd->dw4.pic_height_minus1 = priv_state->height_in_mb - 1;
4284     cmd->dw4.hme_enable = !!generic_state->hme_enabled;
4285     cmd->dw4.slice_type = slice_type;
4286     cmd->dw4.use_actual_ref_qp_value = 0;
4287     cmd->dw6.frame_width = priv_state->width_in_mb * 16;
4288     cmd->dw6.frame_height = priv_state->height_in_mb * 16;
4289     cmd->dw7.intra_part_mask = 3;
4290     cmd->dw8.mode0_cost = mode_cost[0];
4291     cmd->dw8.mode1_cost = mode_cost[1];
4292     cmd->dw8.mode2_cost = mode_cost[2];
4293     cmd->dw8.mode3_cost = mode_cost[3];
4294     cmd->dw9.mode4_cost = mode_cost[4];
4295     cmd->dw9.mode5_cost = mode_cost[5];
4296     cmd->dw9.mode6_cost = mode_cost[6];
4297     cmd->dw9.mode7_cost = mode_cost[7];
4298     cmd->dw10.mode8_cost = mode_cost[8];
4299     cmd->dw10.mode9_cost = mode_cost[9];
4300     cmd->dw10.ref_id_cost = mode_cost[10];
4301     cmd->dw10.chroma_intra_mode_cost = mode_cost[11];
4302     cmd->dw11.mv0_cost = mv_cost[0];
4303     cmd->dw11.mv1_cost = mv_cost[1];
4304     cmd->dw11.mv2_cost = mv_cost[2];
4305     cmd->dw11.mv3_cost = mv_cost[3];
4306     cmd->dw12.mv4_cost = mv_cost[4];
4307     cmd->dw12.mv5_cost = mv_cost[5];
4308     cmd->dw12.mv6_cost = mv_cost[6];
4309     cmd->dw12.mv7_cost = mv_cost[7];
4310
4311     cmd->dw13.qp_prime_y = slice_qp;
4312     qp_bd_offset_c = 6 * priv_state->bit_depth_chroma_minus8;
4313     q_pi = CLAMP(-qp_bd_offset_c, 51, slice_qp + pic_param->pps_cb_qp_offset);
4314     qp_c = (q_pi < 30) ? q_pi : hevc_qpc_table[q_pi - 30];
4315     cmd->dw13.qp_prime_cb = qp_c;
4316     q_pi = CLAMP(-qp_bd_offset_c, 51, slice_qp + pic_param->pps_cr_qp_offset);
4317     qp_c = (q_pi < 30) ? q_pi : hevc_qpc_table[q_pi - 30];
4318     cmd->dw13.qp_prime_cr = qp_c;
4319
4320     cmd->dw14.sic_fwd_trans_coeff_thread_0 = forward_trans_thd[0];
4321     cmd->dw14.sic_fwd_trans_coeff_thread_1 = forward_trans_thd[1];
4322     cmd->dw14.sic_fwd_trans_coeff_thread_2 = forward_trans_thd[2];
4323     cmd->dw15.sic_fwd_trans_coeff_thread_3 = forward_trans_thd[3];
4324     cmd->dw15.sic_fwd_trans_coeff_thread_4 = forward_trans_thd[4];
4325     cmd->dw15.sic_fwd_trans_coeff_thread_5 = forward_trans_thd[5];
4326     cmd->dw15.sic_fwd_trans_coeff_thread_6 = forward_trans_thd[6];
4327     cmd->dw32.skip_val =
4328         HEVC_ENC_SKIPVAL_B[cmd->dw3.block_based_skip_enable][transform_8x8_mode_flag][slice_qp];
4329
4330     if (priv_state->picture_coding_type == HEVC_SLICE_I)
4331         *(float *)&cmd->dw34.lambda_me = 0.0;
4332     else
4333         *(float *)&cmd->dw34.lambda_me = (float)priv_ctx->lambda_me_table[slice_type][slice_qp];
4334
4335     cmd->dw35.mode_cost_sp = mode_cost_sp;
4336     cmd->dw35.simp_intra_inter_threashold = simplest_intra_inter_threshold;
4337     cmd->dw36.num_refidx_l0_minus_one = slice_param->num_ref_idx_l0_active_minus1;
4338     cmd->dw36.num_refidx_l1_minus_one = slice_param->num_ref_idx_l1_active_minus1;
4339     cmd->dw36.brc_enable = !!generic_state->brc_enabled;
4340     cmd->dw36.lcu_brc_enable = !!priv_state->lcu_brc_enabled;
4341     cmd->dw36.power_saving = priv_state->power_saving;
4342     cmd->dw36.roi_enable = (priv_state->num_roi > 0);
4343     cmd->dw36.fast_surveillance_flag = priv_state->picture_coding_type == HEVC_SLICE_I ?
4344                                        0 : priv_state->video_surveillance_flag;
4345
4346     if (priv_state->picture_coding_type != HEVC_SLICE_I) {
4347         cmd->dw37.actual_qp_refid0_list0 = gen9_hevc_get_qp_from_ref_list(ctx, slice_param, 0, 0);
4348         cmd->dw37.actual_qp_refid1_list0 = gen9_hevc_get_qp_from_ref_list(ctx, slice_param, 0, 1);
4349         cmd->dw37.actual_qp_refid2_list0 = gen9_hevc_get_qp_from_ref_list(ctx, slice_param, 0, 2);
4350         cmd->dw37.actual_qp_refid3_list0 = gen9_hevc_get_qp_from_ref_list(ctx, slice_param, 0, 3);
4351
4352         if (priv_state->picture_coding_type == HEVC_SLICE_B) {
4353             cmd->dw39.actual_qp_refid0_list1 = gen9_hevc_get_qp_from_ref_list(ctx, slice_param, 1, 0);
4354             cmd->dw39.actual_qp_refid1_list1 = gen9_hevc_get_qp_from_ref_list(ctx, slice_param, 1, 1);
4355         }
4356     }
4357
4358     cmd->dw44.max_vmvr = 511 * 4;
4359     cmd->dw44.max_num_merge_candidates = slice_param->max_num_merge_cand;
4360
4361     if (priv_state->picture_coding_type != HEVC_SLICE_I) {
4362         cmd->dw44.max_num_ref_list0 = cmd->dw36.num_refidx_l0_minus_one + 1;
4363         cmd->dw44.max_num_ref_list1 = cmd->dw36.num_refidx_l1_minus_one + 1;
4364         cmd->dw45.temporal_mvp_enable_flag = slice_param->slice_fields.bits.slice_temporal_mvp_enabled_flag;
4365         cmd->dw45.hme_combine_len_pslice = 8;
4366         if (priv_state->picture_coding_type == HEVC_SLICE_B)
4367             cmd->dw45.hme_combine_len_bslice = 8;
4368     }
4369
4370     cmd->dw45.log2_parallel_merge_level = pic_param->log2_parallel_merge_level_minus2 + 2;
4371     cmd->dw46.log2_min_tu_size = seq_param->log2_min_transform_block_size_minus2 + 2;
4372     cmd->dw46.log2_max_tu_size = cmd->dw46.log2_min_tu_size + seq_param->log2_diff_max_min_transform_block_size;
4373     cmd->dw46.log2_min_cu_size = seq_param->log2_min_luma_coding_block_size_minus3 + 3;
4374     cmd->dw46.log2_max_cu_size = cmd->dw46.log2_min_cu_size + seq_param->log2_diff_max_min_luma_coding_block_size;
4375     cmd->dw47.num_regions_in_slice = priv_state->num_regions_in_slice;
4376     cmd->dw47.type_of_walking_pattern = priv_state->walking_pattern_26;
4377     cmd->dw47.chroma_flatness_check_flag = (priv_state->tu_mode == HEVC_TU_BEST_SPEED) ? 0 : 1;
4378     cmd->dw47.enable_intra_early_exit = (priv_state->tu_mode == HEVC_TU_RT_SPEED);
4379     cmd->dw47.skip_intra_krn_flag = (priv_state->tu_mode == HEVC_TU_BEST_SPEED);
4380     cmd->dw47.collocated_from_l0_flag = slice_param->slice_fields.bits.collocated_from_l0_flag;
4381     cmd->dw47.is_low_delay = priv_state->low_delay;
4382     cmd->dw47.screen_content_flag = !!pic_param->pic_fields.bits.screen_content_flag;
4383     cmd->dw47.multi_slice_flag = (encode_state->num_slice_params_ext > 1);
4384     cmd->dw47.arbitary_slice_flag = priv_state->arbitrary_num_mb_in_slice;
4385     cmd->dw47.num_region_minus1 = param->num_region - 1;
4386
4387     if (priv_state->picture_coding_type != HEVC_SLICE_I) {
4388         cmd->dw48.current_td_l0_0 = gen9_hevc_get_poc_diff_from_ref_list(pic_param, slice_param, 0, 0);
4389         cmd->dw48.current_td_l0_1 = gen9_hevc_get_poc_diff_from_ref_list(pic_param, slice_param, 0, 1);
4390         cmd->dw49.current_td_l0_2 = gen9_hevc_get_poc_diff_from_ref_list(pic_param, slice_param, 0, 2);
4391         cmd->dw49.current_td_l0_3 = gen9_hevc_get_poc_diff_from_ref_list(pic_param, slice_param, 0, 3);
4392
4393         if (priv_state->picture_coding_type == HEVC_SLICE_B) {
4394             cmd->dw50.current_td_l1_0 = gen9_hevc_get_poc_diff_from_ref_list(pic_param, slice_param, 1, 0);
4395             cmd->dw50.current_td_l1_1 = gen9_hevc_get_poc_diff_from_ref_list(pic_param, slice_param, 1, 1);
4396         }
4397     }
4398
4399     cmd->dw52.num_of_units_in_region = param->num_units_in_region;
4400     cmd->dw52.max_height_in_region = param->max_height_in_region;
4401
4402     if (priv_state->rolling_intra_refresh) {
4403         cmd->dw35.widi_intra_refresh_en = 1;
4404         cmd->dw35.widi_first_intra_refresh = priv_state->widi_first_intra_refresh;
4405         cmd->dw35.half_update_mixed_lcu = 0;
4406         cmd->dw35.enable_rolling_intra = 1;
4407         cmd->dw38.widi_num_frame_in_gob = priv_state->widi_frame_num_in_gob;
4408         cmd->dw38.widi_num_intra_refresh_off_frames = priv_state->widi_frame_num_without_intra_refresh;
4409         cmd->dw51.widi_intra_refresh_qp_delta = priv_state->widi_intra_refresh_qp_delta;
4410         cmd->dw51.widi_intra_refresh_mb_num = priv_state->widi_intra_insertion_location;
4411         cmd->dw51.widi_intra_refresh_unit_in_mb = priv_state->widi_intra_insertion_size;
4412         cmd->dw53.widi_intra_refresh_ref_height = 40;
4413         cmd->dw53.widi_intra_refresh_ref_width = 48;
4414
4415         priv_state->widi_first_intra_refresh = 0;
4416         priv_state->widi_frame_num_without_intra_refresh = 0;
4417     } else if (priv_state->picture_coding_type != HEVC_SLICE_I)
4418         priv_state->widi_frame_num_without_intra_refresh++;
4419
4420     cmd->dw56.bti_cu_record = bti_idx++;
4421     cmd->dw57.bti_pak_cmd = bti_idx++;
4422     cmd->dw58.bti_src_y = bti_idx++;
4423     bti_idx++;
4424     cmd->dw59.bti_intra_dist = bti_idx++;
4425     cmd->dw60.bti_min_dist = bti_idx++;
4426     cmd->dw61.bti_hme_mv_pred_fwd_bwd_surf_index = bti_idx++;
4427     cmd->dw62.bti_hme_dist_surf_index = bti_idx++;
4428     cmd->dw63.bti_slice_map = bti_idx++;
4429     cmd->dw64.bti_vme_saved_uni_sic = bti_idx++;
4430     cmd->dw65.bti_simplest_intra = bti_idx++;
4431     cmd->dw66.bti_collocated_refframe = bti_idx++;
4432     cmd->dw67.bti_reserved = bti_idx++;
4433     cmd->dw68.bti_brc_input = bti_idx++;
4434     cmd->dw69.bti_lcu_qp = bti_idx++;
4435     cmd->dw70.bti_brc_data = bti_idx++;
4436     cmd->dw71.bti_vme_inter_prediction_surf_index = bti_idx++;
4437     bti_idx += 16;
4438
4439     if (priv_state->picture_coding_type == HEVC_SLICE_P) {
4440         cmd->dw72.bti_concurrent_thread_map = bti_idx++;
4441         cmd->dw73.bti_mb_data_cur_frame = bti_idx++;
4442         cmd->dw74.bti_mvp_cur_frame = bti_idx++;
4443         cmd->dw75.bti_debug = bti_idx++;
4444     } else {
4445         cmd->dw72.bti_vme_inter_prediction_b_surf_index = bti_idx++;
4446         bti_idx += 8;
4447
4448         cmd->dw73.bti_concurrent_thread_map = bti_idx++;
4449         cmd->dw74.bti_mb_data_cur_frame = bti_idx++;
4450         cmd->dw75.bti_mvp_cur_frame = bti_idx++;
4451         cmd->dw76.bti_debug = bti_idx++;
4452     }
4453
4454     i965_gpe_context_unmap_curbe(gpe_context);
4455 }
4456
4457 static void
4458 gen9_hevc_8x8_b_mbenc_set_surfaces(VADriverContextP ctx,
4459                                    struct encode_state *encode_state,
4460                                    struct intel_encoder_context *encoder_context,
4461                                    struct i965_gpe_context *gpe_context)
4462 {
4463     struct i965_driver_data *i965 = i965_driver_data(ctx);
4464     struct encoder_vme_mfc_context *vme_context = NULL;
4465     struct gen9_hevc_encoder_context *priv_ctx = NULL;
4466     struct gen9_hevc_encoder_state *priv_state = NULL;
4467     VAEncSliceParameterBufferHEVC *slice_param = NULL;
4468     VAEncPictureParameterBufferHEVC *pic_param = NULL;
4469     struct gen9_hevc_surface_priv *surface_priv = NULL;
4470     struct object_surface *obj_surface = NULL;
4471     dri_bo *collocated_mv_temporal_bo = NULL;
4472     int bti_idx = 0;
4473     int size = 0;
4474     int i = 0;
4475
4476     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
4477     priv_ctx = (struct gen9_hevc_encoder_context *)vme_context->private_enc_ctx;
4478     priv_state = (struct gen9_hevc_encoder_state *)vme_context->private_enc_state;
4479     pic_param = (VAEncPictureParameterBufferHEVC *)encode_state->pic_param_ext->buffer;
4480     slice_param = (VAEncSliceParameterBufferHEVC *)encode_state->slice_params_ext[0]->buffer;
4481
4482     if (priv_state->picture_coding_type != HEVC_SLICE_I &&
4483         slice_param->slice_fields.bits.slice_temporal_mvp_enabled_flag &&
4484         slice_param->slice_fields.bits.collocated_from_l0_flag) {
4485         if (pic_param->collocated_ref_pic_index != 0xff &&
4486             pic_param->collocated_ref_pic_index < GEN9_MAX_REF_SURFACES) {
4487             VASurfaceID idx = VA_INVALID_SURFACE;
4488
4489             idx = pic_param->reference_frames[pic_param->collocated_ref_pic_index].picture_id;
4490             obj_surface = SURFACE(idx);
4491             if (obj_surface) {
4492                 surface_priv = (struct gen9_hevc_surface_priv *)obj_surface->private_data;
4493                 if (surface_priv)
4494                     collocated_mv_temporal_bo = surface_priv->motion_vector_temporal_bo;
4495             }
4496         }
4497     }
4498
4499     size = priv_state->width_in_cu * priv_state->height_in_cu *
4500            priv_state->cu_record_size;
4501     gen9_hevc_set_gpe_1d_surface(ctx, priv_ctx, gpe_context,
4502                                  HEVC_ENC_SURFACE_CU_RECORD, bti_idx++,
4503                                  0, size, priv_state->mb_data_offset, NULL, NULL);
4504
4505     size = priv_state->mb_data_offset;
4506     gen9_hevc_set_gpe_1d_surface(ctx, priv_ctx, gpe_context,
4507                                  HEVC_ENC_SURFACE_HCP_PAK, bti_idx++,
4508                                  0, size, 0, NULL, NULL);
4509
4510     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
4511                                  HEVC_ENC_SURFACE_RAW_Y_UV, bti_idx++,
4512                                  1, 1, I965_SURFACEFORMAT_R8_UNORM,
4513                                  NULL, NULL);
4514     bti_idx++;
4515
4516     gen9_hevc_set_gpe_1d_surface(ctx, priv_ctx, gpe_context,
4517                                  HEVC_ENC_SURFACE_INTRA_DIST, bti_idx++,
4518                                  0, 0, 0, NULL, NULL);
4519
4520     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
4521                                  HEVC_ENC_SURFACE_MIN_DIST, bti_idx++,
4522                                  0, 1, I965_SURFACEFORMAT_R8_UNORM,
4523                                  NULL, NULL);
4524
4525     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
4526                                  HEVC_ENC_SURFACE_HME_MVP, bti_idx++,
4527                                  0, 1, I965_SURFACEFORMAT_R8_UNORM,
4528                                  NULL, NULL);
4529
4530     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
4531                                  HEVC_ENC_SURFACE_HME_DIST, bti_idx++,
4532                                  0, 1, I965_SURFACEFORMAT_R8_UNORM,
4533                                  NULL, NULL);
4534
4535     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
4536                                  HEVC_ENC_SURFACE_SLICE_MAP, bti_idx++,
4537                                  0, 1, I965_SURFACEFORMAT_R8_UNORM,
4538                                  NULL, NULL);
4539
4540     gen9_hevc_set_gpe_1d_surface(ctx, priv_ctx, gpe_context,
4541                                  HEVC_ENC_SURFACE_VME_UNI_SIC_DATA, bti_idx++,
4542                                  0, 0, 0, NULL, NULL);
4543
4544     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
4545                                  HEVC_ENC_SURFACE_SIMPLIFIED_INTRA, bti_idx++,
4546                                  0, 1, I965_SURFACEFORMAT_R8_UNORM,
4547                                  NULL, NULL);
4548
4549     if (collocated_mv_temporal_bo)
4550         gen9_hevc_set_gpe_1d_surface(ctx, priv_ctx, gpe_context,
4551                                      HEVC_ENC_SURFACE_COL_MB_MV, bti_idx++,
4552                                      0, 0, 0, NULL, collocated_mv_temporal_bo);
4553     else
4554         bti_idx++;
4555
4556     bti_idx++;
4557
4558     gen9_hevc_set_gpe_1d_surface(ctx, priv_ctx, gpe_context,
4559                                  HEVC_ENC_SURFACE_BRC_INPUT, bti_idx++,
4560                                  0, 0, 0, NULL, NULL);
4561
4562     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
4563                                  HEVC_ENC_SURFACE_LCU_QP, bti_idx++,
4564                                  0, 1, I965_SURFACEFORMAT_R8_UNORM,
4565                                  NULL, NULL);
4566
4567     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
4568                                  HEVC_ENC_SURFACE_BRC_DATA, bti_idx++,
4569                                  0, 1, I965_SURFACEFORMAT_R8_UNORM,
4570                                  NULL, NULL);
4571
4572     gen9_hevc_set_gpe_adv_surface(ctx, priv_ctx, gpe_context,
4573                                   HEVC_ENC_SURFACE_RAW_VME, bti_idx++,
4574                                   NULL);
4575
4576     for (i = 0; i < 8; i++) {
4577         if (i <= slice_param->num_ref_idx_l0_active_minus1)
4578             obj_surface = SURFACE(slice_param->ref_pic_list0[i].picture_id);
4579         else
4580             obj_surface = NULL;
4581
4582         if (obj_surface) {
4583             surface_priv = (struct gen9_hevc_surface_priv *)obj_surface->private_data;
4584
4585             gen9_hevc_set_gpe_adv_surface(ctx, priv_ctx, gpe_context,
4586                                           HEVC_ENC_SURFACE_REF_FRAME_VME, bti_idx++,
4587                                           surface_priv->surface_reff);
4588         } else
4589             bti_idx++;
4590
4591         if (i <= slice_param->num_ref_idx_l1_active_minus1)
4592             obj_surface = SURFACE(slice_param->ref_pic_list1[i].picture_id);
4593         else
4594             obj_surface = NULL;
4595
4596         if (obj_surface) {
4597             surface_priv = (struct gen9_hevc_surface_priv *)obj_surface->private_data;
4598
4599             gen9_hevc_set_gpe_adv_surface(ctx, priv_ctx, gpe_context,
4600                                           HEVC_ENC_SURFACE_REF_FRAME_VME, bti_idx++,
4601                                           surface_priv->surface_reff);
4602         } else
4603             bti_idx++;
4604     }
4605
4606     if (priv_state->picture_coding_type != HEVC_SLICE_P) {
4607         gen9_hevc_set_gpe_adv_surface(ctx, priv_ctx, gpe_context,
4608                                       HEVC_ENC_SURFACE_RAW_VME, bti_idx++,
4609                                       NULL);
4610
4611         for (i = 0; i < 4; i++) {
4612             if (i <= slice_param->num_ref_idx_l1_active_minus1)
4613                 obj_surface = SURFACE(slice_param->ref_pic_list1[i].picture_id);
4614             else
4615                 obj_surface = NULL;
4616
4617             if (obj_surface) {
4618                 surface_priv = (struct gen9_hevc_surface_priv *)obj_surface->private_data;
4619
4620                 gen9_hevc_set_gpe_adv_surface(ctx, priv_ctx, gpe_context,
4621                                               HEVC_ENC_SURFACE_REF_FRAME_VME, bti_idx++,
4622                                               surface_priv->surface_reff);
4623             } else
4624                 bti_idx++;
4625
4626             bti_idx++;
4627         }
4628     }
4629
4630     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
4631                                  HEVC_ENC_SURFACE_CONCURRENT_THREAD, bti_idx++,
4632                                  0, 1, I965_SURFACEFORMAT_R8_UNORM,
4633                                  NULL, NULL);
4634
4635     gen9_hevc_set_gpe_1d_surface(ctx, priv_ctx, gpe_context,
4636                                  HEVC_ENC_SURFACE_MB_MV_INDEX, bti_idx++,
4637                                  0, 0, 0, NULL, NULL);
4638
4639     gen9_hevc_set_gpe_1d_surface(ctx, priv_ctx, gpe_context,
4640                                  HEVC_ENC_SURFACE_MVP_INDEX, bti_idx++,
4641                                  0, 0, 0, NULL, NULL);
4642
4643     gen9_hevc_set_gpe_1d_surface(ctx, priv_ctx, gpe_context,
4644                                  HEVC_ENC_SURFACE_KERNEL_DEBUG, bti_idx++,
4645                                  0, 0, 0, NULL, NULL);
4646 }
4647
4648 static void
4649 gen9_hevc_8x8_b_mbenc(VADriverContextP ctx,
4650                       struct encode_state *encode_state,
4651                       struct intel_encoder_context *encoder_context)
4652 {
4653     struct encoder_vme_mfc_context *vme_context = NULL;
4654     struct gen9_hevc_encoder_context *priv_ctx = NULL;
4655     struct gen9_hevc_encoder_state *priv_state = NULL;
4656     struct gen9_hevc_walking_pattern_parameter param;
4657     struct i965_gpe_context *gpe_context = NULL;
4658     int media_state = HEVC_ENC_MEDIA_STATE_HEVC_B_MBENC;
4659     int gpe_idx = HEVC_MBENC_BENC_IDX;
4660
4661     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
4662     priv_ctx = (struct gen9_hevc_encoder_context *)vme_context->private_enc_ctx;
4663     priv_state = (struct gen9_hevc_encoder_state *)vme_context->private_enc_state;
4664
4665     if (priv_state->picture_coding_type == HEVC_SLICE_P)
4666         gpe_idx = priv_state->rolling_intra_refresh ? HEVC_MBENC_P_WIDI_IDX : HEVC_MBENC_PENC_IDX;
4667     else if (priv_state->picture_coding_type != HEVC_SLICE_I)
4668         gpe_idx = priv_state->rolling_intra_refresh ? HEVC_MBENC_MBENC_WIDI_IDX : HEVC_MBENC_BENC_IDX;
4669
4670     gpe_context = &priv_ctx->mbenc_context.gpe_contexts[gpe_idx];
4671
4672     gen8_gpe_context_init(ctx, gpe_context);
4673     gen9_gpe_reset_binding_table(ctx, gpe_context);
4674
4675     memset((void *)&param, 0, sizeof(param));
4676     gen9_hevc_set_control_region(ctx, encode_state, encoder_context, gpe_context, &param);
4677
4678     gen9_hevc_8x8_b_mbenc_set_curbe(ctx, encode_state, encoder_context, gpe_context, &param);
4679     gen9_hevc_8x8_b_mbenc_set_surfaces(ctx, encode_state, encoder_context, gpe_context);
4680     gen8_gpe_setup_interface_data(ctx, gpe_context);
4681
4682     param.gpe_param.color_count_minus1 = param.num_region - 1;
4683     gen9_hevc_run_object_walker(ctx, encoder_context, gpe_context, &param.gpe_param,
4684                                 media_state);
4685 }
4686
4687 static void
4688 gen9_hevc_8x8_pu_fmode_set_curbe(VADriverContextP ctx,
4689                                  struct encode_state *encode_state,
4690                                  struct intel_encoder_context *encoder_context,
4691                                  struct i965_gpe_context *gpe_context)
4692 {
4693     struct i965_driver_data *i965 = i965_driver_data(ctx);
4694     struct encoder_vme_mfc_context *vme_context = NULL;
4695     struct gen9_hevc_encoder_context *priv_ctx = NULL;
4696     struct generic_enc_codec_state *generic_state = NULL;
4697     struct gen9_hevc_encoder_state *priv_state = NULL;
4698     gen9_hevc_mbenc_8x8_pu_fmode_curbe_data *cmd = NULL;
4699     VAEncSequenceParameterBufferHEVC *seq_param = NULL;
4700     VAEncPictureParameterBufferHEVC *pic_param = NULL;
4701     VAEncSliceParameterBufferHEVC *slice_param = NULL;
4702     unsigned int slice_qp = 0;
4703     int bti_idx = 0;
4704
4705     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
4706     priv_ctx = (struct gen9_hevc_encoder_context *)vme_context->private_enc_ctx;
4707     generic_state = (struct generic_enc_codec_state *)vme_context->generic_enc_state;
4708     priv_state = (struct gen9_hevc_encoder_state *)vme_context->private_enc_state;
4709     seq_param = (VAEncSequenceParameterBufferHEVC *)encode_state->seq_param_ext->buffer;
4710     pic_param = (VAEncPictureParameterBufferHEVC *)encode_state->pic_param_ext->buffer;
4711     slice_param = (VAEncSliceParameterBufferHEVC *)encode_state->slice_params_ext[0]->buffer;
4712
4713     slice_qp = pic_param->pic_init_qp + slice_param->slice_qp_delta;
4714
4715     cmd = i965_gpe_context_map_curbe(gpe_context);
4716     if (!cmd)
4717         return;
4718
4719     memset((void *)cmd, 0, sizeof(*cmd));
4720
4721     cmd->dw0.frame_width = ALIGN(priv_state->picture_width, 16);
4722     cmd->dw0.frame_height = ALIGN(priv_state->picture_height, 16);
4723     cmd->dw1.slice_type = priv_state->picture_coding_type;
4724     cmd->dw1.pu_type = 2;
4725     cmd->dw1.pak_reording_flag = priv_state->picture_coding_type == HEVC_SLICE_I ? 1 : 0;
4726
4727     if (seq_param->log2_min_luma_coding_block_size_minus3 + 3 +
4728         seq_param->log2_diff_max_min_luma_coding_block_size == 6)
4729         cmd->dw1.lcu_type = 0;
4730     else
4731         cmd->dw1.lcu_type = 1;
4732
4733     cmd->dw1.screen_content_flag = !!pic_param->pic_fields.bits.screen_content_flag;
4734     cmd->dw1.enable_intra_early_exit = priv_state->tu_mode == HEVC_TU_RT_SPEED ?
4735                                        (priv_state->picture_coding_type == HEVC_SLICE_I ? 0 : 1) : 0;
4736     cmd->dw1.enable_debug_dump = 0;
4737     cmd->dw1.brc_enable = generic_state->brc_enabled;
4738     cmd->dw1.lcu_brc_enable = priv_state->lcu_brc_enabled;
4739     cmd->dw1.roi_enable = (priv_state->num_roi > 0);
4740     cmd->dw1.fast_surveillance_flag = priv_state->picture_coding_type == HEVC_SLICE_I ?
4741                                       0 : priv_state->video_surveillance_flag;
4742     cmd->dw1.enable_rolling_intra = priv_state->rolling_intra_refresh;
4743     cmd->dw1.widi_intra_refresh_en = priv_state->rolling_intra_refresh;
4744     cmd->dw1.half_update_mixed_lcu = 0;
4745     cmd->dw2.luma_lambda = priv_state->fixed_point_lambda_for_luma;
4746
4747     if (priv_state->picture_coding_type != HEVC_SLICE_I) {
4748         double lambda_md;
4749         float had_bias = 2.0f;
4750
4751         lambda_md = priv_ctx->lambda_md_table[cmd->dw1.slice_type][slice_qp];
4752         lambda_md = lambda_md * had_bias;
4753         cmd->dw3.lambda_for_dist_calculation = (unsigned int)(lambda_md * (1 << 10));
4754     }
4755
4756     cmd->dw4.mode_cost_for_8x8_pu_tu8 = 0;
4757     cmd->dw5.mode_cost_for_8x8_pu_tu4 = 0;
4758     cmd->dw6.satd_16x16_pu_threshold = MAX(200 * ((int)slice_qp - 12), 0);
4759     cmd->dw6.bias_factor_toward_8x8 = pic_param->pic_fields.bits.screen_content_flag ? 1024 : 1126 + 102;
4760     cmd->dw7.qp = slice_qp;
4761     cmd->dw7.qp_for_inter = 0;
4762     cmd->dw8.simplified_flag_for_inter = 0;
4763     cmd->dw8.kbl_control_flag = (IS_KBL(i965->intel.device_info) || IS_GLK(i965->intel.device_info));
4764     cmd->dw9.widi_intra_refresh_qp_delta = priv_state->widi_intra_refresh_qp_delta;
4765     cmd->dw9.widi_intra_refresh_mb_num = priv_state->widi_intra_insertion_location;
4766     cmd->dw9.widi_intra_refresh_unit_in_mb = priv_state->widi_intra_insertion_size;
4767
4768     cmd->dw16.bti_pak_object = bti_idx++;
4769     cmd->dw17.bti_vme_8x8_mode = bti_idx++;
4770     cmd->dw18.bti_intra_mode = bti_idx++;
4771     cmd->dw19.bti_pak_command = bti_idx++;
4772     cmd->dw20.bti_slice_map = bti_idx++;
4773     cmd->dw21.bti_intra_dist = bti_idx++;
4774     cmd->dw22.bti_brc_input = bti_idx++;
4775     cmd->dw23.bti_simplest_intra = bti_idx++;
4776     cmd->dw24.bti_lcu_qp_surface = bti_idx++;
4777     cmd->dw25.bti_brc_data = bti_idx++;
4778     cmd->dw26.bti_debug = bti_idx++;
4779
4780     i965_gpe_context_unmap_curbe(gpe_context);
4781 }
4782
4783 static void
4784 gen9_hevc_8x8_pu_fmode_set_surfaces(VADriverContextP ctx,
4785                                     struct encode_state *encode_state,
4786                                     struct intel_encoder_context *encoder_context,
4787                                     struct i965_gpe_context *gpe_context)
4788 {
4789     struct encoder_vme_mfc_context *vme_context = NULL;
4790     struct gen9_hevc_encoder_context *priv_ctx = NULL;
4791     struct gen9_hevc_encoder_state *priv_state = NULL;
4792     int bti_idx = 0;
4793     int size = 0;
4794
4795     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
4796     priv_ctx = (struct gen9_hevc_encoder_context *)vme_context->private_enc_ctx;
4797     priv_state = (struct gen9_hevc_encoder_state *)vme_context->private_enc_state;
4798
4799     size = priv_state->width_in_cu * priv_state->height_in_cu *
4800            priv_state->cu_record_size;
4801     gen9_hevc_set_gpe_1d_surface(ctx, priv_ctx, gpe_context,
4802                                  HEVC_ENC_SURFACE_CU_RECORD, bti_idx++,
4803                                  0, size, priv_state->mb_data_offset, NULL, NULL);
4804
4805     gen9_hevc_set_gpe_1d_surface(ctx, priv_ctx, gpe_context,
4806                                  HEVC_ENC_SURFACE_VME_8x8, bti_idx++,
4807                                  0, 0, 0, NULL, NULL);
4808
4809     gen9_hevc_set_gpe_1d_surface(ctx, priv_ctx, gpe_context,
4810                                  HEVC_ENC_SURFACE_INTRA_MODE, bti_idx++,
4811                                  0, 0, 0, NULL, NULL);
4812
4813     size = priv_state->mb_data_offset;
4814     gen9_hevc_set_gpe_1d_surface(ctx, priv_ctx, gpe_context,
4815                                  HEVC_ENC_SURFACE_HCP_PAK, bti_idx++,
4816                                  0, size, 0, NULL, NULL);
4817
4818     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
4819                                  HEVC_ENC_SURFACE_SLICE_MAP, bti_idx++,
4820                                  0, 1, I965_SURFACEFORMAT_R8_UNORM,
4821                                  NULL, NULL);
4822
4823     gen9_hevc_set_gpe_1d_surface(ctx, priv_ctx, gpe_context,
4824                                  HEVC_ENC_SURFACE_INTRA_DIST, bti_idx++,
4825                                  0, 0, 0, NULL, NULL);
4826
4827     gen9_hevc_set_gpe_1d_surface(ctx, priv_ctx, gpe_context,
4828                                  HEVC_ENC_SURFACE_BRC_INPUT, bti_idx++,
4829                                  0, 0, 0, NULL, NULL);
4830
4831     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
4832                                  HEVC_ENC_SURFACE_SIMPLIFIED_INTRA, bti_idx++,
4833                                  0, 1, I965_SURFACEFORMAT_R8_UNORM,
4834                                  NULL, NULL);
4835
4836     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
4837                                  HEVC_ENC_SURFACE_LCU_QP, bti_idx++,
4838                                  0, 1, I965_SURFACEFORMAT_R8_UNORM,
4839                                  NULL, NULL);
4840
4841     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
4842                                  HEVC_ENC_SURFACE_BRC_DATA, bti_idx++,
4843                                  0, 1, I965_SURFACEFORMAT_R8_UNORM,
4844                                  NULL, NULL);
4845
4846     gen9_hevc_set_gpe_1d_surface(ctx, priv_ctx, gpe_context,
4847                                  HEVC_ENC_SURFACE_KERNEL_DEBUG, bti_idx++,
4848                                  0, 0, 0, NULL, NULL);
4849 }
4850
4851 static void
4852 gen9_hevc_8x8_pu_fmode(VADriverContextP ctx,
4853                        struct encode_state *encode_state,
4854                        struct intel_encoder_context *encoder_context)
4855 {
4856     struct encoder_vme_mfc_context *vme_context = NULL;
4857     struct gen9_hevc_encoder_context *priv_ctx = NULL;
4858     struct gen9_hevc_encoder_state *priv_state = NULL;
4859     struct gpe_media_object_walker_parameter param;
4860     struct hevc_enc_kernel_walker_parameter hevc_walker_param;
4861     struct i965_gpe_context *gpe_context = NULL;
4862     int media_state = HEVC_ENC_MEDIA_STATE_8x8_PU_FMODE;
4863
4864     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
4865     priv_ctx = (struct gen9_hevc_encoder_context *)vme_context->private_enc_ctx;
4866     priv_state = (struct gen9_hevc_encoder_state *)vme_context->private_enc_state;
4867
4868     gpe_context = &priv_ctx->mbenc_context.gpe_contexts[HEVC_MBENC_8x8FMODE_IDX];
4869
4870     gen8_gpe_context_init(ctx, gpe_context);
4871     gen9_gpe_reset_binding_table(ctx, gpe_context);
4872     gen9_hevc_8x8_pu_fmode_set_curbe(ctx, encode_state, encoder_context, gpe_context);
4873     gen9_hevc_8x8_pu_fmode_set_surfaces(ctx, encode_state, encoder_context, gpe_context);
4874     gen8_gpe_setup_interface_data(ctx, gpe_context);
4875
4876     memset((void *)&hevc_walker_param, 0, sizeof(hevc_walker_param));
4877     hevc_walker_param.resolution_x = priv_state->width_in_lcu;
4878     hevc_walker_param.resolution_y = priv_state->height_in_lcu;
4879     hevc_walker_param.no_dependency = 1;
4880     gen9_hevc_init_object_walker(&hevc_walker_param, &param);
4881     gen9_hevc_run_object_walker(ctx, encoder_context, gpe_context, &param,
4882                                 media_state);
4883 }
4884
4885 static void
4886 gen9_hevc_8x8_pu_mode_set_curbe(VADriverContextP ctx,
4887                                 struct encode_state *encode_state,
4888                                 struct intel_encoder_context *encoder_context,
4889                                 struct i965_gpe_context *gpe_context)
4890 {
4891     struct encoder_vme_mfc_context *vme_context = NULL;
4892     struct generic_enc_codec_state *generic_state = NULL;
4893     struct gen9_hevc_encoder_state *priv_state = NULL;
4894     gen9_hevc_mbenc_8x8_pu_curbe_data *cmd = NULL;
4895     VAEncSequenceParameterBufferHEVC *seq_param = NULL;
4896     VAEncPictureParameterBufferHEVC *pic_param = NULL;
4897     VAEncSliceParameterBufferHEVC *slice_param = NULL;
4898     int bti_idx = 0;
4899
4900     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
4901     generic_state = (struct generic_enc_codec_state *)vme_context->generic_enc_state;
4902     priv_state = (struct gen9_hevc_encoder_state *)vme_context->private_enc_state;
4903     seq_param = (VAEncSequenceParameterBufferHEVC *)encode_state->seq_param_ext->buffer;
4904     pic_param = (VAEncPictureParameterBufferHEVC *)encode_state->pic_param_ext->buffer;
4905     slice_param = (VAEncSliceParameterBufferHEVC *)encode_state->slice_params_ext[0]->buffer;
4906
4907     cmd = i965_gpe_context_map_curbe(gpe_context);
4908     if (!cmd)
4909         return;
4910
4911     memset((void *)cmd, 0, sizeof(*cmd));
4912
4913     cmd->dw0.frame_width = ALIGN(priv_state->picture_width, 16);
4914     cmd->dw0.frame_height = ALIGN(priv_state->picture_height, 16);
4915     cmd->dw1.slice_type = priv_state->picture_coding_type;
4916     cmd->dw1.pu_type = 2;
4917     cmd->dw1.dc_filter_flag = 1;
4918     cmd->dw1.angle_refine_flag = 1;
4919     if (seq_param->log2_min_luma_coding_block_size_minus3 + 3 +
4920         seq_param->log2_diff_max_min_luma_coding_block_size == 6)
4921         cmd->dw1.lcu_type = 0;
4922     else
4923         cmd->dw1.lcu_type = 1;
4924     cmd->dw1.screen_content_flag = !!pic_param->pic_fields.bits.screen_content_flag;
4925     cmd->dw1.enable_intra_early_exit = priv_state->tu_mode == HEVC_TU_RT_SPEED ?
4926                                        (priv_state->picture_coding_type == HEVC_SLICE_I ? 0 : 1) : 0;
4927     cmd->dw1.enable_debug_dump = 0;
4928     cmd->dw1.brc_enable = generic_state->brc_enabled;
4929     cmd->dw1.lcu_brc_enable = priv_state->lcu_brc_enabled;
4930     cmd->dw1.roi_enable = (priv_state->num_roi > 0);
4931     cmd->dw1.fast_surveillance_flag = priv_state->picture_coding_type == HEVC_SLICE_I ?
4932                                       0 : priv_state->video_surveillance_flag;
4933     if (priv_state->rolling_intra_refresh) {
4934         cmd->dw1.enable_rolling_intra = 1;
4935         cmd->dw1.widi_intra_refresh_en = 1;
4936         cmd->dw1.half_update_mixed_lcu = 0;
4937
4938         cmd->dw5.widi_intra_refresh_mb_num = priv_state->widi_intra_insertion_location;
4939         cmd->dw5.widi_intra_refresh_mb_num = priv_state->widi_intra_insertion_location;
4940         cmd->dw5.widi_intra_refresh_unit_in_mb = priv_state->widi_intra_insertion_size;
4941
4942         cmd->dw1.qp_value = pic_param->pic_init_qp + slice_param->slice_qp_delta;
4943     }
4944
4945     cmd->dw2.luma_lambda = priv_state->fixed_point_lambda_for_luma;
4946     cmd->dw3.chroma_lambda = priv_state->fixed_point_lambda_for_chroma;
4947     cmd->dw4.simplified_flag_for_inter = 0;
4948     cmd->dw4.harr_trans_form_flag = priv_state->picture_coding_type == HEVC_SLICE_I ? 0 : 1;
4949
4950     cmd->dw8.bti_src_y = bti_idx++;
4951     bti_idx++;
4952     cmd->dw9.bti_slice_map = bti_idx++;
4953     cmd->dw10.bti_vme_8x8_mode = bti_idx++;
4954     cmd->dw11.bti_intra_mode = bti_idx++;
4955     cmd->dw12.bti_brc_input = bti_idx++;
4956     cmd->dw13.bti_simplest_intra = bti_idx++;
4957     cmd->dw14.bti_lcu_qp_surface = bti_idx++;
4958     cmd->dw15.bti_brc_data = bti_idx++;
4959     cmd->dw16.bti_debug = bti_idx++;
4960
4961     i965_gpe_context_unmap_curbe(gpe_context);
4962 }
4963
4964 static void
4965 gen9_hevc_8x8_pu_mode_set_surfaces(VADriverContextP ctx,
4966                                    struct encode_state *encode_state,
4967                                    struct intel_encoder_context *encoder_context,
4968                                    struct i965_gpe_context *gpe_context)
4969 {
4970     struct encoder_vme_mfc_context *vme_context = NULL;
4971     struct gen9_hevc_encoder_context *priv_ctx = NULL;
4972     int bti_idx = 0;
4973
4974     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
4975     priv_ctx = (struct gen9_hevc_encoder_context *)vme_context->private_enc_ctx;
4976
4977     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
4978                                  HEVC_ENC_SURFACE_RAW_Y_UV, bti_idx++,
4979                                  1, 1, I965_SURFACEFORMAT_R8_UNORM,
4980                                  NULL, NULL);
4981     bti_idx++;
4982
4983     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
4984                                  HEVC_ENC_SURFACE_SLICE_MAP, bti_idx++,
4985                                  0, 1, I965_SURFACEFORMAT_R8_UNORM,
4986                                  NULL, NULL);
4987
4988     gen9_hevc_set_gpe_1d_surface(ctx, priv_ctx, gpe_context,
4989                                  HEVC_ENC_SURFACE_VME_8x8, bti_idx++,
4990                                  0, 0, 0, NULL, NULL);
4991
4992     gen9_hevc_set_gpe_1d_surface(ctx, priv_ctx, gpe_context,
4993                                  HEVC_ENC_SURFACE_INTRA_MODE, bti_idx++,
4994                                  0, 0, 0, NULL, NULL);
4995
4996     gen9_hevc_set_gpe_1d_surface(ctx, priv_ctx, gpe_context,
4997                                  HEVC_ENC_SURFACE_BRC_INPUT, bti_idx++,
4998                                  0, 0, 0, NULL, NULL);
4999
5000     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
5001                                  HEVC_ENC_SURFACE_SIMPLIFIED_INTRA, bti_idx++,
5002                                  0, 1, I965_SURFACEFORMAT_R8_UNORM,
5003                                  NULL, NULL);
5004
5005     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
5006                                  HEVC_ENC_SURFACE_LCU_QP, bti_idx++,
5007                                  0, 1, I965_SURFACEFORMAT_R8_UNORM,
5008                                  NULL, NULL);
5009
5010     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
5011                                  HEVC_ENC_SURFACE_BRC_DATA, bti_idx++,
5012                                  0, 1, I965_SURFACEFORMAT_R8_UNORM,
5013                                  NULL, NULL);
5014
5015     gen9_hevc_set_gpe_1d_surface(ctx, priv_ctx, gpe_context,
5016                                  HEVC_ENC_SURFACE_KERNEL_DEBUG, bti_idx++,
5017                                  0, 0, 0, NULL, NULL);
5018 }
5019
5020 static void
5021 gen9_hevc_8x8_pu_mode(VADriverContextP ctx,
5022                       struct encode_state *encode_state,
5023                       struct intel_encoder_context *encoder_context)
5024 {
5025     struct encoder_vme_mfc_context *vme_context = NULL;
5026     struct gen9_hevc_encoder_context *priv_ctx = NULL;
5027     struct gen9_hevc_encoder_state *priv_state = NULL;
5028     struct gpe_media_object_walker_parameter param;
5029     struct hevc_enc_kernel_walker_parameter hevc_walker_param;
5030     struct i965_gpe_context *gpe_context = NULL;
5031     int media_state = HEVC_ENC_MEDIA_STATE_8x8_PU;
5032
5033     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
5034     priv_ctx = (struct gen9_hevc_encoder_context *)vme_context->private_enc_ctx;
5035     priv_state = (struct gen9_hevc_encoder_state *)vme_context->private_enc_state;
5036
5037     gpe_context = &priv_ctx->mbenc_context.gpe_contexts[HEVC_MBENC_8x8PU_IDX];
5038
5039     gen8_gpe_context_init(ctx, gpe_context);
5040     gen9_gpe_reset_binding_table(ctx, gpe_context);
5041     gen9_hevc_8x8_pu_mode_set_curbe(ctx, encode_state, encoder_context, gpe_context);
5042     gen9_hevc_8x8_pu_mode_set_surfaces(ctx, encode_state, encoder_context, gpe_context);
5043     gen8_gpe_setup_interface_data(ctx, gpe_context);
5044
5045     memset((void *)&hevc_walker_param, 0, sizeof(hevc_walker_param));
5046     hevc_walker_param.resolution_x = ALIGN(priv_state->picture_width, 16) >> 3;
5047     hevc_walker_param.resolution_y = ALIGN(priv_state->picture_height, 16) >> 3;
5048     hevc_walker_param.no_dependency = 1;
5049     gen9_hevc_init_object_walker(&hevc_walker_param, &param);
5050     gen9_hevc_run_object_walker(ctx, encoder_context, gpe_context, &param,
5051                                 media_state);
5052 }
5053
5054 static void
5055 gen9_hevc_16x16_pu_mode_set_curbe(VADriverContextP ctx,
5056                                   struct encode_state *encode_state,
5057                                   struct intel_encoder_context *encoder_context,
5058                                   struct i965_gpe_context *gpe_context)
5059 {
5060     struct encoder_vme_mfc_context *vme_context = NULL;
5061     struct gen9_hevc_encoder_context *priv_ctx = NULL;
5062     struct generic_enc_codec_state *generic_state = NULL;
5063     struct gen9_hevc_encoder_state *priv_state = NULL;
5064     gen9_hevc_enc_16x16_pu_curbe_data *cmd = NULL;
5065     VAEncSequenceParameterBufferHEVC *seq_param = NULL;
5066     VAEncPictureParameterBufferHEVC *pic_param = NULL;
5067     VAEncSliceParameterBufferHEVC *slice_param = NULL;
5068     double squred_lambda, qp_lambda, lambda_scaling_factor;
5069     unsigned int slice_qp = 0, slice_type = HEVC_SLICE_I;
5070     unsigned int new_point_lambda_for_luma;
5071     unsigned char mode_cost[12];
5072     int bti_idx = 0;
5073
5074     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
5075     priv_ctx = (struct gen9_hevc_encoder_context *)vme_context->private_enc_ctx;
5076     generic_state = (struct generic_enc_codec_state *)vme_context->generic_enc_state;
5077     priv_state = (struct gen9_hevc_encoder_state *)vme_context->private_enc_state;
5078     seq_param = (VAEncSequenceParameterBufferHEVC *)encode_state->seq_param_ext->buffer;
5079     pic_param = (VAEncPictureParameterBufferHEVC *)encode_state->pic_param_ext->buffer;
5080     slice_param = (VAEncSliceParameterBufferHEVC *)encode_state->slice_params_ext[0]->buffer;
5081
5082     slice_qp = pic_param->pic_init_qp + slice_param->slice_qp_delta;
5083     slice_type = priv_state->picture_coding_type;
5084
5085     lambda_scaling_factor = 0.46 + slice_qp - 22;
5086     if (lambda_scaling_factor < 0)
5087         lambda_scaling_factor = 0.46;
5088     else if (lambda_scaling_factor > 15)
5089         lambda_scaling_factor = 15;
5090
5091     squred_lambda = lambda_scaling_factor * pow(2.0, ((double)slice_qp - 12.0) / 6);
5092     priv_state->fixed_point_lambda_for_luma = (unsigned int)(squred_lambda * (1 << 10));
5093
5094     lambda_scaling_factor = 1.0;
5095     qp_lambda = priv_ctx->lambda_md_table[slice_type][slice_qp];
5096     squred_lambda = qp_lambda * qp_lambda;
5097     priv_state->fixed_point_lambda_for_chroma = (unsigned int)(lambda_scaling_factor * squred_lambda * (1 << 10));
5098
5099     qp_lambda = sqrt(0.57 * pow(2.0, ((double)slice_qp - 12.0) / 3));
5100     squred_lambda = qp_lambda * qp_lambda;
5101     new_point_lambda_for_luma = (unsigned int)(squred_lambda * (1 << 10));
5102
5103     gen9_hevc_mbenc_set_costs(priv_ctx, slice_type, HEVC_ENC_INTRA_TRANS_HAAR, slice_qp,
5104                               mode_cost, NULL, NULL, NULL);
5105
5106     cmd = i965_gpe_context_map_curbe(gpe_context);
5107     if (!cmd)
5108         return;
5109
5110     memset((void *)cmd, 0, sizeof(*cmd));
5111
5112     cmd->dw0.frame_width = ALIGN(priv_state->picture_width, 16);
5113     cmd->dw0.frame_height = ALIGN(priv_state->picture_height, 16);
5114     cmd->dw1.log2_min_cu_size = seq_param->log2_min_luma_coding_block_size_minus3 + 3;
5115     cmd->dw1.log2_max_cu_size = cmd->dw1.log2_min_cu_size + seq_param->log2_diff_max_min_luma_coding_block_size;
5116     cmd->dw1.log2_min_tu_size = seq_param->log2_min_transform_block_size_minus2 + 2;
5117     cmd->dw1.slice_qp = slice_qp;
5118     cmd->dw2.fixed_point_lambda_pred_mode = priv_state->fixed_point_lambda_for_chroma;
5119     cmd->dw3.lambda_scaling_factor = 1;
5120     cmd->dw3.slice_type = slice_type;
5121     cmd->dw3.enable_intra_early_exit = priv_state->tu_mode == HEVC_TU_RT_SPEED ?
5122                                        (priv_state->picture_coding_type == HEVC_SLICE_I ? 0 : 1) : 0;
5123     cmd->dw3.brc_enable = !!generic_state->brc_enabled;
5124     cmd->dw3.lcu_brc_enable = !!priv_state->lcu_brc_enabled;
5125     cmd->dw3.roi_enable = (priv_state->num_roi > 0);
5126     cmd->dw3.fast_surveillance_flag = priv_state->picture_coding_type == HEVC_SLICE_I ?
5127                                       0 : priv_state->video_surveillance_flag;
5128     cmd->dw3.enable_rolling_intra = priv_state->rolling_intra_refresh;
5129     cmd->dw3.widi_intra_refresh_en = priv_state->rolling_intra_refresh;
5130     cmd->dw3.half_update_mixed_lcu = 0;
5131     cmd->dw4.penalty_for_intra_8x8_non_dc_pred_mode = 0;
5132     cmd->dw4.intra_compute_type = 1;
5133     cmd->dw4.avc_intra_8x8_mask = 0;
5134     cmd->dw4.intra_sad_adjust = 2;
5135     cmd->dw5.fixed_point_lambda_cu_mode_for_cost_calculation = new_point_lambda_for_luma;
5136     cmd->dw6.screen_content_flag = !!pic_param->pic_fields.bits.screen_content_flag;
5137     cmd->dw7.mode_cost_intra_non_pred = mode_cost[0];
5138     cmd->dw7.mode_cost_intra_16x16 = mode_cost[1];
5139     cmd->dw7.mode_cost_intra_8x8 = mode_cost[2];
5140     cmd->dw7.mode_cost_intra_4x4 = mode_cost[3];
5141     cmd->dw8.fixed_point_lambda_cu_mode_for_luma = priv_state->fixed_point_lambda_for_luma;
5142
5143     if (priv_state->rolling_intra_refresh) {
5144         cmd->dw9.widi_intra_refresh_mb_num = priv_state->widi_intra_insertion_location;
5145         cmd->dw9.widi_intra_refresh_mb_num = priv_state->widi_intra_insertion_location;
5146         cmd->dw9.widi_intra_refresh_unit_in_mb = priv_state->widi_intra_insertion_size;
5147     }
5148
5149     cmd->dw10.simplified_flag_for_inter = 0;
5150     cmd->dw10.haar_transform_mode = priv_state->picture_coding_type == HEVC_SLICE_I ? 0 : 1;
5151
5152     cmd->dw16.bti_src_y = bti_idx++;
5153     bti_idx++;
5154     cmd->dw17.bti_sad_16x16_pu = bti_idx++;
5155     cmd->dw18.bti_pak_object = bti_idx++;
5156     cmd->dw19.bti_sad_32x32_pu_mode = bti_idx++;
5157     cmd->dw20.bti_vme_mode_8x8 = bti_idx++;
5158     cmd->dw21.bti_slice_map = bti_idx++;
5159     cmd->dw22.bti_vme_src = bti_idx++;
5160     cmd->dw23.bti_brc_input = bti_idx++;
5161     cmd->dw24.bti_simplest_intra = bti_idx++;
5162     cmd->dw25.bti_lcu_qp_surface = bti_idx++;
5163     cmd->dw26.bti_brc_data = bti_idx++;
5164     cmd->dw27.bti_debug = bti_idx++;
5165
5166     i965_gpe_context_unmap_curbe(gpe_context);
5167 }
5168
5169 static void
5170 gen9_hevc_16x16_pu_mode_set_surfaces(VADriverContextP ctx,
5171                                      struct encode_state *encode_state,
5172                                      struct intel_encoder_context *encoder_context,
5173                                      struct i965_gpe_context *gpe_context)
5174 {
5175     struct encoder_vme_mfc_context *vme_context = NULL;
5176     struct gen9_hevc_encoder_context *priv_ctx = NULL;
5177     struct gen9_hevc_encoder_state *priv_state = NULL;
5178     int bti_idx = 0;
5179     int size = 0;
5180
5181     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
5182     priv_ctx = (struct gen9_hevc_encoder_context *)vme_context->private_enc_ctx;
5183     priv_state = (struct gen9_hevc_encoder_state *)vme_context->private_enc_state;
5184
5185     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
5186                                  HEVC_ENC_SURFACE_RAW_Y_UV, bti_idx++,
5187                                  1, 1, I965_SURFACEFORMAT_R8_UNORM,
5188                                  NULL, NULL);
5189     bti_idx++;
5190
5191     gen9_hevc_set_gpe_1d_surface(ctx, priv_ctx, gpe_context,
5192                                  HEVC_ENC_SURFACE_16x16PU_SAD, bti_idx++,
5193                                  0, 0, 0, NULL, NULL);
5194
5195     size = priv_state->width_in_cu * priv_state->height_in_cu *
5196            priv_state->cu_record_size;
5197     gen9_hevc_set_gpe_1d_surface(ctx, priv_ctx, gpe_context,
5198                                  HEVC_ENC_SURFACE_CU_RECORD, bti_idx++,
5199                                  0, size, priv_state->mb_data_offset, NULL, NULL);
5200
5201     gen9_hevc_set_gpe_1d_surface(ctx, priv_ctx, gpe_context,
5202                                  HEVC_ENC_SURFACE_32x32_PU_OUTPUT, bti_idx++,
5203                                  0, 0, 0, NULL, NULL);
5204
5205     gen9_hevc_set_gpe_1d_surface(ctx, priv_ctx, gpe_context,
5206                                  HEVC_ENC_SURFACE_VME_8x8, bti_idx++,
5207                                  0, 0, 0, NULL, NULL);
5208
5209     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
5210                                  HEVC_ENC_SURFACE_SLICE_MAP, bti_idx++,
5211                                  0, 1, I965_SURFACEFORMAT_R8_UNORM,
5212                                  NULL, NULL);
5213
5214     gen9_hevc_set_gpe_adv_surface(ctx, priv_ctx, gpe_context,
5215                                   HEVC_ENC_SURFACE_RAW_VME, bti_idx++,
5216                                   NULL);
5217
5218     gen9_hevc_set_gpe_1d_surface(ctx, priv_ctx, gpe_context,
5219                                  HEVC_ENC_SURFACE_BRC_INPUT, bti_idx++,
5220                                  0, 0, 0, NULL, NULL);
5221
5222     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
5223                                  HEVC_ENC_SURFACE_SIMPLIFIED_INTRA, bti_idx++,
5224                                  0, 1, I965_SURFACEFORMAT_R8_UNORM,
5225                                  NULL, NULL);
5226
5227     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
5228                                  HEVC_ENC_SURFACE_LCU_QP, bti_idx++,
5229                                  0, 1, I965_SURFACEFORMAT_R8_UNORM,
5230                                  NULL, NULL);
5231
5232     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
5233                                  HEVC_ENC_SURFACE_BRC_DATA, bti_idx++,
5234                                  0, 1, I965_SURFACEFORMAT_R8_UNORM,
5235                                  NULL, NULL);
5236
5237     gen9_hevc_set_gpe_1d_surface(ctx, priv_ctx, gpe_context,
5238                                  HEVC_ENC_SURFACE_KERNEL_DEBUG, bti_idx++,
5239                                  0, 0, 0, NULL, NULL);
5240 }
5241
5242 static void
5243 gen9_hevc_16x16_pu_mode(VADriverContextP ctx,
5244                         struct encode_state *encode_state,
5245                         struct intel_encoder_context *encoder_context)
5246 {
5247     struct encoder_vme_mfc_context *vme_context = NULL;
5248     struct gen9_hevc_encoder_context *priv_ctx = NULL;
5249     struct gen9_hevc_encoder_state *priv_state = NULL;
5250     struct gpe_media_object_walker_parameter param;
5251     struct hevc_enc_kernel_walker_parameter hevc_walker_param;
5252     struct i965_gpe_context *gpe_context = NULL;
5253     int media_state = HEVC_ENC_MEDIA_STATE_16x16_PU_MODE_DECISION;
5254
5255     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
5256     priv_ctx = (struct gen9_hevc_encoder_context *)vme_context->private_enc_ctx;
5257     priv_state = (struct gen9_hevc_encoder_state *)vme_context->private_enc_state;
5258
5259     gpe_context = &priv_ctx->mbenc_context.gpe_contexts[HEVC_MBENC_16x16MD_IDX];
5260
5261     gen8_gpe_context_init(ctx, gpe_context);
5262     gen9_gpe_reset_binding_table(ctx, gpe_context);
5263     gen9_hevc_16x16_pu_mode_set_curbe(ctx, encode_state, encoder_context, gpe_context);
5264     gen9_hevc_16x16_pu_mode_set_surfaces(ctx, encode_state, encoder_context, gpe_context);
5265     gen8_gpe_setup_interface_data(ctx, gpe_context);
5266
5267     memset((void *)&hevc_walker_param, 0, sizeof(hevc_walker_param));
5268     hevc_walker_param.resolution_x = ALIGN(priv_state->picture_width, 32) >> 5;
5269     hevc_walker_param.resolution_y = ALIGN(priv_state->picture_height, 32) >> 5;
5270     hevc_walker_param.no_dependency = 1;
5271     gen9_hevc_init_object_walker(&hevc_walker_param, &param);
5272     gen9_hevc_run_object_walker(ctx, encoder_context, gpe_context, &param,
5273                                 media_state);
5274 }
5275
5276 static void
5277 gen9_hevc_16x16_sad_pu_comp_set_curbe(VADriverContextP ctx,
5278                                       struct encode_state *encode_state,
5279                                       struct intel_encoder_context *encoder_context,
5280                                       struct i965_gpe_context *gpe_context)
5281 {
5282     struct encoder_vme_mfc_context *vme_context = NULL;
5283     struct gen9_hevc_encoder_state *priv_state = NULL;
5284     gen9_hevc_mbenc_16x16_sad_curbe_data *cmd = NULL;
5285     VAEncSequenceParameterBufferHEVC *seq_param = NULL;
5286     int bti_idx = 0;
5287
5288     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
5289     priv_state = (struct gen9_hevc_encoder_state *)vme_context->private_enc_state;
5290     seq_param = (VAEncSequenceParameterBufferHEVC *)encode_state->seq_param_ext->buffer;
5291
5292     cmd = i965_gpe_context_map_curbe(gpe_context);
5293     if (!cmd)
5294         return;
5295
5296     memset((void *)cmd, 0, sizeof(*cmd));
5297
5298     cmd->dw0.frame_width = ALIGN(priv_state->picture_width, 16);
5299     cmd->dw0.frame_height = ALIGN(priv_state->picture_height, 16);
5300     cmd->dw1.log2_min_cu_size = seq_param->log2_min_luma_coding_block_size_minus3 + 3;
5301     cmd->dw1.log2_max_cu_size = cmd->dw1.log2_min_cu_size + seq_param->log2_diff_max_min_luma_coding_block_size;
5302     cmd->dw1.log2_min_tu_size = seq_param->log2_min_transform_block_size_minus2 + 2;
5303     cmd->dw1.enable_intra_early_exit = priv_state->tu_mode == HEVC_TU_RT_SPEED ?
5304                                        (priv_state->picture_coding_type == HEVC_SLICE_I ? 0 : 1) : 0;
5305     cmd->dw2.sim_flag_for_inter = 0;
5306     cmd->dw2.slice_type = priv_state->picture_coding_type;
5307     cmd->dw2.fast_surveillance_flag = priv_state->picture_coding_type == HEVC_SLICE_I ?
5308                                       0 : priv_state->video_surveillance_flag;
5309
5310     cmd->dw8.bti_src_y = bti_idx++;
5311     bti_idx++;
5312     cmd->dw9.bti_sad_16x16_pu_output = bti_idx++;
5313     cmd->dw10.bti_32x32_pu_mode_decision = bti_idx++;
5314     cmd->dw11.bti_slice_map = bti_idx++;
5315     cmd->dw12.bti_simplest_intra = bti_idx++;
5316     cmd->dw13.bti_debug = bti_idx++;
5317
5318     i965_gpe_context_unmap_curbe(gpe_context);
5319 }
5320
5321 static void
5322 gen9_hevc_16x16_sad_pu_comp_set_surfaces(VADriverContextP ctx,
5323                                          struct encode_state *encode_state,
5324                                          struct intel_encoder_context *encoder_context,
5325                                          struct i965_gpe_context *gpe_context)
5326 {
5327     struct encoder_vme_mfc_context *vme_context = NULL;
5328     struct gen9_hevc_encoder_context *priv_ctx = NULL;
5329     int bti_idx = 0;
5330
5331     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
5332     priv_ctx = (struct gen9_hevc_encoder_context *)vme_context->private_enc_ctx;
5333
5334     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
5335                                  HEVC_ENC_SURFACE_RAW_Y_UV, bti_idx++,
5336                                  1, 1, I965_SURFACEFORMAT_R8_UNORM,
5337                                  NULL, NULL);
5338     bti_idx++;
5339
5340     gen9_hevc_set_gpe_1d_surface(ctx, priv_ctx, gpe_context,
5341                                  HEVC_ENC_SURFACE_16x16PU_SAD, bti_idx++,
5342                                  0, 0, 0, NULL, NULL);
5343
5344     gen9_hevc_set_gpe_1d_surface(ctx, priv_ctx, gpe_context,
5345                                  HEVC_ENC_SURFACE_32x32_PU_OUTPUT, bti_idx++,
5346                                  0, 0, 0, NULL, NULL);
5347
5348     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
5349                                  HEVC_ENC_SURFACE_SLICE_MAP, bti_idx++,
5350                                  0, 1, I965_SURFACEFORMAT_R8_UNORM,
5351                                  NULL, NULL);
5352
5353     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
5354                                  HEVC_ENC_SURFACE_SIMPLIFIED_INTRA, bti_idx++,
5355                                  0, 1, I965_SURFACEFORMAT_R8_UNORM,
5356                                  NULL, NULL);
5357
5358     gen9_hevc_set_gpe_1d_surface(ctx, priv_ctx, gpe_context,
5359                                  HEVC_ENC_SURFACE_KERNEL_DEBUG, bti_idx++,
5360                                  0, 0, 0, NULL, NULL);
5361 }
5362
5363 static void
5364 gen9_hevc_16x16_sad_pu_computation(VADriverContextP ctx,
5365                                    struct encode_state *encode_state,
5366                                    struct intel_encoder_context *encoder_context)
5367 {
5368     struct encoder_vme_mfc_context *vme_context = NULL;
5369     struct gen9_hevc_encoder_context *priv_ctx = NULL;
5370     struct gen9_hevc_encoder_state *priv_state = NULL;
5371     struct gpe_media_object_walker_parameter param;
5372     struct hevc_enc_kernel_walker_parameter hevc_walker_param;
5373     struct i965_gpe_context *gpe_context = NULL;
5374     int media_state = HEVC_ENC_MEDIA_STATE_16x16_PU_SAD;
5375
5376     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
5377     priv_ctx = (struct gen9_hevc_encoder_context *)vme_context->private_enc_ctx;
5378     priv_state = (struct gen9_hevc_encoder_state *)vme_context->private_enc_state;
5379
5380     gpe_context = &priv_ctx->mbenc_context.gpe_contexts[HEVC_MBENC_16x16SAD_IDX];
5381
5382     gen8_gpe_context_init(ctx, gpe_context);
5383     gen9_gpe_reset_binding_table(ctx, gpe_context);
5384     gen9_hevc_16x16_sad_pu_comp_set_curbe(ctx, encode_state, encoder_context, gpe_context);
5385     gen9_hevc_16x16_sad_pu_comp_set_surfaces(ctx, encode_state, encoder_context, gpe_context);
5386     gen8_gpe_setup_interface_data(ctx, gpe_context);
5387
5388     memset((void *)&hevc_walker_param, 0, sizeof(hevc_walker_param));
5389     hevc_walker_param.resolution_x = ALIGN(priv_state->picture_width, 16) >> 4;
5390     hevc_walker_param.resolution_y = ALIGN(priv_state->picture_height, 16) >> 4;
5391     hevc_walker_param.no_dependency = 1;
5392     gen9_hevc_init_object_walker(&hevc_walker_param, &param);
5393     gen9_hevc_run_object_walker(ctx, encoder_context, gpe_context, &param,
5394                                 media_state);
5395 }
5396
5397 static void
5398 gen9_hevc_32x32_b_intra_set_curbe(VADriverContextP ctx,
5399                                   struct encode_state *encode_state,
5400                                   struct intel_encoder_context *encoder_context,
5401                                   struct i965_gpe_context *gpe_context)
5402 {
5403     struct encoder_vme_mfc_context *vme_context = NULL;
5404     struct generic_enc_codec_state *generic_state = NULL;
5405     struct gen9_hevc_encoder_state *priv_state = NULL;
5406     gen9_hevc_mbenc_b_32x32_pu_intra_curbe_data *cmd = NULL;
5407     VAEncSequenceParameterBufferHEVC *seq_param = NULL;
5408     int bti_idx = 0;
5409
5410     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
5411     generic_state = (struct generic_enc_codec_state *)vme_context->generic_enc_state;
5412     priv_state = (struct gen9_hevc_encoder_state *)vme_context->private_enc_state;
5413     seq_param = (VAEncSequenceParameterBufferHEVC *)encode_state->seq_param_ext->buffer;
5414
5415     cmd = i965_gpe_context_map_curbe(gpe_context);
5416     if (!cmd)
5417         return;
5418
5419     memset((void *)cmd, 0, sizeof(*cmd));
5420
5421     cmd->dw0.frame_width = ALIGN(priv_state->picture_width, 16);
5422     cmd->dw0.frame_height = ALIGN(priv_state->picture_height, 16);
5423     cmd->dw1.enable_debug_dump = 0;
5424     cmd->dw1.enable_intra_early_exit = priv_state->tu_mode == HEVC_TU_RT_SPEED ? 1 : 0;
5425     cmd->dw1.flags = 0;
5426     cmd->dw1.log2_min_tu_size = seq_param->log2_min_transform_block_size_minus2 + 2;
5427     cmd->dw1.slice_type = priv_state->picture_coding_type;
5428     cmd->dw1.hme_enable = generic_state->hme_enabled;
5429     cmd->dw1.fast_surveillance_flag = priv_state->picture_coding_type == HEVC_SLICE_I ?
5430                                       0 : priv_state->video_surveillance_flag;
5431
5432     cmd->dw2.qp_multiplier = 100;
5433     cmd->dw2.qp_value = 0;
5434
5435     cmd->dw8.bti_per_32x32_pu_intra_checck = bti_idx++;
5436     cmd->dw9.bti_src_y = bti_idx++;
5437     bti_idx++;
5438     cmd->dw10.bti_src_y2x = bti_idx++;
5439     cmd->dw11.bti_slice_map = bti_idx++;
5440     cmd->dw12.bti_vme_y2x = bti_idx++;
5441     cmd->dw13.bti_simplest_intra = bti_idx++;
5442     cmd->dw14.bti_hme_mv_pred = bti_idx++;
5443     cmd->dw15.bti_hme_dist = bti_idx++;
5444     cmd->dw16.bti_lcu_skip = bti_idx++;
5445     cmd->dw17.bti_debug = bti_idx++;
5446
5447     i965_gpe_context_unmap_curbe(gpe_context);
5448 }
5449
5450 static void
5451 gen9_hevc_32x32_b_intra_set_surfaces(VADriverContextP ctx,
5452                                      struct encode_state *encode_state,
5453                                      struct intel_encoder_context *encoder_context,
5454                                      struct i965_gpe_context *gpe_context)
5455 {
5456     struct encoder_vme_mfc_context *vme_context = NULL;
5457     struct gen9_hevc_encoder_context *priv_ctx = NULL;
5458     int bti_idx = 0;
5459
5460     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
5461     priv_ctx = (struct gen9_hevc_encoder_context *)vme_context->private_enc_ctx;
5462
5463     gen9_hevc_set_gpe_1d_surface(ctx, priv_ctx, gpe_context,
5464                                  HEVC_ENC_SURFACE_32x32_PU_OUTPUT, bti_idx++,
5465                                  0, 0, 0, NULL, NULL);
5466
5467     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
5468                                  HEVC_ENC_SURFACE_RAW_Y_UV, bti_idx++,
5469                                  1, 1, I965_SURFACEFORMAT_R8_UNORM,
5470                                  NULL, NULL);
5471     bti_idx++;
5472
5473     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
5474                                  HEVC_ENC_SURFACE_Y_2X, bti_idx++,
5475                                  0, 1, I965_SURFACEFORMAT_R8_UNORM,
5476                                  NULL, NULL);
5477
5478     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
5479                                  HEVC_ENC_SURFACE_SLICE_MAP, bti_idx++,
5480                                  0, 1, I965_SURFACEFORMAT_R8_UNORM,
5481                                  NULL, NULL);
5482
5483     gen9_hevc_set_gpe_adv_surface(ctx, priv_ctx, gpe_context,
5484                                   HEVC_ENC_SURFACE_Y_2X_VME, bti_idx++,
5485                                   NULL);
5486
5487     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
5488                                  HEVC_ENC_SURFACE_SIMPLIFIED_INTRA, bti_idx++,
5489                                  0, 1, I965_SURFACEFORMAT_R8_UNORM,
5490                                  NULL, NULL);
5491
5492     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
5493                                  HEVC_ENC_SURFACE_HME_MVP, bti_idx++,
5494                                  0, 1, I965_SURFACEFORMAT_R8_UNORM,
5495                                  NULL, NULL);
5496
5497     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
5498                                  HEVC_ENC_SURFACE_HME_DIST, bti_idx++,
5499                                  0, 1, I965_SURFACEFORMAT_R8_UNORM,
5500                                  NULL, NULL);
5501
5502     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
5503                                  HEVC_ENC_SURFACE_LCU_QP, bti_idx++,
5504                                  0, 1, I965_SURFACEFORMAT_R8_UNORM,
5505                                  NULL, NULL);
5506
5507     gen9_hevc_set_gpe_1d_surface(ctx, priv_ctx, gpe_context,
5508                                  HEVC_ENC_SURFACE_KERNEL_DEBUG, bti_idx,
5509                                  0, 0, 0, NULL, NULL);
5510 }
5511
5512 static void
5513 gen9_hevc_32x32_b_intra(VADriverContextP ctx,
5514                         struct encode_state *encode_state,
5515                         struct intel_encoder_context *encoder_context)
5516 {
5517     struct encoder_vme_mfc_context *vme_context = NULL;
5518     struct gen9_hevc_encoder_context *priv_ctx = NULL;
5519     struct gen9_hevc_encoder_state *priv_state = NULL;
5520     struct gpe_media_object_walker_parameter param;
5521     struct hevc_enc_kernel_walker_parameter hevc_walker_param;
5522     struct i965_gpe_context *gpe_context = NULL;
5523     int media_state = HEVC_ENC_MEDIA_STATE_32x32_B_INTRA_CHECK;
5524
5525     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
5526     priv_ctx = (struct gen9_hevc_encoder_context *)vme_context->private_enc_ctx;
5527     priv_state = (struct gen9_hevc_encoder_state *)vme_context->private_enc_state;
5528
5529     gpe_context = &priv_ctx->mbenc_context.gpe_contexts[HEVC_MBENC_32x32INTRACHECK_IDX];
5530
5531     gen8_gpe_context_init(ctx, gpe_context);
5532     gen9_gpe_reset_binding_table(ctx, gpe_context);
5533     gen9_hevc_32x32_b_intra_set_curbe(ctx, encode_state, encoder_context, gpe_context);
5534     gen9_hevc_32x32_b_intra_set_surfaces(ctx, encode_state, encoder_context, gpe_context);
5535     gen8_gpe_setup_interface_data(ctx, gpe_context);
5536
5537     memset((void *)&hevc_walker_param, 0, sizeof(hevc_walker_param));
5538     hevc_walker_param.resolution_x = ALIGN(priv_state->picture_width, 32) >> 5;
5539     hevc_walker_param.resolution_y = ALIGN(priv_state->picture_height, 32) >> 5;
5540     hevc_walker_param.no_dependency = 1;
5541     gen9_hevc_init_object_walker(&hevc_walker_param, &param);
5542     gen9_hevc_run_object_walker(ctx, encoder_context, gpe_context, &param,
5543                                 media_state);
5544 }
5545
5546 static void
5547 gen9_hevc_32x32_pu_mode_set_curbe(VADriverContextP ctx,
5548                                   struct encode_state *encode_state,
5549                                   struct intel_encoder_context *encoder_context,
5550                                   struct i965_gpe_context *gpe_context)
5551 {
5552     struct encoder_vme_mfc_context *vme_context = NULL;
5553     struct gen9_hevc_encoder_context *priv_ctx = NULL;
5554     struct generic_enc_codec_state *generic_state = NULL;
5555     struct gen9_hevc_encoder_state *priv_state = NULL;
5556     VAEncPictureParameterBufferHEVC *pic_param = NULL;
5557     VAEncSliceParameterBufferHEVC *slice_param = NULL;
5558     gen9_hevc_mbenc_32x32_pu_mode_curbe_data *cmd = NULL;
5559     double lambda_scaling_factor = 1.0, qp_lambda = 0.0, squared_qp_lambda = 0.0;
5560     unsigned int slice_qp = 0, fixed_point_lambda = 0;
5561     int bti_idx = 0;
5562
5563     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
5564     priv_ctx = (struct gen9_hevc_encoder_context *)vme_context->private_enc_ctx;
5565     generic_state = (struct generic_enc_codec_state *)vme_context->generic_enc_state;
5566     priv_state = (struct gen9_hevc_encoder_state *)vme_context->private_enc_state;
5567     pic_param = (VAEncPictureParameterBufferHEVC *)encode_state->pic_param_ext->buffer;
5568     slice_param = (VAEncSliceParameterBufferHEVC *)encode_state->slice_params_ext[0]->buffer;
5569
5570     slice_qp = pic_param->pic_init_qp + slice_param->slice_qp_delta;
5571     gen9_hevc_set_lambda_tables(priv_ctx, HEVC_SLICE_I, HEVC_ENC_INTRA_TRANS_HAAR);
5572     lambda_scaling_factor = 1.0;
5573     qp_lambda = priv_ctx->lambda_md_table[HEVC_SLICE_I][slice_qp];
5574     squared_qp_lambda = qp_lambda * qp_lambda;
5575     fixed_point_lambda = (unsigned int)(lambda_scaling_factor * squared_qp_lambda * (1 << 10));
5576
5577     cmd = i965_gpe_context_map_curbe(gpe_context);
5578     if (!cmd)
5579         return;
5580
5581     memset((void *)cmd, 0, sizeof(*cmd));
5582
5583     cmd->dw0.frame_width = ALIGN(priv_state->picture_width, 16);
5584     cmd->dw0.frame_height = ALIGN(priv_state->picture_height, 16);
5585     cmd->dw1.enable_debug_dump = 0;
5586     cmd->dw1.lcu_type = priv_state->lcu_size == 64 ? 0 : 1;
5587     cmd->dw1.pu_type = 0;
5588     cmd->dw1.brc_enable = !!generic_state->brc_enabled;
5589     cmd->dw1.lcu_brc_enable = priv_state->lcu_brc_enabled;
5590     cmd->dw1.slice_type = priv_state->picture_coding_type;
5591     cmd->dw1.fast_surveillance_flag = priv_state->picture_coding_type == HEVC_SLICE_I ?
5592                                       0 : priv_state->video_surveillance_flag;
5593     cmd->dw1.roi_enable = (priv_state->num_roi > 0);
5594
5595     cmd->dw2.lambda = fixed_point_lambda;
5596     cmd->dw3.mode_cost_32x32 = 0;
5597     cmd->dw4.early_exit = (unsigned int) - 1;
5598
5599     cmd->dw8.bti_32x32_pu_output = bti_idx++;
5600     cmd->dw9.bti_src_y = bti_idx++;
5601     bti_idx++;
5602     cmd->dw10.bti_src_y2x = bti_idx++;
5603     cmd->dw11.bti_slice_map = bti_idx++;
5604     cmd->dw12.bti_src_y2x_vme = bti_idx++;
5605     cmd->dw13.bti_brc_input = bti_idx++;
5606     cmd->dw14.bti_lcu_qp_surface = bti_idx++;
5607     cmd->dw15.bti_brc_data = bti_idx++;
5608     cmd->dw16.bti_kernel_debug = bti_idx++;
5609
5610     i965_gpe_context_unmap_curbe(gpe_context);
5611 }
5612
5613 static void
5614 gen9_hevc_32x32_pu_mode_set_surfaces(VADriverContextP ctx,
5615                                      struct encode_state *encode_state,
5616                                      struct intel_encoder_context *encoder_context,
5617                                      struct i965_gpe_context *gpe_context)
5618 {
5619     struct encoder_vme_mfc_context *vme_context = NULL;
5620     struct gen9_hevc_encoder_context *priv_ctx = NULL;
5621     int bti_idx = 0;
5622
5623     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
5624     priv_ctx = (struct gen9_hevc_encoder_context *)vme_context->private_enc_ctx;
5625
5626     gen9_hevc_set_gpe_1d_surface(ctx, priv_ctx, gpe_context,
5627                                  HEVC_ENC_SURFACE_32x32_PU_OUTPUT, bti_idx++,
5628                                  0, 0, 0, NULL, NULL);
5629
5630     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
5631                                  HEVC_ENC_SURFACE_RAW_Y, bti_idx++,
5632                                  1, 1, I965_SURFACEFORMAT_R8_UNORM,
5633                                  NULL, NULL);
5634     bti_idx++;
5635
5636     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
5637                                  HEVC_ENC_SURFACE_Y_2X, bti_idx++,
5638                                  0, 1, I965_SURFACEFORMAT_R8_UNORM,
5639                                  NULL, NULL);
5640
5641     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
5642                                  HEVC_ENC_SURFACE_SLICE_MAP, bti_idx++,
5643                                  0, 1, I965_SURFACEFORMAT_R8_UNORM,
5644                                  NULL, NULL);
5645
5646     gen9_hevc_set_gpe_adv_surface(ctx, priv_ctx, gpe_context,
5647                                   HEVC_ENC_SURFACE_Y_2X_VME, bti_idx++,
5648                                   NULL);
5649
5650     gen9_hevc_set_gpe_1d_surface(ctx, priv_ctx, gpe_context,
5651                                  HEVC_ENC_SURFACE_BRC_INPUT, bti_idx++,
5652                                  0, 0, 0, NULL, NULL);
5653
5654     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
5655                                  HEVC_ENC_SURFACE_LCU_QP, bti_idx++,
5656                                  0, 1, I965_SURFACEFORMAT_R8_UNORM,
5657                                  NULL, NULL);
5658
5659     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
5660                                  HEVC_ENC_SURFACE_BRC_DATA, bti_idx++,
5661                                  0, 1, I965_SURFACEFORMAT_R8_UNORM,
5662                                  NULL, NULL);
5663
5664     gen9_hevc_set_gpe_1d_surface(ctx, priv_ctx, gpe_context,
5665                                  HEVC_ENC_SURFACE_KERNEL_DEBUG, bti_idx,
5666                                  0, 0, 0, NULL, NULL);
5667 }
5668
5669 static void
5670 gen9_hevc_32x32_pu_mode(VADriverContextP ctx,
5671                         struct encode_state *encode_state,
5672                         struct intel_encoder_context *encoder_context)
5673 {
5674     struct encoder_vme_mfc_context *vme_context = NULL;
5675     struct gen9_hevc_encoder_context *priv_ctx = NULL;
5676     struct gen9_hevc_encoder_state *priv_state = NULL;
5677     struct gpe_media_object_walker_parameter param;
5678     struct hevc_enc_kernel_walker_parameter hevc_walker_param;
5679     struct i965_gpe_context *gpe_context = NULL;
5680     int media_state = HEVC_ENC_MEDIA_STATE_32x32_PU_MODE_DECISION;
5681
5682     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
5683     priv_ctx = (struct gen9_hevc_encoder_context *)vme_context->private_enc_ctx;
5684     priv_state = (struct gen9_hevc_encoder_state *)vme_context->private_enc_state;
5685
5686     gpe_context = &priv_ctx->mbenc_context.gpe_contexts[HEVC_MBENC_32x32MD_IDX];
5687
5688     gen8_gpe_context_init(ctx, gpe_context);
5689     gen9_gpe_reset_binding_table(ctx, gpe_context);
5690     gen9_hevc_32x32_pu_mode_set_curbe(ctx, encode_state, encoder_context, gpe_context);
5691     gen9_hevc_32x32_pu_mode_set_surfaces(ctx, encode_state, encoder_context, gpe_context);
5692     gen8_gpe_setup_interface_data(ctx, gpe_context);
5693
5694     memset((void *)&hevc_walker_param, 0, sizeof(hevc_walker_param));
5695     hevc_walker_param.resolution_x = ALIGN(priv_state->picture_width, 32) >> 5;
5696     hevc_walker_param.resolution_y = ALIGN(priv_state->picture_height, 32) >> 5;
5697     hevc_walker_param.no_dependency = 1;
5698     gen9_hevc_init_object_walker(&hevc_walker_param, &param);
5699     gen9_hevc_run_object_walker(ctx, encoder_context, gpe_context, &param,
5700                                 media_state);
5701 }
5702
5703 static void
5704 gen9_hevc_2x_scaling_set_curbe(VADriverContextP ctx,
5705                                struct encode_state *encode_state,
5706                                struct intel_encoder_context *encoder_context,
5707                                struct i965_gpe_context *gpe_context)
5708 {
5709     struct encoder_vme_mfc_context *vme_context = NULL;
5710     struct gen9_hevc_encoder_state *priv_state = NULL;
5711     gen9_hevc_mbenc_downscaling2x_curbe_data *cmd = NULL;
5712     int bti_idx = 0;
5713
5714     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
5715     priv_state = (struct gen9_hevc_encoder_state *)vme_context->private_enc_state;
5716
5717     cmd = i965_gpe_context_map_curbe(gpe_context);
5718     if (!cmd)
5719         return;
5720
5721     memset((void *)cmd, 0, sizeof(gen9_hevc_mbenc_downscaling2x_curbe_data));
5722
5723     cmd->dw0.pic_width = ALIGN(priv_state->picture_width, 16);
5724     cmd->dw0.pic_height = ALIGN(priv_state->picture_height, 16);
5725
5726     cmd->dw8.bti_src_y = bti_idx++;
5727     cmd->dw9.bit_dst_y = bti_idx;
5728
5729     i965_gpe_context_unmap_curbe(gpe_context);
5730 }
5731
5732 static void
5733 gen9_hevc_2x_scaling_set_surfaces(VADriverContextP ctx,
5734                                   struct encode_state *encode_state,
5735                                   struct intel_encoder_context *encoder_context,
5736                                   struct i965_gpe_context *gpe_context)
5737 {
5738     struct encoder_vme_mfc_context *vme_context = NULL;
5739     struct gen9_hevc_encoder_context *priv_ctx = NULL;
5740     int bti_idx = 0;
5741
5742     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
5743     priv_ctx = (struct gen9_hevc_encoder_context *)vme_context->private_enc_ctx;
5744
5745     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
5746                                  HEVC_ENC_SURFACE_RAW_Y, bti_idx++,
5747                                  0, 1, I965_SURFACEFORMAT_R16_UNORM,
5748                                  NULL, NULL);
5749
5750     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
5751                                  HEVC_ENC_SURFACE_Y_2X, bti_idx,
5752                                  0, 1, I965_SURFACEFORMAT_R16_UNORM,
5753                                  NULL, NULL);
5754 }
5755
5756 static void
5757 gen9_hevc_2x_scaling(VADriverContextP ctx,
5758                      struct encode_state *encode_state,
5759                      struct intel_encoder_context *encoder_context)
5760 {
5761     struct encoder_vme_mfc_context *vme_context = NULL;
5762     struct gen9_hevc_encoder_context *priv_ctx = NULL;
5763     struct gen9_hevc_encoder_state *priv_state = NULL;
5764     struct gpe_media_object_walker_parameter param;
5765     struct hevc_enc_kernel_walker_parameter hevc_walker_param;
5766     struct i965_gpe_context *gpe_context = NULL;
5767     int media_state = HEVC_ENC_MEDIA_STATE_2X_SCALING;
5768
5769     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
5770     priv_ctx = (struct gen9_hevc_encoder_context *)vme_context->private_enc_ctx;
5771     priv_state = (struct gen9_hevc_encoder_state *)vme_context->private_enc_state;
5772
5773     gpe_context = &priv_ctx->mbenc_context.gpe_contexts[HEVC_MBENC_2xSCALING_IDX];
5774
5775     gen8_gpe_context_init(ctx, gpe_context);
5776     gen9_gpe_reset_binding_table(ctx, gpe_context);
5777     gen9_hevc_2x_scaling_set_curbe(ctx, encode_state, encoder_context, gpe_context);
5778     gen9_hevc_2x_scaling_set_surfaces(ctx, encode_state, encoder_context, gpe_context);
5779     gen8_gpe_setup_interface_data(ctx, gpe_context);
5780
5781     memset((void *)&hevc_walker_param, 0, sizeof(hevc_walker_param));
5782     hevc_walker_param.resolution_x = ALIGN(priv_state->picture_width, 32) >> 5;
5783     hevc_walker_param.resolution_y = ALIGN(priv_state->picture_height, 32) >> 5;
5784     hevc_walker_param.no_dependency = 1;
5785     gen9_hevc_init_object_walker(&hevc_walker_param, &param);
5786
5787     gen9_hevc_run_object_walker(ctx, encoder_context, gpe_context, &param,
5788                                 media_state);
5789 }
5790
5791 static void
5792 gen9_hevc_mbenc(VADriverContextP ctx,
5793                 struct encode_state *encode_state,
5794                 struct intel_encoder_context *encoder_context)
5795 {
5796     struct encoder_vme_mfc_context *vme_context = NULL;
5797     struct gen9_hevc_encoder_state *priv_state = NULL;
5798     int fast_encoding = 0;
5799
5800     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
5801     priv_state = (struct gen9_hevc_encoder_state *)vme_context->private_enc_state;
5802     fast_encoding = (priv_state->tu_mode == HEVC_TU_BEST_SPEED);
5803
5804     if (!fast_encoding) {
5805         if (!priv_state->bit_depth_luma_minus8)
5806             gen9_hevc_2x_scaling(ctx, encode_state, encoder_context);
5807
5808         if (priv_state->picture_coding_type == HEVC_SLICE_I)
5809             gen9_hevc_32x32_pu_mode(ctx, encode_state, encoder_context);
5810         else
5811             gen9_hevc_32x32_b_intra(ctx, encode_state, encoder_context);
5812
5813         gen9_hevc_16x16_sad_pu_computation(ctx, encode_state, encoder_context);
5814         gen9_hevc_16x16_pu_mode(ctx, encode_state, encoder_context);
5815         gen9_hevc_8x8_pu_mode(ctx, encode_state, encoder_context);
5816         gen9_hevc_8x8_pu_fmode(ctx, encode_state, encoder_context);
5817     }
5818
5819     if (priv_state->picture_coding_type != HEVC_SLICE_I ||
5820         fast_encoding) {
5821         gen9_hevc_8x8_b_mbenc(ctx, encode_state, encoder_context);
5822         gen9_hevc_8x8_b_pak(ctx, encode_state, encoder_context);
5823     }
5824 }
5825
5826 static VAStatus
5827 gen9_hevc_vme_gpe_init(VADriverContextP ctx,
5828                        struct encode_state *encode_state,
5829                        struct intel_encoder_context *encoder_context)
5830 {
5831     struct encoder_vme_mfc_context *vme_context = NULL;
5832     struct gen9_hevc_encoder_context *priv_ctx = NULL;
5833     struct gen9_hevc_encoder_state *priv_state = NULL;
5834     VAEncSliceParameterBufferHEVC *slice_param = NULL;
5835     int i = 0, j = 0;
5836
5837     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
5838     priv_ctx = (struct gen9_hevc_encoder_context *)vme_context->private_enc_ctx;
5839     priv_state = (struct gen9_hevc_encoder_state *)vme_context->private_enc_state;
5840
5841     i965_zero_gpe_resource(&priv_ctx->res_mb_code_surface);
5842
5843     i965_zero_gpe_resource(&priv_ctx->res_slice_map_buffer);
5844     if (encode_state->num_slice_params_ext > 1) {
5845         struct gen9_hevc_slice_map *pslice_map = NULL;
5846         int width = priv_state->width_in_lcu;
5847         int pitch = ALIGN(priv_state->frame_width_in_max_lcu >> 3, 64);
5848         void *ptr_start = NULL;
5849         int lcu_count = 0;
5850
5851         ptr_start = (void *)i965_map_gpe_resource(&priv_ctx->res_slice_map_buffer);
5852
5853         if (!ptr_start)
5854             return VA_STATUS_ERROR_UNKNOWN;
5855
5856         pslice_map = (struct gen9_hevc_slice_map *)ptr_start;
5857         for (i = 0; i < encode_state->num_slice_params_ext; i++) {
5858             slice_param = (VAEncSliceParameterBufferHEVC *)encode_state->slice_params_ext[i]->buffer;
5859
5860             for (j = 0; j < slice_param->num_ctu_in_slice; j++) {
5861                 pslice_map[lcu_count++].slice_id = i;
5862
5863                 if (lcu_count >= width) {
5864                     lcu_count = 0;
5865                     ptr_start += pitch;
5866
5867                     pslice_map = (struct gen9_hevc_slice_map *)ptr_start;
5868                 }
5869             }
5870         }
5871
5872         i965_unmap_gpe_resource(&priv_ctx->res_slice_map_buffer);
5873     }
5874
5875     return VA_STATUS_SUCCESS;
5876 }
5877
5878 static VAStatus
5879 gen9_hevc_vme_gpe_run(VADriverContextP ctx,
5880                       struct encode_state *encode_state,
5881                       struct intel_encoder_context *encoder_context)
5882 {
5883     struct encoder_vme_mfc_context *vme_context = NULL;
5884     struct generic_enc_codec_state *generic_state = NULL;
5885     struct gen9_hevc_encoder_state *priv_state = NULL;
5886
5887     vme_context = encoder_context->vme_context;
5888     generic_state = vme_context->generic_enc_state;
5889     priv_state = vme_context->private_enc_state;
5890
5891     if (generic_state->brc_enabled &&
5892         (generic_state->brc_need_reset || !generic_state->brc_inited)) {
5893         gen9_hevc_brc_init_reset(ctx, encode_state, encoder_context,
5894                                  generic_state->brc_inited ? 1 : 0);
5895         generic_state->brc_need_reset = 0;
5896         generic_state->brc_inited = 1;
5897     }
5898
5899     if (generic_state->hme_supported || generic_state->brc_enabled) {
5900         gen9_hevc_hme_scaling(ctx, encode_state, encoder_context);
5901
5902         if (generic_state->brc_enabled)
5903             gen9_hevc_brc_intra_dist(ctx, encode_state, encoder_context);
5904
5905         if (generic_state->hme_enabled)
5906             gen9_hevc_hme_encode_me(ctx, encode_state, encoder_context);
5907
5908         if (generic_state->brc_enabled)
5909             gen9_hevc_brc_update(ctx, encode_state, encoder_context);
5910     }
5911
5912     if (priv_state->num_roi && !generic_state->brc_enabled)
5913         gen9_hevc_brc_update_lcu_based(ctx, encode_state, encoder_context);
5914
5915     if (priv_state->bit_depth_luma_minus8)
5916         gen9_hevc_ref_frame_depth_conversion(ctx, encode_state, encoder_context);
5917
5918     gen9_hevc_mbenc(ctx, encode_state, encoder_context);
5919
5920     return VA_STATUS_SUCCESS;
5921 }
5922
5923 static VAStatus
5924 gen9_hevc_vme_pipeline(VADriverContextP ctx,
5925                        VAProfile profile,
5926                        struct encode_state *encode_state,
5927                        struct intel_encoder_context *encoder_context)
5928 {
5929     VAStatus va_status = VA_STATUS_SUCCESS;
5930
5931     va_status = gen9_hevc_enc_init_parameters(ctx, encode_state, encoder_context);
5932     if (va_status != VA_STATUS_SUCCESS)
5933         goto EXIT;
5934
5935     va_status = gen9_hevc_vme_gpe_init(ctx, encode_state, encoder_context);
5936     if (va_status != VA_STATUS_SUCCESS)
5937         goto EXIT;
5938
5939     va_status = gen9_hevc_vme_gpe_run(ctx, encode_state, encoder_context);
5940
5941 EXIT:
5942     return va_status;
5943 }
5944
5945 static void
5946 gen9_hevc_vme_scaling_context_init(VADriverContextP ctx,
5947                                    struct intel_encoder_context *encoder_context)
5948 {
5949     struct encoder_vme_mfc_context *vme_context = NULL;
5950     struct gen9_hevc_encoder_context *priv_ctx = NULL;
5951     struct gen9_hevc_encoder_state *priv_state = NULL;
5952     struct i965_gpe_context *gpe_context = NULL;
5953     struct i965_kernel kernel_info;
5954     struct gen9_hevc_scaling_context *scaling_ctx = NULL;
5955     GEN9_ENC_OPERATION kernel_idx;
5956     int curbe_size = 0;
5957     int i = 0;
5958
5959     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
5960     priv_ctx = (struct gen9_hevc_encoder_context *)vme_context->private_enc_ctx;
5961     priv_state = (struct gen9_hevc_encoder_state *)vme_context->private_enc_state;
5962
5963     scaling_ctx = &priv_ctx->scaling_context;
5964
5965     for (i = 0; i < NUM_HEVC_ENC_SCALING; i++) {
5966         if (i == HEVC_ENC_SCALING_4X ||
5967             i == HEVC_ENC_SCALING_16X) {
5968             curbe_size = sizeof(gen9_hevc_scaling4x_curbe_data);
5969             kernel_idx = GEN9_ENC_SCALING4X;
5970         } else if (i == HEVC_ENC_SCALING_32X) {
5971             curbe_size = sizeof(gen9_hevc_scaling2x_curbe_data);
5972             kernel_idx = GEN9_ENC_SCALING2X;
5973         }
5974
5975         gpe_context = &scaling_ctx->gpe_contexts[i];
5976
5977         gen9_hevc_vme_init_gpe_context(ctx, gpe_context,
5978                                        curbe_size,
5979                                        curbe_size);
5980         gen9_hevc_vme_init_scoreboard(gpe_context,
5981                                       0xFF,
5982                                       priv_state->use_hw_scoreboard,
5983                                       priv_state->use_hw_non_stalling_scoreborad);
5984
5985         memset(&kernel_info, 0, sizeof(kernel_info));
5986         gen9_hevc_get_kernel_header_and_size((void *)hevc_enc_kernel_ptr,
5987                                              hevc_enc_kernel_size,
5988                                              kernel_idx,
5989                                              0,
5990                                              &kernel_info);
5991         gen8_gpe_load_kernels(ctx,
5992                               gpe_context,
5993                               &kernel_info,
5994                               1);
5995     }
5996 }
5997
5998 static void
5999 gen9_hevc_vme_me_context_init(VADriverContextP ctx,
6000                               struct intel_encoder_context *encoder_context)
6001 {
6002     struct encoder_vme_mfc_context *vme_context = NULL;
6003     struct gen9_hevc_encoder_context *priv_ctx = NULL;
6004     struct gen9_hevc_encoder_state *priv_state = NULL;
6005     struct i965_gpe_context *gpe_context = NULL;
6006     struct i965_kernel kernel_info;
6007     struct gen9_hevc_me_context *me_ctx = NULL;
6008     int i = 0, j = 0;
6009
6010     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
6011     priv_ctx = (struct gen9_hevc_encoder_context *)vme_context->private_enc_ctx;
6012     priv_state = (struct gen9_hevc_encoder_state *)vme_context->private_enc_state;
6013
6014     me_ctx = &priv_ctx->me_context;
6015
6016     for (i = 0; i < NUM_HEVC_ENC_ME; i++) {
6017         for (j = 0; j < NUM_HEVC_ENC_ME_TYPES; j++) {
6018             gpe_context = &me_ctx->gpe_context[j][i];
6019
6020             gen9_hevc_vme_init_gpe_context(ctx, gpe_context,
6021                                            sizeof(gen9_hevc_me_curbe_data),
6022                                            0);
6023             gen9_hevc_vme_init_scoreboard(gpe_context,
6024                                           0xFF,
6025                                           priv_state->use_hw_scoreboard,
6026                                           priv_state->use_hw_non_stalling_scoreborad);
6027
6028             memset(&kernel_info, 0, sizeof(kernel_info));
6029             gen9_hevc_get_kernel_header_and_size((void *)hevc_enc_kernel_ptr,
6030                                                  hevc_enc_kernel_size,
6031                                                  GEN9_ENC_ME,
6032                                                  i,
6033                                                  &kernel_info);
6034             gen8_gpe_load_kernels(ctx,
6035                                   gpe_context,
6036                                   &kernel_info,
6037                                   1);
6038         }
6039     }
6040 }
6041
6042 static void
6043 gen9_hevc_vme_mbenc_context_init(VADriverContextP ctx,
6044                                  struct intel_encoder_context *encoder_context)
6045 {
6046     struct encoder_vme_mfc_context *vme_context = NULL;
6047     struct gen9_hevc_encoder_context *priv_ctx = NULL;
6048     struct gen9_hevc_encoder_state *priv_state = NULL;
6049     struct i965_gpe_context *gpe_context = NULL;
6050     struct i965_kernel kernel_info;
6051     struct gen9_hevc_mbenc_context *mbenc_ctx = NULL;
6052     int i = 0;
6053
6054     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
6055     priv_ctx = (struct gen9_hevc_encoder_context *)vme_context->private_enc_ctx;
6056     priv_state = (struct gen9_hevc_encoder_state *)vme_context->private_enc_state;
6057
6058     mbenc_ctx = &priv_ctx->mbenc_context;
6059
6060     mbenc_ctx->kernel_num = GEN8_HEVC_ENC_MBENC_TOTAL_NUM;
6061
6062     for (i = 0; i < mbenc_ctx->kernel_num; i++) {
6063         gpe_context = &mbenc_ctx->gpe_contexts[i];
6064
6065         gen9_hevc_vme_init_gpe_context(ctx, gpe_context,
6066                                        hevc_mbenc_curbe_size[i],
6067                                        0);
6068         gen9_hevc_vme_init_scoreboard(gpe_context,
6069                                       0xFF,
6070                                       priv_state->use_hw_scoreboard,
6071                                       priv_state->use_hw_non_stalling_scoreborad);
6072
6073         memset(&kernel_info, 0, sizeof(kernel_info));
6074         gen9_hevc_get_kernel_header_and_size((void *)hevc_enc_kernel_ptr,
6075                                              hevc_enc_kernel_size,
6076                                              GEN9_ENC_MBENC,
6077                                              i,
6078                                              &kernel_info);
6079         gen8_gpe_load_kernels(ctx,
6080                               gpe_context,
6081                               &kernel_info,
6082                               1);
6083     }
6084 }
6085
6086 static void
6087 gen9_hevc_vme_brc_context_init(VADriverContextP ctx,
6088                                struct intel_encoder_context *encoder_context)
6089 {
6090     struct encoder_vme_mfc_context *vme_context = NULL;
6091     struct gen9_hevc_encoder_context *priv_ctx = NULL;
6092     struct gen9_hevc_encoder_state *priv_state = NULL;
6093     struct i965_gpe_context *gpe_context = NULL;
6094     struct i965_kernel kernel_info;
6095     struct gen9_hevc_brc_context *brc_ctx = NULL;
6096     int i = 0;
6097
6098     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
6099     priv_ctx = (struct gen9_hevc_encoder_context *)vme_context->private_enc_ctx;
6100     priv_state = (struct gen9_hevc_encoder_state *)vme_context->private_enc_state;
6101
6102     brc_ctx = &priv_ctx->brc_context;
6103
6104     for (i = 0; i < GEN9_HEVC_ENC_BRC_NUM; i++) {
6105         gpe_context = &brc_ctx->gpe_contexts[i];
6106
6107         gen9_hevc_vme_init_gpe_context(ctx, gpe_context,
6108                                        hevc_brc_curbe_size[i],
6109                                        0);
6110         gen9_hevc_vme_init_scoreboard(gpe_context,
6111                                       0xFF,
6112                                       priv_state->use_hw_scoreboard,
6113                                       priv_state->use_hw_non_stalling_scoreborad);
6114
6115         memset(&kernel_info, 0, sizeof(kernel_info));
6116         gen9_hevc_get_kernel_header_and_size((void *)hevc_enc_kernel_ptr,
6117                                              hevc_enc_kernel_size,
6118                                              GEN9_ENC_BRC,
6119                                              i,
6120                                              &kernel_info);
6121         gen8_gpe_load_kernels(ctx,
6122                               gpe_context,
6123                               &kernel_info,
6124                               1);
6125     }
6126 }
6127
6128 static void
6129 gen9_hevc_vme_scaling_context_destroy(struct gen9_hevc_scaling_context *scaling_context)
6130 {
6131     int i;
6132
6133     for (i = 0; i < NUM_HEVC_ENC_SCALING; i++)
6134         gen8_gpe_context_destroy(&scaling_context->gpe_contexts[i]);
6135 }
6136
6137 static void
6138 gen9_hevc_vme_me_context_destroy(struct gen9_hevc_me_context *me_context)
6139 {
6140     int i, j;
6141
6142     for (i = 0; i < NUM_HEVC_ENC_ME; i++)
6143         for (j = 0; j < NUM_HEVC_ENC_ME_TYPES; j++)
6144             gen8_gpe_context_destroy(&me_context->gpe_context[j][i]);
6145 }
6146
6147 static void
6148 gen9_hevc_vme_mbenc_context_destroy(struct gen9_hevc_mbenc_context *mbenc_context)
6149 {
6150     int i;
6151
6152     for (i = 0; i < mbenc_context->kernel_num; i++)
6153         gen8_gpe_context_destroy(&mbenc_context->gpe_contexts[i]);
6154 }
6155
6156 static void
6157 gen9_hevc_vme_brc_context_destroy(struct gen9_hevc_brc_context *brc_context)
6158 {
6159     int i;
6160
6161     for (i = 0; i < GEN9_HEVC_ENC_BRC_NUM; i++)
6162         gen8_gpe_context_destroy(&brc_context->gpe_contexts[i]);
6163 }
6164
6165 static void
6166 gen9_hevc_vme_kernels_context_init(VADriverContextP ctx,
6167                                    struct intel_encoder_context *encoder_context)
6168 {
6169     gen9_hevc_vme_scaling_context_init(ctx, encoder_context);
6170     gen9_hevc_vme_me_context_init(ctx, encoder_context);
6171     gen9_hevc_vme_mbenc_context_init(ctx, encoder_context);
6172     gen9_hevc_vme_brc_context_init(ctx, encoder_context);
6173 }
6174
6175 static void
6176 gen9_hevc_vme_kernels_context_destroy(struct encoder_vme_mfc_context *vme_context)
6177 {
6178     struct gen9_hevc_encoder_context *priv_ctx = NULL;
6179
6180     priv_ctx = (struct gen9_hevc_encoder_context *)vme_context->private_enc_ctx;
6181
6182     gen9_hevc_vme_scaling_context_destroy(&priv_ctx->scaling_context);
6183     gen9_hevc_vme_me_context_destroy(&priv_ctx->me_context);
6184     gen9_hevc_vme_mbenc_context_destroy(&priv_ctx->mbenc_context);
6185     gen9_hevc_vme_brc_context_destroy(&priv_ctx->brc_context);
6186 }
6187
6188 static void
6189 gen9_hevc_vme_context_destroy(void *context)
6190 {
6191     struct encoder_vme_mfc_context *vme_context = (struct encoder_vme_mfc_context *)context;
6192
6193     if (!vme_context)
6194         return;
6195
6196     gen9_hevc_enc_free_resources(vme_context);
6197
6198     gen9_hevc_vme_kernels_context_destroy(vme_context);
6199
6200     if (vme_context->private_enc_ctx) free(vme_context->private_enc_ctx);
6201     if (vme_context->generic_enc_state) free(vme_context->generic_enc_state);
6202     if (vme_context->private_enc_state) free(vme_context->private_enc_state);
6203
6204     free(vme_context);
6205 }
6206
6207 #define PAK_IMPLEMENTATION_START
6208
6209 static void
6210 gen9_hevc_pak_pipe_mode_select(VADriverContextP ctx,
6211                                struct encode_state *encode_state,
6212                                struct intel_encoder_context *encoder_context)
6213 {
6214     struct intel_batchbuffer *batch = encoder_context->base.batch;
6215
6216     BEGIN_BCS_BATCH(batch, 6);
6217
6218     OUT_BCS_BATCH(batch, HCP_PIPE_MODE_SELECT | (6 - 2));
6219     OUT_BCS_BATCH(batch,
6220                   (0 << 5) |
6221                   (0 << 3) |
6222                   HCP_CODEC_SELECT_ENCODE);
6223     OUT_BCS_BATCH(batch, 0);
6224     OUT_BCS_BATCH(batch, 0);
6225     OUT_BCS_BATCH(batch, 0);
6226     OUT_BCS_BATCH(batch, 0);
6227
6228     ADVANCE_BCS_BATCH(batch);
6229 }
6230
6231 static void
6232 gen9_hevc_pak_add_surface_state(VADriverContextP ctx,
6233                                 struct encode_state *encode_state,
6234                                 struct intel_encoder_context *encoder_context,
6235                                 struct object_surface *obj_surface,
6236                                 enum GEN9_HEVC_ENC_SURFACE_TYPE type)
6237 {
6238     struct intel_batchbuffer *batch = encoder_context->base.batch;
6239
6240     BEGIN_BCS_BATCH(batch, 3);
6241
6242     OUT_BCS_BATCH(batch, HCP_SURFACE_STATE | (3 - 2));
6243     OUT_BCS_BATCH(batch,
6244                   (type << 28) |
6245                   (obj_surface->width - 1));
6246     OUT_BCS_BATCH(batch,
6247                   (((obj_surface->fourcc == VA_FOURCC_P010) ?
6248                     SURFACE_FORMAT_P010 :
6249                     SURFACE_FORMAT_PLANAR_420_8) << 28) |
6250                   (obj_surface->y_cb_offset));
6251
6252     ADVANCE_BCS_BATCH(batch);
6253 }
6254
6255 #define OUT_BUFFER(buf_bo, is_target, ma)                               \
6256     do {                                                                \
6257         if (buf_bo) {                                                   \
6258             OUT_RELOC64(batch,                                          \
6259                           buf_bo,                                       \
6260                           I915_GEM_DOMAIN_RENDER,                       \
6261                           is_target ? I915_GEM_DOMAIN_RENDER : 0,       \
6262                           0);                                           \
6263         } else {                                                        \
6264             OUT_BCS_BATCH(batch, 0);                                    \
6265             OUT_BCS_BATCH(batch, 0);                                    \
6266         }                                                               \
6267         if (ma)                                                         \
6268             OUT_BCS_BATCH(batch, priv_ctx->mocs);                       \
6269     } while (0);
6270
6271 #define OUT_BUFFER_MA_TARGET(buf_bo)       OUT_BUFFER(buf_bo, 1, 1)
6272 #define OUT_BUFFER_MA_REFERENCE(buf_bo)    OUT_BUFFER(buf_bo, 0, 1)
6273 #define OUT_BUFFER_NMA_REFERENCE(buf_bo)   OUT_BUFFER(buf_bo, 0, 0)
6274
6275 static void
6276 gen9_hevc_pak_add_pipe_buf_addr_state(VADriverContextP ctx,
6277                                       struct encode_state *encode_state,
6278                                       struct intel_encoder_context *encoder_context)
6279 {
6280     struct i965_driver_data *i965 = i965_driver_data(ctx);
6281     struct intel_batchbuffer *batch = encoder_context->base.batch;
6282     struct encoder_vme_mfc_context *pak_context = NULL;
6283     struct gen9_hevc_encoder_context *priv_ctx = NULL;
6284     dri_bo *bo = NULL;
6285     int i = 0;
6286
6287     pak_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
6288     priv_ctx = (struct gen9_hevc_encoder_context *)pak_context->private_enc_ctx;
6289
6290     if (IS_KBL(i965->intel.device_info) || IS_GLK(i965->intel.device_info)) {
6291         BEGIN_BCS_BATCH(batch, 104);
6292
6293         OUT_BCS_BATCH(batch, HCP_PIPE_BUF_ADDR_STATE | (104 - 2));
6294     } else {
6295         BEGIN_BCS_BATCH(batch, 95);
6296
6297         OUT_BCS_BATCH(batch, HCP_PIPE_BUF_ADDR_STATE | (95 - 2));
6298     }
6299
6300     OUT_BUFFER_MA_TARGET(priv_ctx->reconstructed_object.obj_surface->bo);
6301     OUT_BUFFER_MA_TARGET(priv_ctx->deblocking_filter_line_buffer.bo);
6302     OUT_BUFFER_MA_TARGET(priv_ctx->deblocking_filter_tile_line_buffer.bo);
6303     OUT_BUFFER_MA_TARGET(priv_ctx->deblocking_filter_tile_column_buffer.bo);
6304     OUT_BUFFER_MA_TARGET(priv_ctx->metadata_line_buffer.bo);
6305     OUT_BUFFER_MA_TARGET(priv_ctx->metadata_tile_line_buffer.bo);
6306     OUT_BUFFER_MA_TARGET(priv_ctx->metadata_tile_column_buffer.bo);
6307     OUT_BUFFER_MA_TARGET(priv_ctx->sao_line_buffer.bo);
6308     OUT_BUFFER_MA_TARGET(priv_ctx->sao_tile_line_buffer.bo);
6309     OUT_BUFFER_MA_TARGET(priv_ctx->sao_tile_column_buffer.bo);
6310     OUT_BUFFER_MA_TARGET(priv_ctx->
6311                          mv_temporal_buffer[GEN9_MAX_MV_TEMPORAL_BUFFERS - 1].bo);
6312     OUT_BUFFER_MA_TARGET(NULL);
6313
6314     for (i = 0; i < GEN9_MAX_REF_SURFACES; i++) {
6315         if (priv_ctx->reference_surfaces[i].obj_surface &&
6316             priv_ctx->reference_surfaces[i].obj_surface->bo) {
6317             bo = priv_ctx->reference_surfaces[i].obj_surface->bo;
6318
6319             OUT_BUFFER_NMA_REFERENCE(bo);
6320         } else
6321             OUT_BUFFER_NMA_REFERENCE(NULL);
6322     }
6323     OUT_BCS_BATCH(batch, priv_ctx->mocs);
6324
6325     OUT_BUFFER_MA_TARGET(priv_ctx->
6326                          uncompressed_picture_source.obj_surface->bo);
6327     OUT_BUFFER_MA_TARGET(NULL);
6328     OUT_BUFFER_MA_TARGET(NULL);
6329     OUT_BUFFER_MA_TARGET(NULL);
6330
6331     for (i = 0; i < GEN9_MAX_MV_TEMPORAL_BUFFERS - 1; i++) {
6332         bo = priv_ctx->mv_temporal_buffer[i].bo;
6333
6334         if (bo) {
6335             OUT_BUFFER_NMA_REFERENCE(bo);
6336         } else
6337             OUT_BUFFER_NMA_REFERENCE(NULL);
6338     }
6339     OUT_BCS_BATCH(batch, priv_ctx->mocs);
6340
6341     OUT_BUFFER_MA_TARGET(NULL);
6342     OUT_BUFFER_MA_TARGET(NULL);
6343     OUT_BUFFER_MA_TARGET(NULL);
6344     OUT_BUFFER_MA_TARGET(NULL);
6345
6346     if (IS_KBL(i965->intel.device_info) || IS_GLK(i965->intel.device_info)) {
6347         for (i = 0; i < 9; i++)
6348             OUT_BCS_BATCH(batch, 0);
6349     }
6350
6351     ADVANCE_BCS_BATCH(batch);
6352 }
6353
6354 static void
6355 gen9_hevc_pak_add_ind_obj_base_addr_state(VADriverContextP ctx,
6356                                           struct encode_state *encode_state,
6357                                           struct intel_encoder_context *encoder_context)
6358 {
6359     struct intel_batchbuffer *batch = encoder_context->base.batch;
6360     struct encoder_vme_mfc_context *pak_context = NULL;
6361     struct gen9_hevc_encoder_context *priv_ctx = NULL;
6362     struct gen9_hevc_encoder_state *priv_state = NULL;
6363
6364     pak_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
6365     priv_ctx = (struct gen9_hevc_encoder_context *)pak_context->private_enc_ctx;
6366     priv_state = (struct gen9_hevc_encoder_state *)pak_context->private_enc_state;
6367
6368     BEGIN_BCS_BATCH(batch, 14);
6369
6370     OUT_BCS_BATCH(batch, HCP_IND_OBJ_BASE_ADDR_STATE | (14 - 2));
6371     OUT_BUFFER_MA_REFERENCE(NULL);
6372     OUT_BUFFER_NMA_REFERENCE(NULL);
6373
6374     OUT_RELOC64(batch,
6375                 priv_ctx->res_mb_code_surface.bo,
6376                 I915_GEM_DOMAIN_INSTRUCTION, 0,
6377                 priv_state->mb_data_offset);
6378     OUT_BCS_BATCH(batch, priv_ctx->mocs);
6379
6380     OUT_RELOC64(batch,
6381                 priv_ctx->indirect_pak_bse_object.bo,
6382                 I915_GEM_DOMAIN_RENDER, I915_GEM_DOMAIN_RENDER,
6383                 priv_ctx->indirect_pak_bse_object.offset);
6384     OUT_BCS_BATCH(batch, priv_ctx->mocs);
6385
6386     OUT_RELOC64(batch,
6387                 priv_ctx->indirect_pak_bse_object.bo,
6388                 I915_GEM_DOMAIN_RENDER, 0,
6389                 priv_ctx->indirect_pak_bse_object.end_offset);
6390
6391     ADVANCE_BCS_BATCH(batch);
6392 }
6393
6394 static void
6395 gen9_hevc_pak_set_qm(VADriverContextP ctx,
6396                      int size_id,
6397                      int color_component,
6398                      int pred_type,
6399                      int dc,
6400                      unsigned int *qm,
6401                      int qm_length,
6402                      struct intel_encoder_context *encoder_context)
6403 {
6404     struct intel_batchbuffer *batch = encoder_context->base.batch;
6405     unsigned int qm_buffer[16];
6406
6407     memset(qm_buffer, 0, sizeof(qm_buffer));
6408     memcpy(qm_buffer, qm, qm_length * 4);
6409
6410     BEGIN_BCS_BATCH(batch, 18);
6411
6412     OUT_BCS_BATCH(batch, HCP_QM_STATE | (18 - 2));
6413     OUT_BCS_BATCH(batch,
6414                   dc << 5 |
6415                   color_component << 3 |
6416                   size_id << 1 |
6417                   pred_type);
6418     intel_batchbuffer_data(batch, qm_buffer, 16 * 4);
6419
6420     ADVANCE_BCS_BATCH(batch);
6421 }
6422
6423 static unsigned int qm_default[16] = {
6424     0x10101010, 0x10101010, 0x10101010, 0x10101010,
6425     0x10101010, 0x10101010, 0x10101010, 0x10101010,
6426     0x10101010, 0x10101010, 0x10101010, 0x10101010,
6427     0x10101010, 0x10101010, 0x10101010, 0x10101010
6428 };
6429
6430 static void
6431 gen9_hevc_pak_add_qm_state(VADriverContextP ctx,
6432                            struct encode_state *encode_state,
6433                            struct intel_encoder_context *encoder_context)
6434 {
6435     int i;
6436
6437     for (i = 0; i < 6; i++)
6438         gen9_hevc_pak_set_qm(ctx,
6439                              0, i % 3, i / 3, 0,
6440                              qm_default, 4,
6441                              encoder_context);
6442
6443     for (i = 0; i < 6; i++)
6444         gen9_hevc_pak_set_qm(ctx,
6445                              1, i % 3, i / 3, 0,
6446                              qm_default, 16,
6447                              encoder_context);
6448
6449     for (i = 0; i < 6; i++)
6450         gen9_hevc_pak_set_qm(ctx,
6451                              2, i % 3, i / 3, 16,
6452                              qm_default, 16,
6453                              encoder_context);
6454
6455     for (i = 0; i < 2; i++)
6456         gen9_hevc_pak_set_qm(ctx,
6457                              3, 0, i % 2, 16,
6458                              qm_default, 16,
6459                              encoder_context);
6460 }
6461
6462 static void
6463 gen9_hevc_pak_set_fqm(VADriverContextP ctx,
6464                       int size_id,
6465                       int color_component,
6466                       int pred_type,
6467                       int dc,
6468                       unsigned int *fqm,
6469                       int fqm_length,
6470                       struct intel_encoder_context *encoder_context)
6471 {
6472     struct intel_batchbuffer *batch = encoder_context->base.batch;
6473     unsigned int fqm_buffer[32];
6474
6475     memset(fqm_buffer, 0, sizeof(fqm_buffer));
6476     memcpy(fqm_buffer, fqm, fqm_length * 4);
6477
6478     BEGIN_BCS_BATCH(batch, 34);
6479
6480     OUT_BCS_BATCH(batch, HCP_FQM_STATE | (34 - 2));
6481     OUT_BCS_BATCH(batch,
6482                   dc << 16 |
6483                   color_component << 3 |
6484                   size_id << 1 |
6485                   pred_type);
6486     intel_batchbuffer_data(batch, fqm_buffer, 32 * 4);
6487
6488     ADVANCE_BCS_BATCH(batch);
6489 }
6490
6491 static unsigned int fm_default[32] = {
6492     0x10001000, 0x10001000, 0x10001000, 0x10001000,
6493     0x10001000, 0x10001000, 0x10001000, 0x10001000,
6494     0x10001000, 0x10001000, 0x10001000, 0x10001000,
6495     0x10001000, 0x10001000, 0x10001000, 0x10001000,
6496     0x10001000, 0x10001000, 0x10001000, 0x10001000,
6497     0x10001000, 0x10001000, 0x10001000, 0x10001000,
6498     0x10001000, 0x10001000, 0x10001000, 0x10001000,
6499     0x10001000, 0x10001000, 0x10001000, 0x10001000
6500 };
6501
6502 static void
6503 gen9_hevc_pak_add_fm_state(VADriverContextP ctx,
6504                            struct encode_state *encode_state,
6505                            struct intel_encoder_context *encoder_context)
6506 {
6507     gen9_hevc_pak_set_fqm(ctx,
6508                           0, 0, 0, 0,
6509                           fm_default, 8,
6510                           encoder_context);
6511     gen9_hevc_pak_set_fqm(ctx,
6512                           0, 0, 1, 0,
6513                           fm_default, 8,
6514                           encoder_context);
6515     gen9_hevc_pak_set_fqm(ctx,
6516                           1, 0, 0, 0,
6517                           fm_default, 32,
6518                           encoder_context);
6519     gen9_hevc_pak_set_fqm(ctx,
6520                           1, 0, 1, 0,
6521                           fm_default, 32,
6522                           encoder_context);
6523     gen9_hevc_pak_set_fqm(ctx,
6524                           2, 0, 0, 0x1000,
6525                           fm_default, 0,
6526                           encoder_context);
6527     gen9_hevc_pak_set_fqm(ctx,
6528                           2, 0, 1, 0x1000,
6529                           fm_default, 0,
6530                           encoder_context);
6531     gen9_hevc_pak_set_fqm(ctx,
6532                           3, 0, 0, 0x1000,
6533                           fm_default, 0,
6534                           encoder_context);
6535     gen9_hevc_pak_set_fqm(ctx,
6536                           3, 0, 1, 0x1000,
6537                           fm_default, 0,
6538                           encoder_context);
6539 }
6540
6541 static void
6542 gen9_hevc_set_reflist(VADriverContextP ctx,
6543                       struct gen9_hevc_encoder_context *priv_ctx,
6544                       VAEncPictureParameterBufferHEVC *pic_param,
6545                       VAEncSliceParameterBufferHEVC *slice_param,
6546                       int list_idx,
6547                       struct intel_batchbuffer *batch)
6548 {
6549     struct i965_driver_data *i965 = i965_driver_data(ctx);
6550     int num_ref_minus1 = (list_idx ?
6551                           slice_param->num_ref_idx_l1_active_minus1 :
6552                           slice_param->num_ref_idx_l0_active_minus1);
6553     VAPictureHEVC *ref_list = (list_idx ?
6554                                slice_param->ref_pic_list1 :
6555                                slice_param->ref_pic_list0);
6556     VAPictureHEVC *curr_pic = &pic_param->decoded_curr_pic;
6557     struct object_surface *obj_surface = NULL;
6558     int frame_idx;
6559     int i = 0, j = 0;
6560
6561     BEGIN_BCS_BATCH(batch, 18);
6562
6563     OUT_BCS_BATCH(batch, HCP_REF_IDX_STATE | (18 - 2));
6564     OUT_BCS_BATCH(batch,
6565                   num_ref_minus1 << 1 |
6566                   list_idx);
6567
6568     for (i = 0; i < 16; i++) {
6569         frame_idx = -1;
6570         obj_surface = SURFACE(ref_list[i].picture_id);
6571         if (i < MIN((num_ref_minus1 + 1), GEN9_MAX_REF_SURFACES) && obj_surface) {
6572             for (j = 0; j < GEN9_MAX_REF_SURFACES; j++) {
6573                 if (obj_surface == priv_ctx->reference_surfaces[j].obj_surface) {
6574                     frame_idx = j;
6575                     break;
6576                 }
6577             }
6578         }
6579
6580         if (i < MIN((num_ref_minus1 + 1), GEN9_MAX_REF_SURFACES) &&
6581             frame_idx >= 0) {
6582             OUT_BCS_BATCH(batch,
6583                           1 << 15 |
6584                           0 << 14 |
6585                           !!(ref_list[i].flags & VA_PICTURE_HEVC_LONG_TERM_REFERENCE) << 13 |
6586                           0 << 12 |
6587                           0 << 11 |
6588                           frame_idx << 8 |
6589                           (CLAMP(-128, 127, curr_pic->pic_order_cnt -
6590                                  ref_list[i].pic_order_cnt) & 0xff));
6591         } else {
6592             OUT_BCS_BATCH(batch, 0);
6593         }
6594     }
6595
6596     ADVANCE_BCS_BATCH(batch);
6597 }
6598
6599 static void
6600 gen9_hevc_pak_add_slice_state(VADriverContextP ctx,
6601                               struct encode_state *encode_state,
6602                               struct intel_encoder_context *encoder_context,
6603                               int slice_idx,
6604                               struct intel_batchbuffer *batch)
6605 {
6606     struct i965_driver_data *i965 = i965_driver_data(ctx);
6607     struct encoder_vme_mfc_context *pak_context = NULL;
6608     struct gen9_hevc_encoder_state *priv_state = NULL;
6609     VAEncPictureParameterBufferHEVC *pic_param = NULL;
6610     VAEncSliceParameterBufferHEVC *slice_param = NULL;
6611     int slice_hor_pos, slice_ver_pos, next_slice_hor_pos, next_slice_ver_pos;
6612     int slice_type = 0, slice_end = 0, last_slice = 0;
6613     int collocated_ref_idx = 0;
6614
6615     pak_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
6616     priv_state = (struct gen9_hevc_encoder_state *)pak_context->private_enc_state;
6617     pic_param = (VAEncPictureParameterBufferHEVC *)encode_state->pic_param_ext->buffer;
6618     slice_param = (VAEncSliceParameterBufferHEVC *)encode_state->slice_params_ext[slice_idx]->buffer;
6619
6620     slice_type = slice_param->slice_type;
6621     slice_end = slice_param->slice_segment_address + slice_param->num_ctu_in_slice;
6622     slice_hor_pos = slice_param->slice_segment_address % priv_state->width_in_lcu;
6623     slice_ver_pos = slice_param->slice_segment_address / priv_state->width_in_lcu;
6624     next_slice_hor_pos = slice_end % priv_state->width_in_lcu;
6625     next_slice_ver_pos = slice_end / priv_state->width_in_lcu;
6626
6627     if (slice_end >= priv_state->width_in_lcu * priv_state->height_in_lcu ||
6628         slice_idx == encode_state->num_slice_params_ext - 1)
6629         last_slice = 1;
6630
6631     if (priv_state->picture_coding_type != HEVC_SLICE_I &&
6632         slice_param->slice_fields.bits.slice_temporal_mvp_enabled_flag &&
6633         slice_param->slice_fields.bits.collocated_from_l0_flag)
6634         collocated_ref_idx = pic_param->collocated_ref_pic_index;
6635
6636     if (IS_KBL(i965->intel.device_info) || IS_GLK(i965->intel.device_info)) {
6637         BEGIN_BCS_BATCH(batch, 11);
6638
6639         OUT_BCS_BATCH(batch, HCP_SLICE_STATE | (11 - 2));
6640     } else {
6641         BEGIN_BCS_BATCH(batch, 9);
6642
6643         OUT_BCS_BATCH(batch, HCP_SLICE_STATE | (9 - 2));
6644     }
6645
6646     OUT_BCS_BATCH(batch,
6647                   slice_ver_pos << 16 |
6648                   slice_hor_pos);
6649     OUT_BCS_BATCH(batch,
6650                   next_slice_ver_pos << 16 |
6651                   next_slice_hor_pos);
6652     OUT_BCS_BATCH(batch,
6653                   (slice_param->slice_cr_qp_offset & 0x1f) << 17 |
6654                   (slice_param->slice_cb_qp_offset & 0x1f) << 12 |
6655                   (pic_param->pic_init_qp + slice_param->slice_qp_delta) << 6 |
6656                   slice_param->slice_fields.bits.slice_temporal_mvp_enabled_flag << 5 |
6657                   slice_param->slice_fields.bits.dependent_slice_segment_flag << 4 |
6658                   last_slice << 2 |
6659                   slice_type);
6660     OUT_BCS_BATCH(batch,
6661                   collocated_ref_idx << 26 |
6662                   (slice_param->max_num_merge_cand - 1)  << 23 |
6663                   slice_param->slice_fields.bits.cabac_init_flag << 22 |
6664                   slice_param->luma_log2_weight_denom << 19 |
6665                   (slice_param->luma_log2_weight_denom + slice_param->delta_chroma_log2_weight_denom) << 16 |
6666                   slice_param->slice_fields.bits.collocated_from_l0_flag << 15 |
6667                   priv_state->low_delay << 14 |
6668                   slice_param->slice_fields.bits.mvd_l1_zero_flag << 13 |
6669                   slice_param->slice_fields.bits.slice_sao_luma_flag << 12 |
6670                   slice_param->slice_fields.bits.slice_sao_chroma_flag << 11 |
6671                   slice_param->slice_fields.bits.slice_loop_filter_across_slices_enabled_flag << 10 |
6672                   (slice_param->slice_beta_offset_div2 & 0xf) << 5 |
6673                   (slice_param->slice_tc_offset_div2 & 0xf) << 1 |
6674                   slice_param->slice_fields.bits.slice_deblocking_filter_disabled_flag);
6675     OUT_BCS_BATCH(batch, 0);
6676
6677     if (!pic_param->pic_fields.bits.reference_pic_flag &&
6678         priv_state->picture_coding_type != HEVC_SLICE_I)
6679         OUT_BCS_BATCH(batch, 0 << 26 |
6680                       8 << 20);
6681     else
6682         OUT_BCS_BATCH(batch, 5 << 26 |
6683                       11 << 20);
6684
6685     OUT_BCS_BATCH(batch,
6686                   1 << 10 |
6687                   1 << 9  |
6688                   1 << 2  |
6689                   1 << 1  |
6690                   0);
6691     OUT_BCS_BATCH(batch, 0);
6692
6693     if (IS_KBL(i965->intel.device_info) || IS_GLK(i965->intel.device_info)) {
6694         OUT_BCS_BATCH(batch, 0);
6695         OUT_BCS_BATCH(batch, 0);
6696     }
6697
6698     ADVANCE_BCS_BATCH(batch);
6699 }
6700
6701 static int
6702 gen9_hevc_find_skipemulcnt(unsigned char *buf, unsigned int bits_length)
6703 {
6704     int skip_cnt = 0, i = 0;
6705
6706     if ((bits_length >> 3) < 6)
6707         return 0;
6708
6709     for (i = 0; i < 3; i++)
6710         if (buf[i] != 0)
6711             break;
6712
6713     if (i > 1) {
6714         if (buf[i] == 1)
6715             skip_cnt = i + 3;
6716     }
6717
6718     return skip_cnt;
6719 }
6720
6721 static void
6722 gen9_hevc_pak_insert_object(unsigned int *data_buffer,
6723                             unsigned int data_size,
6724                             unsigned char emulation_flag,
6725                             int is_last_header,
6726                             int is_end_of_slice,
6727                             int skip_emul_byte_cnt,
6728                             struct intel_batchbuffer *batch)
6729 {
6730     int length_in_dws = ALIGN(data_size, 32) >> 5;
6731     int data_bits_in_last_dw = data_size & 0x1f;
6732     int skip_cnt = skip_emul_byte_cnt;
6733
6734     if (data_bits_in_last_dw == 0)
6735         data_bits_in_last_dw = 32;
6736
6737     if (emulation_flag) {
6738         if (!skip_cnt)
6739             skip_cnt = gen9_hevc_find_skipemulcnt((unsigned char *)data_buffer,
6740                                                   data_size);
6741     }
6742
6743     BEGIN_BCS_BATCH(batch, length_in_dws + 2);
6744
6745     OUT_BCS_BATCH(batch, HCP_INSERT_PAK_OBJECT | (length_in_dws + 2 - 2));
6746     OUT_BCS_BATCH(batch,
6747                   (0 << 31) |
6748                   (0 << 16) |
6749                   (0 << 15) |
6750                   (data_bits_in_last_dw << 8) |
6751                   (skip_cnt << 4) |
6752                   ((!!emulation_flag) << 3) |
6753                   ((!!is_last_header) << 2) |
6754                   ((!!is_end_of_slice) << 1) |
6755                   (0 << 0));
6756     intel_batchbuffer_data(batch, data_buffer, length_in_dws * 4);
6757
6758     ADVANCE_BCS_BATCH(batch);
6759 }
6760
6761 static void
6762 gen9_hevc_pak_add_refs(VADriverContextP ctx,
6763                        struct encode_state *encode_state,
6764                        struct intel_encoder_context *encoder_context,
6765                        int slice_idx,
6766                        struct intel_batchbuffer *batch)
6767 {
6768     struct encoder_vme_mfc_context *pak_context = NULL;
6769     struct gen9_hevc_encoder_context *priv_ctx = NULL;
6770     VAEncPictureParameterBufferHEVC *pic_param = NULL;
6771     VAEncSliceParameterBufferHEVC *slice_param = NULL;
6772
6773     pak_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
6774     priv_ctx = (struct gen9_hevc_encoder_context *)pak_context->private_enc_ctx;
6775     pic_param = (VAEncPictureParameterBufferHEVC *)encode_state->pic_param_ext->buffer;
6776     slice_param = (VAEncSliceParameterBufferHEVC *)encode_state->slice_params_ext[slice_idx]->buffer;
6777
6778     if (slice_param->slice_type == HEVC_SLICE_I)
6779         return;
6780
6781     gen9_hevc_set_reflist(ctx, priv_ctx, pic_param, slice_param, 0, batch);
6782
6783     if (slice_param->slice_type == HEVC_SLICE_P)
6784         return;
6785
6786     gen9_hevc_set_reflist(ctx, priv_ctx, pic_param, slice_param, 1, batch);
6787 }
6788
6789 static void
6790 gen9_hevc_pak_insert_packed_data(VADriverContextP ctx,
6791                                  struct encode_state *encode_state,
6792                                  struct intel_encoder_context *encoder_context,
6793                                  int slice_idx,
6794                                  struct intel_batchbuffer *batch)
6795 {
6796     VAEncPackedHeaderParameterBuffer *param = NULL;
6797     unsigned int *header_data = NULL;
6798     unsigned int length_in_bits = 0;
6799     int packed_type = 0;
6800     int idx = 0, idx_offset = 0;
6801     int i = 0;
6802
6803     for (i = 0; i < 4; i++) {
6804         idx_offset = 0;
6805         switch (i) {
6806         case 0:
6807             packed_type = VAEncPackedHeaderHEVC_VPS;
6808             break;
6809         case 1:
6810             packed_type = VAEncPackedHeaderHEVC_VPS;
6811             idx_offset = 1;
6812             break;
6813         case 2:
6814             packed_type = VAEncPackedHeaderHEVC_PPS;
6815             break;
6816         case 3:
6817             packed_type = VAEncPackedHeaderHEVC_SEI;
6818             break;
6819         default:
6820             break;
6821         }
6822
6823         idx = va_enc_packed_type_to_idx(packed_type) + idx_offset;
6824         if (encode_state->packed_header_data[idx]) {
6825             param = (VAEncPackedHeaderParameterBuffer *)encode_state->packed_header_param[idx]->buffer;
6826             header_data = (unsigned int *)encode_state->packed_header_data[idx]->buffer;
6827             length_in_bits = param->bit_length;
6828
6829             gen9_hevc_pak_insert_object(header_data, length_in_bits,
6830                                         !param->has_emulation_bytes, 0, 0, 0,
6831                                         batch);
6832         }
6833     }
6834 }
6835
6836 static void
6837 gen9_hevc_pak_insert_slice_header(VADriverContextP ctx,
6838                                   struct encode_state *encode_state,
6839                                   struct intel_encoder_context *encoder_context,
6840                                   int slice_idx,
6841                                   struct intel_batchbuffer *batch)
6842 {
6843     VAEncPackedHeaderParameterBuffer *param = NULL;
6844     unsigned int *header_data = NULL;
6845     unsigned int length_in_bits = 0;
6846     int count = 0, start_index = -1;
6847     int i = 0;
6848
6849     count = encode_state->slice_rawdata_count[slice_idx];
6850     start_index = encode_state->slice_rawdata_index[slice_idx] &
6851                   SLICE_PACKED_DATA_INDEX_MASK;
6852
6853     for (i = 0; i < count; i++) {
6854         param = (VAEncPackedHeaderParameterBuffer *)
6855                 (encode_state->packed_header_params_ext[start_index + i]->buffer);
6856
6857         if (param->type == VAEncPackedHeaderSlice)
6858             continue;
6859
6860         header_data = (unsigned int *)encode_state->packed_header_data_ext[start_index]->buffer;
6861         length_in_bits = param->bit_length;
6862         gen9_hevc_pak_insert_object(header_data, length_in_bits,
6863                                     !param->has_emulation_bytes, 0, 0, 0,
6864                                     batch);
6865     }
6866
6867     start_index = -1;
6868     if (encode_state->slice_header_index[slice_idx] & SLICE_PACKED_DATA_INDEX_TYPE)
6869         start_index = encode_state->slice_header_index[slice_idx] &
6870                       SLICE_PACKED_DATA_INDEX_MASK;
6871     if (start_index == -1) {
6872         VAEncSequenceParameterBufferHEVC *seq_param = (VAEncSequenceParameterBufferHEVC *)encode_state->seq_param_ext->buffer;
6873         VAEncPictureParameterBufferHEVC *pic_param = (VAEncPictureParameterBufferHEVC *)encode_state->pic_param_ext->buffer;
6874         VAEncSliceParameterBufferHEVC *slice_param = (VAEncSliceParameterBufferHEVC *)encode_state->slice_params_ext[slice_idx]->buffer;
6875         unsigned char *slice_header = NULL;
6876         int slice_header_bits = 0;
6877
6878         slice_header_bits = build_hevc_slice_header(seq_param,
6879                                                     pic_param,
6880                                                     slice_param,
6881                                                     &slice_header,
6882                                                     0);
6883
6884         gen9_hevc_pak_insert_object((unsigned int *)slice_header, slice_header_bits,
6885                                     1, 1, 0, 5,
6886                                     batch);
6887
6888         free(slice_header);
6889     } else {
6890         param = (VAEncPackedHeaderParameterBuffer *)
6891                 (encode_state->packed_header_params_ext[start_index]->buffer);
6892         header_data = (unsigned int *)encode_state->packed_header_data_ext[start_index]->buffer;
6893         length_in_bits = param->bit_length;
6894
6895         gen9_hevc_pak_insert_object(header_data, length_in_bits,
6896                                     !param->has_emulation_bytes, 1, 0, 0,
6897                                     batch);
6898     }
6899 }
6900
6901 static VAStatus
6902 gen9_hevc_pak_slice_level(VADriverContextP ctx,
6903                           struct encode_state *encode_state,
6904                           struct intel_encoder_context *encoder_context,
6905                           int slice_idx)
6906 {
6907     struct intel_batchbuffer *batch = encoder_context->base.batch;
6908     struct encoder_vme_mfc_context *pak_context = NULL;
6909     struct gen9_hevc_encoder_context *priv_ctx = NULL;
6910     struct generic_enc_codec_state *generic_state = NULL;
6911     struct gen9_hevc_encoder_state *priv_state = NULL;
6912     VAEncSliceParameterBufferHEVC *slice_param = NULL;
6913     struct intel_batchbuffer *slice_batch = NULL;
6914     struct gpe_mi_batch_buffer_start_parameter second_level_batch;
6915     VAStatus va_status = VA_STATUS_SUCCESS;
6916     int slice_offset = 0;
6917
6918     pak_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
6919     priv_ctx = (struct gen9_hevc_encoder_context *)pak_context->private_enc_ctx;
6920     generic_state = (struct generic_enc_codec_state *)pak_context->generic_enc_state;
6921     priv_state = (struct gen9_hevc_encoder_state *)pak_context->private_enc_state;
6922     slice_param = (VAEncSliceParameterBufferHEVC *)encode_state->slice_params_ext[slice_idx]->buffer;
6923
6924     slice_batch = priv_ctx->res_pak_slice_batch_buffer;
6925
6926     if (generic_state->curr_pak_pass == 0) {
6927         slice_offset = intel_batchbuffer_used_size(slice_batch);
6928         priv_state->slice_batch_offset[slice_idx] = slice_offset;
6929
6930         if (slice_idx < encode_state->num_slice_params_ext - 1)
6931             priv_state->slice_start_lcu[slice_idx + 1] =
6932                 priv_state->slice_start_lcu[slice_idx] +
6933                 slice_param->num_ctu_in_slice;
6934
6935         gen9_hevc_pak_add_refs(ctx, encode_state, encoder_context,
6936                                slice_idx, slice_batch);
6937         gen9_hevc_pak_add_slice_state(ctx, encode_state, encoder_context,
6938                                       slice_idx, slice_batch);
6939
6940         if (slice_idx == 0)
6941             gen9_hevc_pak_insert_packed_data(ctx, encode_state, encoder_context,
6942                                              slice_idx, slice_batch);
6943
6944         gen9_hevc_pak_insert_slice_header(ctx, encode_state, encoder_context,
6945                                           slice_idx, slice_batch);
6946
6947         BEGIN_BCS_BATCH(slice_batch, 2);
6948         OUT_BCS_BATCH(slice_batch, 0);
6949         OUT_BCS_BATCH(slice_batch, MI_BATCH_BUFFER_END);
6950         ADVANCE_BCS_BATCH(slice_batch);
6951     } else
6952         slice_offset = priv_state->slice_batch_offset[slice_idx];
6953
6954     memset(&second_level_batch, 0, sizeof(second_level_batch));
6955     second_level_batch.offset = slice_offset;
6956     second_level_batch.is_second_level = 1;
6957     second_level_batch.bo = slice_batch->buffer;
6958     gen8_gpe_mi_batch_buffer_start(ctx, batch, &second_level_batch);
6959
6960     memset(&second_level_batch, 0, sizeof(second_level_batch));
6961     second_level_batch.offset = priv_state->slice_start_lcu[slice_idx] *
6962                                 priv_state->pak_obj_size;
6963     second_level_batch.is_second_level = 1;
6964     second_level_batch.bo = priv_ctx->res_mb_code_surface.bo;
6965     gen8_gpe_mi_batch_buffer_start(ctx, batch, &second_level_batch);
6966
6967     return va_status;
6968 }
6969
6970 static VAStatus
6971 gen9_hevc_pak_pipeline_prepare(VADriverContextP ctx,
6972                                struct encode_state *encode_state,
6973                                struct intel_encoder_context *encoder_context)
6974 {
6975     struct i965_driver_data *i965 = i965_driver_data(ctx);
6976     struct encoder_vme_mfc_context *pak_context = NULL;
6977     struct gen9_hevc_encoder_context *priv_ctx = NULL;
6978     struct gen9_hevc_encoder_state *priv_state = NULL;
6979     struct gen9_hevc_surface_priv *surface_priv;
6980     VAEncPictureParameterBufferHEVC *pic_param = NULL;
6981     struct i965_coded_buffer_segment *coded_buffer_segment = NULL;
6982     struct object_surface *obj_surface = NULL;
6983     struct object_buffer *obj_buffer = NULL;
6984     dri_bo *bo = NULL;
6985     int i = 0;
6986
6987     pak_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
6988     priv_ctx = (struct gen9_hevc_encoder_context *)pak_context->private_enc_ctx;
6989     priv_state = (struct gen9_hevc_encoder_state *)pak_context->private_enc_state;
6990     pic_param = (VAEncPictureParameterBufferHEVC *)encode_state->pic_param_ext->buffer;
6991
6992     if (priv_ctx->uncompressed_picture_source.obj_surface &&
6993         priv_ctx->uncompressed_picture_source.obj_surface->bo)
6994         dri_bo_unreference(priv_ctx->uncompressed_picture_source.obj_surface->bo);
6995     priv_ctx->uncompressed_picture_source.obj_surface = encode_state->input_yuv_object;
6996     priv_ctx->uncompressed_picture_source.surface_id = encoder_context->input_yuv_surface;
6997     dri_bo_reference(priv_ctx->uncompressed_picture_source.obj_surface->bo);
6998
6999     if (priv_ctx->reconstructed_object.obj_surface &&
7000         priv_ctx->reconstructed_object.obj_surface->bo)
7001         dri_bo_unreference(priv_ctx->reconstructed_object.obj_surface->bo);
7002     priv_ctx->reconstructed_object.obj_surface = encode_state->reconstructed_object;
7003     priv_ctx->reconstructed_object.surface_id = pic_param->decoded_curr_pic.picture_id;
7004     dri_bo_reference(priv_ctx->reconstructed_object.obj_surface->bo);
7005
7006     surface_priv = (struct gen9_hevc_surface_priv *)encode_state->reconstructed_object->private_data;
7007     if (surface_priv) {
7008         if (priv_ctx->mv_temporal_buffer[GEN9_MAX_MV_TEMPORAL_BUFFERS - 1].bo)
7009             dri_bo_unreference(priv_ctx->mv_temporal_buffer[GEN9_MAX_MV_TEMPORAL_BUFFERS - 1].bo);
7010         priv_ctx->mv_temporal_buffer[GEN9_MAX_MV_TEMPORAL_BUFFERS - 1].bo =
7011             surface_priv->motion_vector_temporal_bo;
7012         dri_bo_reference(surface_priv->motion_vector_temporal_bo);
7013     }
7014
7015     if (priv_state->picture_coding_type != HEVC_SLICE_I) {
7016         for (i = 0; i < GEN9_MAX_REF_SURFACES; i++) {
7017             obj_surface = encode_state->reference_objects[i];
7018             if (obj_surface && obj_surface->bo) {
7019                 if (priv_ctx->reference_surfaces[i].obj_surface &&
7020                     priv_ctx->reference_surfaces[i].obj_surface->bo)
7021                     dri_bo_unreference(priv_ctx->reference_surfaces[i].obj_surface->bo);
7022                 priv_ctx->reference_surfaces[i].obj_surface = obj_surface;
7023                 priv_ctx->reference_surfaces[i].surface_id = pic_param->reference_frames[i].picture_id;
7024                 dri_bo_reference(obj_surface->bo);
7025
7026                 surface_priv = (struct gen9_hevc_surface_priv *) obj_surface->private_data;
7027                 if (surface_priv) {
7028                     if (priv_ctx->mv_temporal_buffer[i].bo)
7029                         dri_bo_unreference(priv_ctx->mv_temporal_buffer[i].bo);
7030                     priv_ctx->mv_temporal_buffer[i].bo = surface_priv->motion_vector_temporal_bo;
7031                     dri_bo_reference(surface_priv->motion_vector_temporal_bo);
7032                 }
7033             } else {
7034                 break;
7035             }
7036         }
7037     }
7038
7039     obj_buffer = encode_state->coded_buf_object;
7040     bo = obj_buffer->buffer_store->bo;
7041
7042     if (priv_ctx->indirect_pak_bse_object.bo)
7043         dri_bo_unreference(priv_ctx->indirect_pak_bse_object.bo);
7044     priv_ctx->indirect_pak_bse_object.offset = I965_CODEDBUFFER_HEADER_SIZE;
7045     priv_ctx->indirect_pak_bse_object.end_offset = ALIGN((obj_buffer->size_element - 0x1000), 0x1000);
7046     priv_ctx->indirect_pak_bse_object.bo = bo;
7047     dri_bo_reference(priv_ctx->indirect_pak_bse_object.bo);
7048
7049     dri_bo_map(bo, 1);
7050
7051     if (!bo->virtual)
7052         return VA_STATUS_ERROR_INVALID_VALUE;
7053
7054     coded_buffer_segment = (struct i965_coded_buffer_segment *)bo->virtual;
7055     coded_buffer_segment->mapped = 0;
7056     coded_buffer_segment->codec = encoder_context->codec;
7057     coded_buffer_segment->status_support = 1;
7058
7059     dri_bo_unmap(bo);
7060
7061     if (priv_ctx->res_pak_slice_batch_buffer)
7062         intel_batchbuffer_free(priv_ctx->res_pak_slice_batch_buffer);
7063
7064     priv_ctx->res_pak_slice_batch_buffer =
7065         intel_batchbuffer_new(&i965->intel, I915_EXEC_BSD,
7066                               GEN9_HEVC_ENC_PAK_SLICE_STATE_SIZE *
7067                               encode_state->num_slice_params_ext);
7068     if (!priv_ctx->res_pak_slice_batch_buffer)
7069         return VA_STATUS_ERROR_ALLOCATION_FAILED;
7070
7071     for (i = 0; i < I965_MAX_NUM_SLICE; i++) {
7072         priv_state->slice_batch_offset[i] = 0;
7073         priv_state->slice_start_lcu[i] = 0;
7074     }
7075
7076     return VA_STATUS_SUCCESS;
7077 }
7078
7079 static VAStatus
7080 gen9_hevc_pak_picture_level(VADriverContextP ctx,
7081                             struct encode_state *encode_state,
7082                             struct intel_encoder_context *encoder_context)
7083 {
7084     struct intel_batchbuffer *batch = encoder_context->base.batch;
7085     struct encoder_vme_mfc_context *pak_context = NULL;
7086     struct gen9_hevc_encoder_context *priv_ctx = NULL;
7087     struct generic_enc_codec_state *generic_state = NULL;
7088     struct gen9_hevc_encoder_state *priv_state = NULL;
7089     struct gpe_mi_batch_buffer_start_parameter second_level_batch;
7090     struct hevc_encode_status_buffer *status_buffer = NULL;
7091     VAStatus va_status = VA_STATUS_SUCCESS;
7092     int i = 0;
7093
7094     pak_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
7095     priv_ctx = (struct gen9_hevc_encoder_context *)pak_context->private_enc_ctx;
7096     generic_state = (struct generic_enc_codec_state *)pak_context->generic_enc_state;
7097     priv_state = (struct gen9_hevc_encoder_state *)pak_context->private_enc_state;
7098     status_buffer = &priv_state->status_buffer;
7099
7100     if (generic_state->brc_enabled &&
7101         generic_state->curr_pak_pass) {
7102         gen9_hevc_conditional_end(ctx,
7103                                   batch, status_buffer->bo,
7104                                   status_buffer->status_image_mask_offset,
7105                                   0);
7106
7107         gen9_hevc_load_reg_mem(ctx,
7108                                batch, status_buffer->bo,
7109                                status_buffer->status_image_ctrl_offset,
7110                                status_buffer->mmio_image_ctrl_offset);
7111
7112         gen9_hevc_store_reg_mem(ctx, batch, priv_ctx->res_brc_pak_statistic_buffer.bo,
7113                                 offsetof(GEN9_HEVC_PAK_STATES, HEVC_ENC_IMAGE_STATUS_CONTROL_FOR_LAST_PASS),
7114                                 status_buffer->mmio_image_ctrl_offset);
7115
7116         gen9_hevc_store_reg_mem(ctx, batch, status_buffer->bo,
7117                                 status_buffer->status_image_ctrl_last_pass_offset,
7118                                 status_buffer->mmio_image_ctrl_offset);
7119     }
7120
7121     gen9_hevc_pak_pipe_mode_select(ctx, encode_state, encoder_context);
7122     gen9_hevc_pak_add_surface_state(ctx, encode_state, encoder_context,
7123                                     encode_state->input_yuv_object,
7124                                     GEN9_HEVC_ENC_SURFACE_SOURCE);
7125     gen9_hevc_pak_add_surface_state(ctx, encode_state, encoder_context,
7126                                     encode_state->reconstructed_object,
7127                                     GEN9_HEVC_ENC_SURFACE_RECON);
7128     gen9_hevc_pak_add_pipe_buf_addr_state(ctx, encode_state, encoder_context);
7129     gen9_hevc_pak_add_ind_obj_base_addr_state(ctx, encode_state, encoder_context);
7130     gen9_hevc_pak_add_qm_state(ctx, encode_state, encoder_context);
7131     gen9_hevc_pak_add_fm_state(ctx, encode_state, encoder_context);
7132
7133     if (generic_state->brc_enabled) {
7134         memset(&second_level_batch, 0, sizeof(second_level_batch));
7135
7136         second_level_batch.offset = generic_state->curr_pak_pass * priv_state->pic_state_size;
7137         second_level_batch.bo = priv_ctx->res_brc_pic_states_write_buffer.bo;
7138         second_level_batch.is_second_level = 1;
7139         gen8_gpe_mi_batch_buffer_start(ctx, batch, &second_level_batch);
7140     } else
7141         gen9_hevc_add_pic_state(ctx, encode_state, encoder_context, NULL, 0, 0);
7142
7143     for (i = 0; i < encode_state->num_slice_params_ext; i++) {
7144         va_status = gen9_hevc_pak_slice_level(ctx, encode_state, encoder_context, i);
7145         if (va_status != VA_STATUS_SUCCESS)
7146             goto EXIT;
7147     }
7148
7149 EXIT:
7150     return va_status;
7151 }
7152
7153 static void
7154 gen9_hevc_pak_read_status(VADriverContextP ctx,
7155                           struct intel_encoder_context *encoder_context)
7156 {
7157     struct intel_batchbuffer *batch = encoder_context->base.batch;
7158     struct encoder_vme_mfc_context *pak_context = NULL;
7159     struct gen9_hevc_encoder_context *priv_ctx = NULL;
7160     struct generic_enc_codec_state *generic_state = NULL;
7161     struct gen9_hevc_encoder_state *priv_state = NULL;
7162     struct gpe_mi_flush_dw_parameter mi_flush_dw_param;
7163     struct hevc_encode_status_buffer *status_buffer = NULL;
7164
7165     pak_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
7166     priv_ctx = (struct gen9_hevc_encoder_context *)pak_context->private_enc_ctx;
7167     generic_state = (struct generic_enc_codec_state *)pak_context->generic_enc_state;
7168     priv_state = (struct gen9_hevc_encoder_state *)pak_context->private_enc_state;
7169
7170     status_buffer = &priv_state->status_buffer;
7171
7172     memset(&mi_flush_dw_param, 0, sizeof(mi_flush_dw_param));
7173     gen8_gpe_mi_flush_dw(ctx, batch, &mi_flush_dw_param);
7174
7175     gen9_hevc_store_reg_mem(ctx, batch, status_buffer->bo,
7176                             status_buffer->status_bs_byte_count_offset,
7177                             status_buffer->mmio_bs_frame_offset);
7178
7179     gen9_hevc_store_reg_mem(ctx, batch, status_buffer->bo,
7180                             status_buffer->status_image_mask_offset,
7181                             status_buffer->mmio_image_mask_offset);
7182
7183     gen9_hevc_store_reg_mem(ctx, batch, status_buffer->bo,
7184                             status_buffer->status_image_ctrl_offset,
7185                             status_buffer->mmio_image_ctrl_offset);
7186
7187     if (generic_state->brc_enabled) {
7188         gen9_hevc_store_reg_mem(ctx, batch, priv_ctx->res_brc_pak_statistic_buffer.bo,
7189                                 offsetof(GEN9_HEVC_PAK_STATES, HEVC_ENC_BYTECOUNT_FRAME),
7190                                 status_buffer->mmio_bs_frame_offset);
7191         gen9_hevc_store_reg_mem(ctx, batch, priv_ctx->res_brc_pak_statistic_buffer.bo,
7192                                 offsetof(GEN9_HEVC_PAK_STATES, HEVC_ENC_BYTECOUNT_FRAME_NOHEADER),
7193                                 status_buffer->mmio_bs_frame_no_header_offset);
7194         gen9_hevc_store_reg_mem(ctx, batch, priv_ctx->res_brc_pak_statistic_buffer.bo,
7195                                 offsetof(GEN9_HEVC_PAK_STATES, HEVC_ENC_IMAGE_STATUS_CONTROL),
7196                                 status_buffer->mmio_image_ctrl_offset);
7197     }
7198
7199     gen8_gpe_mi_flush_dw(ctx, batch, &mi_flush_dw_param);
7200 }
7201
7202 static VAStatus
7203 gen9_hevc_pak_pipeline(VADriverContextP ctx,
7204                        VAProfile profile,
7205                        struct encode_state *encode_state,
7206                        struct intel_encoder_context *encoder_context)
7207 {
7208     struct i965_driver_data *i965 = i965_driver_data(ctx);
7209     struct intel_batchbuffer *batch = encoder_context->base.batch;
7210     struct encoder_vme_mfc_context *pak_context = encoder_context->vme_context;
7211     struct gen9_hevc_encoder_context *priv_ctx = NULL;
7212     struct generic_enc_codec_state *generic_state = NULL;
7213     struct gen9_hevc_encoder_state *priv_state = NULL;
7214     VAStatus va_status = VA_STATUS_SUCCESS;
7215
7216     if (!pak_context || !pak_context->generic_enc_state || !batch) {
7217         va_status = VA_STATUS_ERROR_INVALID_CONTEXT;
7218         goto EXIT;
7219     }
7220
7221     va_status = gen9_hevc_pak_pipeline_prepare(ctx, encode_state, encoder_context);
7222     if (va_status != VA_STATUS_SUCCESS)
7223         goto EXIT;
7224
7225     if (i965->intel.has_bsd2)
7226         intel_batchbuffer_start_atomic_bcs_override(batch, 0x1000, BSD_RING0);
7227     else
7228         intel_batchbuffer_start_atomic_bcs(batch, 0x1000);
7229
7230     intel_batchbuffer_emit_mi_flush(batch);
7231
7232     generic_state = (struct generic_enc_codec_state *)pak_context->generic_enc_state;
7233     priv_state = (struct gen9_hevc_encoder_state *)pak_context->private_enc_state;
7234     priv_ctx = (struct gen9_hevc_encoder_context *)pak_context->private_enc_ctx;
7235
7236     for (generic_state->curr_pak_pass = 0;
7237          generic_state->curr_pak_pass < generic_state->num_pak_passes;
7238          generic_state->curr_pak_pass++) {
7239         va_status = gen9_hevc_pak_picture_level(ctx, encode_state, encoder_context);
7240         if (va_status != VA_STATUS_SUCCESS)
7241             goto EXIT;
7242
7243         gen9_hevc_pak_read_status(ctx, encoder_context);
7244     }
7245
7246     if (priv_ctx->res_pak_slice_batch_buffer) {
7247         intel_batchbuffer_free(priv_ctx->res_pak_slice_batch_buffer);
7248
7249         priv_ctx->res_pak_slice_batch_buffer = NULL;
7250     }
7251
7252     intel_batchbuffer_end_atomic(batch);
7253
7254     intel_batchbuffer_flush(batch);
7255
7256     priv_state->frame_number++;
7257
7258 EXIT:
7259     return va_status;
7260 }
7261
7262 static void
7263 gen9_hevc_pak_context_destroy(void *context)
7264 {
7265     struct encoder_vme_mfc_context *pak_context = (struct encoder_vme_mfc_context *)context;
7266     struct gen9_hevc_encoder_context *priv_ctx = NULL;
7267     int i = 0;
7268
7269     priv_ctx = (struct gen9_hevc_encoder_context *)pak_context->private_enc_ctx;
7270
7271     if (priv_ctx->res_pak_slice_batch_buffer) {
7272         intel_batchbuffer_free(priv_ctx->res_pak_slice_batch_buffer);
7273         priv_ctx->res_pak_slice_batch_buffer = NULL;
7274     }
7275
7276     dri_bo_unreference(priv_ctx->indirect_pak_bse_object.bo);
7277     priv_ctx->indirect_pak_bse_object.bo = NULL;
7278
7279     if (priv_ctx->uncompressed_picture_source.obj_surface &&
7280         priv_ctx->uncompressed_picture_source.obj_surface->bo)
7281         i965_destroy_surface_storage(priv_ctx->uncompressed_picture_source.obj_surface);
7282
7283     if (priv_ctx->reconstructed_object.obj_surface &&
7284         priv_ctx->reconstructed_object.obj_surface->bo)
7285         i965_destroy_surface_storage(priv_ctx->reconstructed_object.obj_surface);
7286
7287     for (i = 0; i < GEN9_MAX_REF_SURFACES; i++)
7288         if (priv_ctx->reference_surfaces[i].obj_surface &&
7289             priv_ctx->reference_surfaces[i].obj_surface->bo)
7290             i965_destroy_surface_storage(priv_ctx->reference_surfaces[i].obj_surface);
7291 }
7292
7293 #define STATUS_IMPLEMENTATION_START
7294
7295 static void
7296 gen9_hevc_status_buffer_init(struct hevc_encode_status_buffer *status_buffer)
7297 {
7298     uint32_t base_offset = offsetof(struct i965_coded_buffer_segment, codec_private_data);
7299
7300     status_buffer->mmio_bs_frame_offset = MMIO_HCP_ENC_BITSTREAM_BYTECOUNT_FRAME_OFFSET;
7301     status_buffer->mmio_bs_frame_no_header_offset = MMIO_HCP_ENC_BITSTREAM_BYTECOUNT_FRAME_NO_HEADER_OFFSET;
7302     status_buffer->mmio_image_mask_offset = MMIO_HCP_ENC_IMAGE_STATUS_MASK_OFFSET;
7303     status_buffer->mmio_image_ctrl_offset = MMIO_HCP_ENC_IMAGE_STATUS_CTRL_OFFSET;
7304
7305     status_buffer->status_image_mask_offset = base_offset +
7306                                               offsetof(struct hevc_encode_status, image_status_mask);
7307     status_buffer->status_image_ctrl_offset = base_offset +
7308                                               offsetof(struct hevc_encode_status, image_status_ctrl);
7309     status_buffer->status_image_ctrl_last_pass_offset = base_offset +
7310                                                         offsetof(struct hevc_encode_status, image_status_ctrl_last_pass);
7311     status_buffer->status_bs_byte_count_offset = base_offset +
7312                                                  offsetof(struct hevc_encode_status, bs_byte_count);
7313     status_buffer->status_media_state_offset = base_offset +
7314                                                offsetof(struct hevc_encode_status, media_state);
7315     status_buffer->status_pass_num_offset = base_offset +
7316                                             offsetof(struct hevc_encode_status, pass_num);
7317 }
7318
7319 static VAStatus
7320 gen9_hevc_get_coded_status(VADriverContextP ctx,
7321                            struct intel_encoder_context *encoder_context,
7322                            struct i965_coded_buffer_segment *coded_buf_seg)
7323 {
7324     struct hevc_encode_status *encode_status;
7325     struct hevc_enc_image_status_ctrl *image_status_ctrl, *image_status_ctrl_last_pass;
7326
7327     if (!encoder_context || !coded_buf_seg)
7328         return VA_STATUS_ERROR_INVALID_BUFFER;
7329
7330     encode_status = (struct hevc_encode_status *)coded_buf_seg->codec_private_data;
7331
7332     coded_buf_seg->base.size = encode_status->bs_byte_count;
7333
7334     image_status_ctrl = (struct hevc_enc_image_status_ctrl *)&encode_status->image_status_ctrl;
7335     image_status_ctrl_last_pass = (struct hevc_enc_image_status_ctrl *)&encode_status->image_status_ctrl_last_pass;
7336
7337     if (image_status_ctrl->total_pass && image_status_ctrl->cumulative_frame_delta_qp == 0)
7338         image_status_ctrl->cumulative_frame_delta_qp = image_status_ctrl_last_pass->cumulative_frame_delta_qp;
7339
7340     image_status_ctrl_last_pass->cumulative_frame_delta_qp = 0;
7341
7342     return VA_STATUS_SUCCESS;
7343 }
7344
7345 // External initial APIs
7346
7347 Bool
7348 gen9_hevc_vme_context_init(VADriverContextP ctx,
7349                            struct intel_encoder_context *encoder_context)
7350 {
7351     struct i965_driver_data *i965 = i965_driver_data(ctx);
7352     struct encoder_vme_mfc_context *vme_context = NULL;
7353     struct gen9_hevc_encoder_context *priv_ctx = NULL;
7354     struct generic_enc_codec_state *generic_state = NULL;
7355     struct gen9_hevc_encoder_state *priv_state = NULL;
7356
7357     hevc_enc_kernel_ptr = (void *)gen9_hevc_encoder_kernels;
7358     hevc_enc_kernel_size = sizeof(gen9_hevc_encoder_kernels);
7359
7360     vme_context = calloc(1, sizeof(*vme_context));
7361     priv_ctx = calloc(1, sizeof(*priv_ctx));
7362     generic_state = calloc(1, sizeof(*generic_state));
7363     priv_state = calloc(1, sizeof(*priv_state));
7364
7365     if (!vme_context || !generic_state ||
7366         !priv_ctx || !priv_state) {
7367         if (vme_context) free(vme_context);
7368         if (generic_state) free(generic_state);
7369         if (priv_ctx) free(priv_ctx);
7370         if (priv_state) free(priv_state);
7371
7372         return false;
7373     }
7374
7375     encoder_context->vme_context = (void *)vme_context;
7376     vme_context->private_enc_ctx = (void *)priv_ctx;
7377     vme_context->generic_enc_state = (void *)generic_state;
7378     vme_context->private_enc_state = (void *)priv_state;
7379
7380     priv_ctx->ctx = ctx;
7381     priv_ctx->mocs = i965->intel.mocs_state;
7382
7383     generic_state->num_pak_passes = 1;
7384     generic_state->brc_enabled = 0;
7385
7386     priv_state->tu_mode = HEVC_TU_RT_SPEED;
7387     priv_state->use_hw_scoreboard = 1;
7388     priv_state->use_hw_non_stalling_scoreborad = 1;
7389     priv_state->rolling_intra_refresh = 0;
7390     priv_state->flatness_check_supported = 0;
7391     priv_state->walking_pattern_26 = 0;
7392     priv_state->num_regions_in_slice = 4;
7393     priv_state->frames_per_100s = 30000;
7394     priv_state->user_max_frame_size = 0;
7395     priv_state->brc_method = HEVC_BRC_CQP;
7396     priv_state->lcu_brc_enabled = 0;
7397     priv_state->parallel_brc = 0;
7398     priv_state->pak_obj_size = ((IS_KBL(i965->intel.device_info) || IS_GLK(i965->intel.device_info)) ?
7399                                 GEN95_HEVC_ENC_PAK_OBJ_SIZE :
7400                                 GEN9_HEVC_ENC_PAK_OBJ_SIZE) *
7401                                4;
7402     priv_state->cu_record_size = GEN9_HEVC_ENC_PAK_CU_RECORD_SIZE * 4;
7403     priv_state->pic_state_size = GEN9_HEVC_ENC_BRC_PIC_STATE_SIZE;
7404
7405     gen9_hevc_status_buffer_init(&priv_state->status_buffer);
7406     gen9_hevc_vme_kernels_context_init(ctx, encoder_context);
7407     gen9_hevc_lambda_tables_init(priv_ctx);
7408
7409     encoder_context->vme_pipeline = gen9_hevc_vme_pipeline;
7410     encoder_context->vme_context_destroy = gen9_hevc_vme_context_destroy;
7411     return true;
7412 }
7413
7414 Bool
7415 gen9_hevc_pak_context_init(VADriverContextP ctx,
7416                            struct intel_encoder_context *encoder_context)
7417 {
7418     struct encoder_vme_mfc_context *pak_context = encoder_context->vme_context;
7419
7420     if (!pak_context)
7421         return false;
7422
7423     encoder_context->mfc_context = pak_context;
7424     encoder_context->mfc_context_destroy = gen9_hevc_pak_context_destroy;
7425     encoder_context->mfc_pipeline = gen9_hevc_pak_pipeline;
7426     encoder_context->mfc_brc_prepare = gen9_hevc_brc_prepare;
7427     encoder_context->get_status = gen9_hevc_get_coded_status;
7428     return true;
7429 }