src/gen9_hevc_encoder.c

   1 /*
   2  * Copyright © 2017 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the
   6  * "Software"), to deal in the Software without restriction, including
   7  * without limitation the rights to use, copy, modify, merge, publish,
   8  * distribute, sub license, and/or sell copies of the Software, and to
   9  * permit persons to whom the Software is furnished to do so, subject to
  10  * the following conditions:
  11  *
  12  * The above copyright notice and this permission notice (including the
  13  * next paragraph) shall be included in all copies or substantial portions
  14  * of the Software.
  15  *
  16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
  17  * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  18  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
  19  * IN NO EVENT SHALL PRECISION INSIGHT AND/OR ITS SUPPLIERS BE LIABLE FOR
  20  * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
  21  * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
  22  * SOFTWAR OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  23  *
  24  * Authors:
  25  *    Chen, Peng <chen.c.peng@intel.com>
  26  *
  27  */
  28
  29 #include <stdio.h>
  30 #include <stdlib.h>
  31 #include <stdbool.h>
  32 #include <string.h>
  33 #include <math.h>
  34 #include <assert.h>
  35 #include <va/va.h>
  36
  37 #include "intel_batchbuffer.h"
  38 #include "intel_driver.h"
  39
  40 #include "i965_defines.h"
  41 #include "i965_drv_video.h"
  42 #include "i965_encoder.h"
  43 #include "i965_encoder_common.h"
  44 #include "i965_encoder_utils.h"
  45 #include "i965_encoder_api.h"
  46 #include "gen9_hevc_enc_kernels.h"
  47 #include "gen9_hevc_enc_kernels_binary.h"
  48 #include "gen9_hevc_enc_utils.h"
  49 #include "gen9_hevc_encoder.h"
  50 #include "gen9_hevc_enc_const_def.h"
  51
  52 static void *hevc_enc_kernel_ptr = NULL;
  53 static int hevc_enc_kernel_size = 0;
  54
  55 static bool
  56 gen9_hevc_get_kernel_header_and_size(void *pvbinary,
  57                                      int binary_size,
  58                                      GEN9_ENC_OPERATION operation,
  59                                      int krnstate_idx,
  60                                      struct i965_kernel *ret_kernel)
  61 {
  62     typedef uint32_t BIN_PTR[4];
  63
  64     char *bin_start;
  65     gen9_hevc_enc_kernels_header_bxt *pkh_table = NULL;
  66     gen9_hevc_enc_kernel_header *pcurr_header = NULL, *pinvalid_entry, *pnext_header;
  67     int next_krnoffset;
  68
  69     if (!pvbinary || !ret_kernel)
  70         return false;
  71
  72     bin_start = (char *)pvbinary;
  73     pkh_table = (gen9_hevc_enc_kernels_header_bxt *)pvbinary;
  74     pinvalid_entry = (gen9_hevc_enc_kernel_header *)(pkh_table + 1);
  75     next_krnoffset = binary_size;
  76
  77     switch (operation) {
  78     case GEN9_ENC_SCALING2X:
  79         pcurr_header = &pkh_table->HEVC_ENC_I_2xDownSampling_Kernel;
  80         break;
  81     case GEN9_ENC_SCALING4X:
  82         pcurr_header = &pkh_table->HEVC_ENC_I_DS4HME;
  83         break;
  84     case GEN9_ENC_ME:
  85         if (krnstate_idx)
  86             pcurr_header = &pkh_table->HEVC_ENC_P_HME;
  87         else
  88             pcurr_header = &pkh_table->HEVC_ENC_B_HME;
  89         break;
  90     case GEN9_ENC_BRC:
  91         switch (krnstate_idx) {
  92         case GEN9_HEVC_ENC_BRC_COARSE_INTRA:
  93             pcurr_header = &pkh_table->HEVC_ENC_I_COARSE;
  94             break;
  95         case GEN9_HEVC_ENC_BRC_INIT:
  96             pcurr_header = &pkh_table->HEVC_ENC_BRC_Init;
  97             break;
  98         case GEN9_HEVC_ENC_BRC_RESET:
  99             pcurr_header = &pkh_table->HEVC_ENC_BRC_Reset;
 100             break;
 101         case GEN9_HEVC_ENC_BRC_FRAME_UPDATE:
 102             pcurr_header = &pkh_table->HEVC_ENC_BRC_Update;
 103             break;
 104         case GEN9_HEVC_ENC_BRC_LCU_UPDATE:
 105             pcurr_header = &pkh_table->HEVC_ENC_BRC_LCU_Update;
 106             break;
 107         default:
 108             break;
 109         }
 110         break;
 111     case GEN9_ENC_MBENC:
 112         switch (krnstate_idx) {
 113         case GEN9_HEVC_ENC_MBENC_2xSCALING:
 114         case GEN9_HEVC_ENC_MBENC_32x32MD:
 115         case GEN9_HEVC_ENC_MBENC_16x16SAD:
 116         case GEN9_HEVC_ENC_MBENC_16x16MD:
 117         case GEN9_HEVC_ENC_MBENC_8x8PU:
 118         case GEN9_HEVC_ENC_MBENC_8x8FMODE:
 119         case GEN9_HEVC_ENC_MBENC_32x32INTRACHECK:
 120         case GEN9_HEVC_ENC_MBENC_BENC:
 121             pcurr_header = &pkh_table->HEVC_ENC_I_2xDownSampling_Kernel;
 122             pcurr_header += krnstate_idx;
 123             break;
 124         case GEN9_HEVC_ENC_MBENC_BPAK:
 125             pcurr_header = &pkh_table->HEVC_ENC_PB_Pak;
 126             break;
 127         case GEN9_HEVC_ENC_MBENC_WIDI:
 128             pcurr_header = &pkh_table->HEVC_ENC_PB_Widi;
 129             break;
 130         case GEN9_HEVC_MBENC_PENC:
 131             pcurr_header = &pkh_table->HEVC_ENC_P_MB;
 132             break;
 133         case GEN9_HEVC_MBENC_P_WIDI:
 134             pcurr_header = &pkh_table->HEVC_ENC_P_Widi;
 135             break;
 136         case GEN9_HEVC_ENC_MBENC_DS_COMBINED:
 137             pcurr_header = &pkh_table->HEVC_ENC_DS_Combined;
 138             break;
 139         default:
 140             break;
 141         }
 142         break;
 143     default:
 144         break;
 145     }
 146
 147     if (!pcurr_header ||
 148         ((pcurr_header->kernel_start_pointer << 6) >= binary_size))
 149         return false;
 150
 151     ret_kernel->bin = (const BIN_PTR *)(bin_start + (pcurr_header->kernel_start_pointer << 6));
 152
 153     pnext_header = (pcurr_header + 1);
 154     if (pnext_header < pinvalid_entry)
 155         next_krnoffset = pnext_header->kernel_start_pointer << 6;
 156
 157     ret_kernel->size = next_krnoffset - (pcurr_header->kernel_start_pointer << 6);
 158
 159     return true;
 160 }
 161
 162 static void
 163 gen9_hevc_store_reg_mem(VADriverContextP ctx,
 164                         struct intel_batchbuffer *batch,
 165                         dri_bo *bo,
 166                         unsigned int offset,
 167                         unsigned int mmio_offset)
 168 {
 169     struct gpe_mi_store_register_mem_parameter mi_store_reg_mem_param;
 170
 171     memset((void *)&mi_store_reg_mem_param, 0,
 172            sizeof(mi_store_reg_mem_param));
 173     mi_store_reg_mem_param.bo = bo;
 174     mi_store_reg_mem_param.offset = offset;
 175     mi_store_reg_mem_param.mmio_offset = mmio_offset;
 176     gen8_gpe_mi_store_register_mem(ctx, batch, &mi_store_reg_mem_param);
 177 }
 178
 179 static void
 180 gen9_hevc_load_reg_mem(VADriverContextP ctx,
 181                        struct intel_batchbuffer *batch,
 182                        dri_bo *bo,
 183                        unsigned int offset,
 184                        unsigned int mmio_offset)
 185 {
 186     struct gpe_mi_load_register_mem_parameter mi_load_reg_mem_param;
 187
 188     memset((void *)&mi_load_reg_mem_param, 0,
 189            sizeof(mi_load_reg_mem_param));
 190     mi_load_reg_mem_param.bo = bo;
 191     mi_load_reg_mem_param.offset = offset;
 192     mi_load_reg_mem_param.mmio_offset = mmio_offset;
 193     gen8_gpe_mi_load_register_mem(ctx, batch, &mi_load_reg_mem_param);
 194 }
 195
 196 static void
 197 gen9_hevc_conditional_end(VADriverContextP ctx,
 198                           struct intel_batchbuffer *batch,
 199                           dri_bo *bo,
 200                           unsigned int offset,
 201                           unsigned int data)
 202 {
 203     struct gpe_mi_conditional_batch_buffer_end_parameter mi_cond_end;
 204
 205     memset(&mi_cond_end, 0, sizeof(mi_cond_end));
 206     mi_cond_end.offset = offset;
 207     mi_cond_end.bo = bo;
 208     mi_cond_end.compare_data = data;
 209     mi_cond_end.compare_mask_mode_disabled = 0;
 210     gen9_gpe_mi_conditional_batch_buffer_end(ctx, batch,
 211                                              &mi_cond_end);
 212 }
 213
 214 static VAStatus
 215 gen9_hevc_ensure_surface(VADriverContextP ctx,
 216                          struct gen9_hevc_encoder_state *priv_state,
 217                          struct object_surface *obj_surface,
 218                          int reallocate_flag)
 219 {
 220     VAStatus va_status = VA_STATUS_SUCCESS;
 221     int update = 0;
 222     unsigned int fourcc = VA_FOURCC_NV12;
 223
 224     if (!obj_surface) {
 225         va_status = VA_STATUS_ERROR_INVALID_PARAMETER;
 226
 227         goto EXIT;
 228     }
 229
 230     if ((priv_state->bit_depth_luma_minus8 > 0)
 231         || (priv_state->bit_depth_chroma_minus8 > 0)) {
 232         if (obj_surface->fourcc != VA_FOURCC_P010) {
 233             update = 1;
 234             fourcc = VA_FOURCC_P010;
 235         }
 236     } else if (obj_surface->fourcc != VA_FOURCC_NV12) {
 237         update = 1;
 238         fourcc = VA_FOURCC_NV12;
 239     }
 240
 241     /* (Re-)allocate the underlying surface buffer store, if necessary */
 242     if (!obj_surface->bo || update) {
 243         if (reallocate_flag) {
 244             struct i965_driver_data * const i965 = i965_driver_data(ctx);
 245
 246             i965_destroy_surface_storage(obj_surface);
 247
 248             va_status = i965_check_alloc_surface_bo(ctx,
 249                                                     obj_surface,
 250                                                     i965->codec_info->has_tiled_surface,
 251                                                     fourcc,
 252                                                     SUBSAMPLE_YUV420);
 253         } else
 254             va_status = VA_STATUS_ERROR_INVALID_PARAMETER;
 255     }
 256
 257 EXIT:
 258     return va_status;
 259 }
 260
 261 static void
 262 gen9_hevc_free_surface_private(void **data)
 263 {
 264     struct gen9_hevc_surface_priv *surface_priv = NULL;
 265     int i = 0;
 266
 267     surface_priv = (struct gen9_hevc_surface_priv *)(*data);
 268     if (!surface_priv)
 269         return;
 270
 271     for (i = 0; i < HEVC_SCALED_SURFS_NUM; i++) {
 272         if (surface_priv->scaled_surface_obj[i]) {
 273             i965_DestroySurfaces(surface_priv->ctx, &surface_priv->scaled_surface_id[i], 1);
 274             surface_priv->scaled_surface_id[i] = VA_INVALID_SURFACE;
 275             surface_priv->scaled_surface_obj[i] = NULL;
 276         }
 277     }
 278
 279     if (surface_priv->surface_obj_nv12) {
 280         i965_DestroySurfaces(surface_priv->ctx, &surface_priv->surface_id_nv12, 1);
 281         surface_priv->surface_id_nv12 = VA_INVALID_SURFACE;
 282         surface_priv->surface_obj_nv12 = NULL;
 283     }
 284
 285     if (surface_priv->motion_vector_temporal_bo)
 286         dri_bo_unreference(surface_priv->motion_vector_temporal_bo);
 287
 288     free(surface_priv);
 289 }
 290
 291 static VAStatus
 292 gen9_hevc_init_surface_private(VADriverContextP ctx,
 293                                struct generic_enc_codec_state *generic_state,
 294                                struct gen9_hevc_encoder_state *priv_state,
 295                                struct object_surface *obj_surface)
 296 {
 297     struct i965_driver_data *i965 = i965_driver_data(ctx);
 298     struct gen9_hevc_surface_priv *surface_priv = NULL;
 299     int size = 0;
 300
 301     if (!obj_surface || !obj_surface->bo)
 302         return VA_STATUS_ERROR_INVALID_SURFACE;
 303
 304     if (obj_surface->private_data &&
 305         obj_surface->free_private_data != gen9_hevc_free_surface_private) {
 306         obj_surface->free_private_data(&obj_surface->private_data);
 307         obj_surface->private_data = NULL;
 308     }
 309
 310     if (obj_surface->private_data) {
 311         surface_priv = (struct gen9_hevc_surface_priv *)obj_surface->private_data;
 312
 313         surface_priv->surface_nv12_valid = 0;
 314     } else {
 315         surface_priv = calloc(1, sizeof(*surface_priv));
 316         if (!surface_priv)
 317             return VA_STATUS_ERROR_ALLOCATION_FAILED;
 318
 319         surface_priv->ctx = ctx;
 320         surface_priv->surface_reff = obj_surface;
 321
 322         obj_surface->private_data = (void *)surface_priv;
 323         obj_surface->free_private_data = gen9_hevc_free_surface_private;
 324
 325         // alloc motion vector temporal buffer
 326         size = MAX(((priv_state->picture_width + 63) >> 6) *
 327                    ((priv_state->picture_height + 15) >> 4),
 328                    ((priv_state->picture_width + 31) >> 5) *
 329                    ((priv_state->picture_height + 31) >> 5));
 330         size = ALIGN(size, 2) * 64;
 331         surface_priv->motion_vector_temporal_bo =
 332             dri_bo_alloc(i965->intel.bufmgr,
 333                          "motion vector temporal buffer",
 334                          size,
 335                          0x1000);
 336         if (!surface_priv->motion_vector_temporal_bo)
 337             return VA_STATUS_ERROR_ALLOCATION_FAILED;
 338
 339         //alloc HME surfaces
 340         i965_CreateSurfaces(ctx,
 341                             priv_state->frame_width_4x,
 342                             priv_state->frame_height_4x,
 343                             VA_RT_FORMAT_YUV420,
 344                             1,
 345                             &surface_priv->scaled_surface_id[HEVC_SCALED_SURF_4X_ID]);
 346
 347         surface_priv->scaled_surface_obj[HEVC_SCALED_SURF_4X_ID] =
 348             SURFACE(surface_priv->scaled_surface_id[HEVC_SCALED_SURF_4X_ID]);
 349
 350         if (!surface_priv->scaled_surface_obj[HEVC_SCALED_SURF_4X_ID])
 351             return VA_STATUS_ERROR_ALLOCATION_FAILED;
 352
 353         i965_check_alloc_surface_bo(ctx, surface_priv->scaled_surface_obj[HEVC_SCALED_SURF_4X_ID], 1,
 354                                     VA_FOURCC('N', 'V', '1', '2'), SUBSAMPLE_YUV420);
 355
 356         if (generic_state->b16xme_supported) {
 357             i965_CreateSurfaces(ctx,
 358                                 priv_state->frame_width_16x,
 359                                 priv_state->frame_height_16x,
 360                                 VA_RT_FORMAT_YUV420,
 361                                 1,
 362                                 &surface_priv->scaled_surface_id[HEVC_SCALED_SURF_16X_ID]);
 363             surface_priv->scaled_surface_obj[HEVC_SCALED_SURF_16X_ID] =
 364                 SURFACE(surface_priv->scaled_surface_id[HEVC_SCALED_SURF_16X_ID]);
 365
 366             if (!surface_priv->scaled_surface_obj[HEVC_SCALED_SURF_16X_ID])
 367                 return VA_STATUS_ERROR_ALLOCATION_FAILED;
 368
 369             i965_check_alloc_surface_bo(ctx, surface_priv->scaled_surface_obj[HEVC_SCALED_SURF_16X_ID], 1,
 370                                         VA_FOURCC('N', 'V', '1', '2'), SUBSAMPLE_YUV420);
 371         }
 372
 373         if (generic_state->b32xme_supported) {
 374             i965_CreateSurfaces(ctx,
 375                                 priv_state->frame_width_32x,
 376                                 priv_state->frame_height_32x,
 377                                 VA_RT_FORMAT_YUV420,
 378                                 1,
 379                                 &surface_priv->scaled_surface_id[HEVC_SCALED_SURF_32X_ID]);
 380             surface_priv->scaled_surface_obj[HEVC_SCALED_SURF_32X_ID] =
 381                 SURFACE(surface_priv->scaled_surface_id[HEVC_SCALED_SURF_32X_ID]);
 382
 383             if (!surface_priv->scaled_surface_obj[HEVC_SCALED_SURF_32X_ID])
 384                 return VA_STATUS_ERROR_ALLOCATION_FAILED;
 385
 386             i965_check_alloc_surface_bo(ctx, surface_priv->scaled_surface_obj[HEVC_SCALED_SURF_32X_ID], 1,
 387                                         VA_FOURCC('N', 'V', '1', '2'), SUBSAMPLE_YUV420);
 388         }
 389
 390         if (obj_surface->fourcc == VA_FOURCC_P010) {
 391             i965_CreateSurfaces(ctx,
 392                                 priv_state->frame_width_in_max_lcu,
 393                                 priv_state->frame_height_in_max_lcu,
 394                                 VA_RT_FORMAT_YUV420,
 395                                 1,
 396                                 &surface_priv->surface_id_nv12);
 397             surface_priv->surface_obj_nv12 = SURFACE(surface_priv->surface_id_nv12);
 398
 399             if (!surface_priv->surface_obj_nv12)
 400                 return VA_STATUS_ERROR_ALLOCATION_FAILED;
 401
 402             i965_check_alloc_surface_bo(ctx, surface_priv->surface_obj_nv12, 1,
 403                                         VA_FOURCC('N', 'V', '1', '2'), SUBSAMPLE_YUV420);
 404         }
 405     }
 406
 407     return VA_STATUS_SUCCESS;
 408 }
 409
 410 static void
 411 gen9_hevc_enc_free_resources(struct encoder_vme_mfc_context *vme_context)
 412 {
 413     struct gen9_hevc_encoder_context *priv_ctx = NULL;
 414
 415     priv_ctx = (struct gen9_hevc_encoder_context *)vme_context->private_enc_ctx;
 416
 417     i965_free_gpe_resource(&priv_ctx->res_brc_pic_states_write_buffer);
 418     i965_free_gpe_resource(&priv_ctx->res_brc_history_buffer);
 419     i965_free_gpe_resource(&priv_ctx->res_brc_intra_dist_buffer);
 420     i965_free_gpe_resource(&priv_ctx->res_brc_pak_statistic_buffer);
 421     i965_free_gpe_resource(&priv_ctx->res_brc_input_buffer_for_enc_kernels);
 422     i965_free_gpe_resource(&priv_ctx->res_brc_constant_data_buffer);
 423
 424     i965_free_gpe_resource(&priv_ctx->res_mb_code_surface);
 425
 426     // free VME buffers
 427     i965_free_gpe_resource(&priv_ctx->res_flatness_check_surface);
 428     i965_free_gpe_resource(&priv_ctx->res_brc_me_dist_buffer);
 429     i965_free_gpe_resource(&priv_ctx->s4x_memv_distortion_buffer);
 430     i965_free_gpe_resource(&priv_ctx->s4x_memv_data_buffer);
 431     i965_free_gpe_resource(&priv_ctx->s16x_memv_data_buffer);
 432     i965_free_gpe_resource(&priv_ctx->s32x_memv_data_buffer);
 433     i965_free_gpe_resource(&priv_ctx->res_32x32_pu_output_buffer);
 434     i965_free_gpe_resource(&priv_ctx->res_simplest_intra_buffer);
 435     i965_free_gpe_resource(&priv_ctx->res_kernel_debug);
 436     i965_free_gpe_resource(&priv_ctx->res_sad_16x16_pu_buffer);
 437     i965_free_gpe_resource(&priv_ctx->res_vme_8x8_mode_buffer);
 438     i965_free_gpe_resource(&priv_ctx->res_intra_mode_buffer);
 439     i965_free_gpe_resource(&priv_ctx->res_intra_distortion_buffer);
 440     i965_free_gpe_resource(&priv_ctx->res_vme_uni_sic_buffer);
 441     i965_free_gpe_resource(&priv_ctx->res_con_corrent_thread_buffer);
 442     i965_free_gpe_resource(&priv_ctx->res_mv_index_buffer);
 443     i965_free_gpe_resource(&priv_ctx->res_mvp_index_buffer);
 444     i965_free_gpe_resource(&priv_ctx->res_roi_buffer);
 445     i965_free_gpe_resource(&priv_ctx->res_mb_statistics_buffer);
 446
 447     if (priv_ctx->scaled_2x_surface_obj) {
 448         i965_DestroySurfaces(priv_ctx->ctx, &priv_ctx->scaled_2x_surface_id, 1);
 449         priv_ctx->scaled_2x_surface_obj = NULL;
 450     }
 451
 452     // free PAK buffers
 453     i965_free_gpe_resource(&priv_ctx->deblocking_filter_line_buffer);
 454     i965_free_gpe_resource(&priv_ctx->deblocking_filter_tile_line_buffer);
 455     i965_free_gpe_resource(&priv_ctx->deblocking_filter_tile_column_buffer);
 456     i965_free_gpe_resource(&priv_ctx->metadata_line_buffer);
 457     i965_free_gpe_resource(&priv_ctx->metadata_tile_line_buffer);
 458     i965_free_gpe_resource(&priv_ctx->metadata_tile_column_buffer);
 459     i965_free_gpe_resource(&priv_ctx->sao_line_buffer);
 460     i965_free_gpe_resource(&priv_ctx->sao_tile_line_buffer);
 461     i965_free_gpe_resource(&priv_ctx->sao_tile_column_buffer);
 462
 463     priv_ctx->res_inited = 0;
 464 }
 465
 466 #define ALLOC_GPE_RESOURCE(RES, NAME, SIZE)                 \
 467     do{                                                     \
 468         i965_free_gpe_resource(&priv_ctx->RES);             \
 469         if (!i965_allocate_gpe_resource(i965->intel.bufmgr, \
 470                                  &priv_ctx->RES,            \
 471                                  SIZE,                      \
 472                                  NAME))                     \
 473             goto FAIL;                                      \
 474     } while(0);
 475
 476 #define ALLOC_GPE_2D_RESOURCE(RES, NAME, W, H, P)               \
 477     do{                                                         \
 478         i965_free_gpe_resource(&priv_ctx->RES);                 \
 479         if (!i965_gpe_allocate_2d_resource(i965->intel.bufmgr,  \
 480                                  &priv_ctx->RES,                \
 481                                  (ALIGN(W, 64)), H,             \
 482                                  (ALIGN(P, 64)),                \
 483                                  NAME))                         \
 484             goto FAIL;                                          \
 485     } while(0);
 486
 487 static VAStatus
 488 gen9_hevc_enc_alloc_resources(VADriverContextP ctx,
 489                               struct encode_state *encode_state,
 490                               struct intel_encoder_context *encoder_context)
 491 {
 492     struct i965_driver_data *i965 = i965_driver_data(ctx);
 493     struct encoder_vme_mfc_context *vme_context = NULL;
 494     struct gen9_hevc_encoder_context *priv_ctx = NULL;
 495     struct generic_enc_codec_state *generic_state = NULL;
 496     struct gen9_hevc_encoder_state *priv_state = NULL;
 497     int res_size = 0, size_shift = 0;
 498     int width = 0, height = 0;
 499
 500     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
 501     priv_ctx = (struct gen9_hevc_encoder_context *)vme_context->private_enc_ctx;
 502     generic_state = (struct generic_enc_codec_state *)vme_context->generic_enc_state;
 503     priv_state = (struct gen9_hevc_encoder_state *)vme_context->private_enc_state;
 504
 505     if (priv_ctx->res_inited)
 506         return VA_STATUS_SUCCESS;
 507
 508     res_size = priv_state->mb_code_size;
 509     ALLOC_GPE_RESOURCE(res_mb_code_surface,
 510                        "Mb code surface", res_size);
 511
 512     res_size = priv_state->pic_state_size * generic_state->num_pak_passes;
 513     ALLOC_GPE_RESOURCE(res_brc_pic_states_write_buffer,
 514                        "Brc pic status write buffer",
 515                        res_size);
 516
 517     res_size = priv_state->pic_state_size * generic_state->num_pak_passes;
 518     ALLOC_GPE_RESOURCE(res_brc_pic_states_read_buffer,
 519                        "Brc pic status read buffer",
 520                        res_size);
 521
 522     res_size = GEN9_HEVC_ENC_BRC_HISTORY_BUFFER_SIZE;
 523     ALLOC_GPE_RESOURCE(res_brc_history_buffer,
 524                        "Brc history buffer",
 525                        res_size);
 526
 527     res_size = GEN9_HEVC_ENC_BRC_PAK_STATISTCS_SIZE;
 528     ALLOC_GPE_RESOURCE(res_brc_pak_statistic_buffer,
 529                        "Brc pak statistic buffer",
 530                        res_size);
 531
 532     res_size = 1024;
 533     ALLOC_GPE_RESOURCE(res_brc_input_buffer_for_enc_kernels,
 534                        "Brc input buffer for enc kernels buffer",
 535                        res_size);
 536
 537     width = ALIGN(priv_state->downscaled_width_4x_in_mb * 8, 64);
 538     height = ALIGN(priv_state->downscaled_height_4x_in_mb * 4, 8) * 2;
 539     ALLOC_GPE_2D_RESOURCE(res_brc_intra_dist_buffer,
 540                           "Brc intra distortion buffer",
 541                           width, height, width);
 542
 543     width = ALIGN(GEN9_HEVC_ENC_BRC_CONSTANT_SURFACE_WIDTH, 64);
 544     height = GEN9_HEVC_ENC_BRC_CONSTANT_SURFACE_HEIGHT;
 545     ALLOC_GPE_2D_RESOURCE(res_brc_constant_data_buffer,
 546                           "Brc constant data buffer",
 547                           width, height, width);
 548
 549     width = ALIGN((priv_state->frame_width_4x * 4 + 31) >> 4, 64);
 550     height = ALIGN((priv_state->frame_height_4x * 4 + 31) >> 5, 4);
 551     ALLOC_GPE_2D_RESOURCE(res_brc_mb_qp_buffer,
 552                           "Brc mb qp buffer",
 553                           width, height, width);
 554
 555     //HME scaling buffer allocation
 556     width = ALIGN(priv_state->downscaled_width_4x_in_mb * 8, 64);
 557     height = ALIGN(priv_state->downscaled_height_4x_in_mb * 4, 8);
 558     ALLOC_GPE_2D_RESOURCE(res_brc_me_dist_buffer,
 559                           "Brc me distortion buffer",
 560                           width, height, width);
 561
 562     if (generic_state->hme_supported) {
 563         width = ALIGN(priv_state->downscaled_width_4x_in_mb * 8, 64);
 564         height = ALIGN(priv_state->downscaled_height_4x_in_mb * 4 * 10, 8) * 2;
 565         ALLOC_GPE_2D_RESOURCE(s4x_memv_distortion_buffer,
 566                               "4x MEMV distortion buffer",
 567                               width, height, width);
 568
 569         width = ALIGN(priv_state->downscaled_width_4x_in_mb * 32, 64);
 570         height = priv_state->downscaled_height_4x_in_mb * 4 * 10;
 571         ALLOC_GPE_2D_RESOURCE(s4x_memv_data_buffer,
 572                               "4x MEMV data buffer",
 573                               width, height, width);
 574
 575         if (generic_state->b16xme_supported) {
 576             width = ALIGN(priv_state->downscaled_width_16x_in_mb * 32, 64);
 577             height = priv_state->downscaled_height_16x_in_mb * 2 * 4 * 10;
 578             ALLOC_GPE_2D_RESOURCE(s16x_memv_data_buffer,
 579                                   "16x MEMV data buffer",
 580                                   width, height, width);
 581
 582             if (generic_state->b32xme_supported) {
 583                 width = ALIGN(priv_state->downscaled_width_32x_in_mb * 32, 64);
 584                 height = priv_state->downscaled_height_32x_in_mb * 2 * 4 * 10;
 585                 ALLOC_GPE_2D_RESOURCE(s32x_memv_data_buffer,
 586                                       "32x MEMV data buffer",
 587                                       width, height, width);
 588             }
 589         }
 590     }
 591
 592     if (priv_state->flatness_check_supported) {
 593         width = ALIGN(priv_state->width_in_mb * 4, 64);
 594         height = priv_state->downscaled_height_4x_in_mb * 4;
 595         ALLOC_GPE_2D_RESOURCE(res_flatness_check_surface,
 596                               "Flatness check buffer",
 597                               width, height, width);
 598     }
 599
 600     if (priv_ctx->scaled_2x_surface_obj)
 601         i965_DestroySurfaces(priv_ctx->ctx, &priv_ctx->scaled_2x_surface_id, 1);
 602
 603     width = priv_state->frame_width_in_max_lcu >> 1;
 604     height = priv_state->frame_height_in_max_lcu >> 1;
 605     if (priv_state->bit_depth_luma_minus8) {
 606         width = ALIGN(width, 32);
 607         height = ALIGN(height, 32);
 608     }
 609
 610     i965_CreateSurfaces(ctx,
 611                         width,
 612                         height,
 613                         VA_RT_FORMAT_YUV420,
 614                         1,
 615                         &priv_ctx->scaled_2x_surface_id);
 616     priv_ctx->scaled_2x_surface_obj =
 617         SURFACE(priv_ctx->scaled_2x_surface_id);
 618
 619     if (!priv_ctx->scaled_2x_surface_obj)
 620         goto FAIL;
 621
 622     i965_check_alloc_surface_bo(ctx, priv_ctx->scaled_2x_surface_obj, 1,
 623                                 VA_FOURCC('N', 'V', '1', '2'), SUBSAMPLE_YUV420);
 624
 625     res_size = (priv_state->frame_width_in_max_lcu >> 5) *
 626                (priv_state->frame_height_in_max_lcu >> 5) *
 627                32;
 628     ALLOC_GPE_RESOURCE(res_32x32_pu_output_buffer,
 629                        "32x32 pu output buffer",
 630                        res_size);
 631
 632     width = priv_state->frame_width_in_max_lcu >> 3;
 633     height = priv_state->frame_height_in_max_lcu >> 5;
 634     ALLOC_GPE_2D_RESOURCE(res_slice_map_buffer,
 635                           "Slice map buffer",
 636                           width, height, width);
 637
 638     res_size = 8192 * 1024;
 639     ALLOC_GPE_RESOURCE(res_kernel_debug,
 640                        "kernel debug",
 641                        res_size);
 642
 643     width = priv_state->frame_width_in_max_lcu >> 3;
 644     height = priv_state->frame_height_in_max_lcu >> 5;
 645     ALLOC_GPE_2D_RESOURCE(res_simplest_intra_buffer,
 646                           "Simplest intra buffer",
 647                           width, height, width);
 648
 649     res_size = (priv_state->frame_width_in_max_lcu >> 4) *
 650                (priv_state->frame_height_in_max_lcu >> 4) * 8 * 4;
 651     ALLOC_GPE_RESOURCE(res_sad_16x16_pu_buffer,
 652                        "Sad 16x16 pu",
 653                        res_size);
 654
 655     res_size = (priv_state->frame_width_in_max_lcu >> 4) *
 656                (priv_state->frame_height_in_max_lcu >> 4) * 32;
 657     ALLOC_GPE_RESOURCE(res_vme_8x8_mode_buffer,
 658                        "Vme 8x8 mode",
 659                        res_size);
 660
 661     res_size = (priv_state->frame_width_in_max_lcu >> 3) *
 662                (priv_state->frame_height_in_max_lcu >> 3) * 32;
 663     ALLOC_GPE_RESOURCE(res_intra_mode_buffer,
 664                        "Intra mode",
 665                        res_size);
 666
 667     res_size = (priv_state->frame_width_in_max_lcu >> 4) *
 668                (priv_state->frame_height_in_max_lcu >> 4) * 16;
 669     ALLOC_GPE_RESOURCE(res_intra_distortion_buffer,
 670                        "Intra distortion",
 671                        res_size);
 672
 673     width = priv_state->frame_width_in_max_lcu >> 1;
 674     height = priv_state->frame_height_in_max_lcu >> 4;
 675     ALLOC_GPE_2D_RESOURCE(res_min_distortion_buffer,
 676                           "Min distortion buffer",
 677                           width, height, width);
 678
 679     res_size = priv_state->frame_width_in_max_lcu *
 680                priv_state->frame_height_in_max_lcu;
 681     ALLOC_GPE_RESOURCE(res_vme_uni_sic_buffer,
 682                        "Vme uni sic buffer",
 683                        res_size);
 684
 685     width = sizeof(gen9_hevc_mbenc_control_region);
 686     height = GEN9_HEVC_ENC_CONCURRENT_SURFACE_HEIGHT;
 687     ALLOC_GPE_2D_RESOURCE(res_con_corrent_thread_buffer,
 688                           "Con corrent thread buffer",
 689                           width, height, width);
 690
 691     res_size = priv_state->frame_width_in_max_lcu *
 692                priv_state->frame_height_in_max_lcu / 4;
 693     ALLOC_GPE_RESOURCE(res_mv_index_buffer,
 694                        "Mv index buffer",
 695                        res_size);
 696
 697     res_size = priv_state->frame_width_in_max_lcu *
 698                priv_state->frame_height_in_max_lcu / 2;
 699     ALLOC_GPE_RESOURCE(res_mvp_index_buffer,
 700                        "Mvp index buffer",
 701                        res_size);
 702
 703     width = ALIGN(priv_state->width_in_mb * 4, 64);
 704     height = ALIGN(priv_state->height_in_mb, 8);
 705     ALLOC_GPE_2D_RESOURCE(res_roi_buffer,
 706                           "ROI buffer",
 707                           width, height, width);
 708
 709     res_size = priv_state->width_in_mb *
 710                priv_state->height_in_mb * 52;
 711     ALLOC_GPE_RESOURCE(res_mb_statistics_buffer,
 712                        "MB statistics buffer",
 713                        res_size);
 714
 715     // PAK pipe buffer allocation
 716     size_shift = (priv_state->bit_depth_luma_minus8 ||
 717                   priv_state->bit_depth_chroma_minus8) ? 2 : 3;
 718
 719     res_size = ALIGN(priv_state->picture_width, 32) >> size_shift;
 720     ALLOC_GPE_RESOURCE(deblocking_filter_line_buffer,
 721                        "Deblocking filter line buffer",
 722                        res_size << 6);
 723     ALLOC_GPE_RESOURCE(deblocking_filter_tile_line_buffer,
 724                        "Deblocking filter tile line buffer",
 725                        res_size << 6);
 726
 727     res_size = ALIGN(priv_state->picture_height +
 728                      priv_state->height_in_lcu * 6, 32) >> size_shift;
 729     ALLOC_GPE_RESOURCE(deblocking_filter_tile_column_buffer,
 730                        "Deblocking filter tile column buffer",
 731                        res_size << 6);
 732
 733     res_size = (((priv_state->picture_width + 15) >> 4) * 188 +
 734                 priv_state->width_in_lcu * 9 + 1023) >> 9;
 735     ALLOC_GPE_RESOURCE(metadata_line_buffer,
 736                        "metadata line buffer",
 737                        res_size << 6);
 738
 739     res_size = (((priv_state->picture_width + 15) >> 4) * 172 +
 740                 priv_state->width_in_lcu * 9 + 1023) >> 9;
 741     ALLOC_GPE_RESOURCE(metadata_tile_line_buffer,
 742                        "metadata tile line buffer",
 743                        res_size << 6);
 744
 745     res_size = (((priv_state->picture_height + 15) >> 4) * 176 +
 746                 priv_state->height_in_lcu * 89 + 1023) >> 9;
 747     ALLOC_GPE_RESOURCE(metadata_tile_column_buffer,
 748                        "metadata tile column buffer",
 749                        res_size << 6);
 750
 751     res_size = ALIGN(((priv_state->picture_width >> 1) +
 752                       priv_state->width_in_lcu * 3), 16) >> size_shift;
 753     ALLOC_GPE_RESOURCE(sao_line_buffer,
 754                        "sao line buffer",
 755                        res_size << 6);
 756
 757     res_size = ALIGN(((priv_state->picture_width >> 1) +
 758                       priv_state->width_in_lcu * 6), 16) >> size_shift;
 759     ALLOC_GPE_RESOURCE(sao_tile_line_buffer,
 760                        "sao tile line buffer",
 761                        res_size << 6);
 762
 763     res_size = ALIGN(((priv_state->picture_height >> 1) +
 764                       priv_state->height_in_lcu * 6), 16) >> size_shift;
 765     ALLOC_GPE_RESOURCE(sao_tile_column_buffer,
 766                        "sao tile column buffer",
 767                        res_size << 6);
 768
 769     priv_ctx->res_inited = 1;
 770     return VA_STATUS_SUCCESS;
 771
 772 FAIL:
 773     gen9_hevc_enc_free_resources(vme_context);
 774     return VA_STATUS_ERROR_ALLOCATION_FAILED;
 775 }
 776
 777 #define VME_IMPLEMENTATION_START
 778
 779 static void
 780 gen9_hevc_set_gpe_1d_surface(VADriverContextP ctx,
 781                              struct gen9_hevc_encoder_context *priv_ctx,
 782                              struct i965_gpe_context *gpe_context,
 783                              enum GEN9_HEVC_ENC_SURFACE_TYPE surface_type,
 784                              int bti_idx,
 785                              int is_raw_buffer,
 786                              int size,
 787                              unsigned int offset,
 788                              struct i965_gpe_resource *gpe_buffer,
 789                              dri_bo *bo)
 790 {
 791     if (!gpe_buffer && !bo) {
 792         gpe_buffer = priv_ctx->gpe_surfaces[surface_type].gpe_resource;
 793         bo = priv_ctx->gpe_surfaces[surface_type].bo;
 794     }
 795
 796     if (gpe_buffer)
 797         gen9_add_buffer_gpe_surface(ctx, gpe_context,
 798                                     gpe_buffer, is_raw_buffer,
 799                                     size == 0 ? gpe_buffer->size - offset : size,
 800                                     offset, bti_idx);
 801     else if (bo)
 802         gen9_add_dri_buffer_gpe_surface(ctx, gpe_context,
 803                                         bo, is_raw_buffer,
 804                                         size == 0 ? bo->size - offset : size,
 805                                         offset, bti_idx);
 806 }
 807
 808 static void
 809 gen9_hevc_set_gpe_2d_surface(VADriverContextP ctx,
 810                              struct gen9_hevc_encoder_context *priv_ctx,
 811                              struct i965_gpe_context *gpe_context,
 812                              enum GEN9_HEVC_ENC_SURFACE_TYPE surface_type,
 813                              int bti_idx,
 814                              int has_uv_surface,
 815                              int is_media_block_rw,
 816                              unsigned int format,
 817                              struct i965_gpe_resource *gpe_buffer,
 818                              struct object_surface *surface_object)
 819 {
 820     if (!gpe_buffer && !surface_object) {
 821         gpe_buffer = priv_ctx->gpe_surfaces[surface_type].gpe_resource;
 822         surface_object = priv_ctx->gpe_surfaces[surface_type].surface_object;
 823     }
 824
 825     if (gpe_buffer) {
 826         gen9_add_buffer_2d_gpe_surface(ctx,
 827                                        gpe_context,
 828                                        gpe_buffer,
 829                                        is_media_block_rw,
 830                                        format,
 831                                        bti_idx);
 832     } else if (surface_object) {
 833         gen9_add_2d_gpe_surface(ctx, gpe_context,
 834                                 surface_object,
 835                                 0, is_media_block_rw, format,
 836                                 bti_idx);
 837
 838         if (has_uv_surface)
 839             gen9_add_2d_gpe_surface(ctx, gpe_context,
 840                                     surface_object,
 841                                     1, is_media_block_rw, format,
 842                                     bti_idx + 1);
 843     }
 844 }
 845
 846 static void
 847 gen9_hevc_set_gpe_adv_surface(VADriverContextP ctx,
 848                               struct gen9_hevc_encoder_context *priv_ctx,
 849                               struct i965_gpe_context *gpe_context,
 850                               enum GEN9_HEVC_ENC_SURFACE_TYPE surface_type,
 851                               int bti_idx,
 852                               struct object_surface *surface_object)
 853 {
 854     if (!surface_object)
 855         surface_object = priv_ctx->gpe_surfaces[surface_type].surface_object;
 856
 857     if (surface_object)
 858         gen9_add_adv_gpe_surface(ctx, gpe_context,
 859                                  surface_object, bti_idx);
 860 }
 861
 862 static void
 863 gen9_hevc_add_gpe_surface(struct gen9_hevc_encoder_context *priv_ctx,
 864                           enum GEN9_HEVC_ENC_SURFACE_TYPE surface_type,
 865                           struct i965_gpe_resource *gpe_buffer,
 866                           struct object_surface *surface_object)
 867 {
 868     if (gpe_buffer && gpe_buffer->bo)
 869         priv_ctx->gpe_surfaces[surface_type].gpe_resource = gpe_buffer;
 870     else if (surface_object)
 871         priv_ctx->gpe_surfaces[surface_type].surface_object = surface_object;
 872 }
 873
 874 static void
 875 gen9_hevc_init_gpe_surfaces_table(VADriverContextP ctx,
 876                                   struct encode_state *encode_state,
 877                                   struct intel_encoder_context *encoder_context)
 878 {
 879     struct encoder_vme_mfc_context *vme_context = NULL;
 880     struct gen9_hevc_encoder_context *priv_ctx = NULL;
 881
 882     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
 883     priv_ctx = (struct gen9_hevc_encoder_context *)vme_context->private_enc_ctx;
 884
 885     gen9_hevc_add_gpe_surface(priv_ctx, HEVC_ENC_SURFACE_RAW_Y, NULL,
 886                               encode_state->input_yuv_object);
 887     gen9_hevc_add_gpe_surface(priv_ctx, HEVC_ENC_SURFACE_RAW_Y_UV, NULL,
 888                               encode_state->input_yuv_object);
 889     gen9_hevc_add_gpe_surface(priv_ctx, HEVC_ENC_SURFACE_RAW_VME, NULL,
 890                               encode_state->input_yuv_object);
 891
 892     if (priv_ctx->scaled_2x_surface_obj) {
 893         gen9_hevc_add_gpe_surface(priv_ctx, HEVC_ENC_SURFACE_Y_2X, NULL,
 894                                   priv_ctx->scaled_2x_surface_obj);
 895         gen9_hevc_add_gpe_surface(priv_ctx, HEVC_ENC_SURFACE_Y_2X_VME, NULL,
 896                                   priv_ctx->scaled_2x_surface_obj);
 897     }
 898
 899     gen9_hevc_add_gpe_surface(priv_ctx, HEVC_ENC_SURFACE_BRC_HISTORY,
 900                               &priv_ctx->res_brc_history_buffer,
 901                               NULL);
 902     gen9_hevc_add_gpe_surface(priv_ctx, HEVC_ENC_SURFACE_BRC_PAST_PAK_INFO,
 903                               &priv_ctx->res_brc_pak_statistic_buffer,
 904                               NULL);
 905     gen9_hevc_add_gpe_surface(priv_ctx, HEVC_ENC_SURFACE_HCP_PAK,
 906                               &priv_ctx->res_mb_code_surface,
 907                               NULL);
 908     gen9_hevc_add_gpe_surface(priv_ctx, HEVC_ENC_SURFACE_CU_RECORD,
 909                               &priv_ctx->res_mb_code_surface,
 910                               NULL);
 911     gen9_hevc_add_gpe_surface(priv_ctx, HEVC_ENC_SURFACE_32x32_PU_OUTPUT,
 912                               &priv_ctx->res_32x32_pu_output_buffer,
 913                               NULL);
 914     gen9_hevc_add_gpe_surface(priv_ctx, HEVC_ENC_SURFACE_SLICE_MAP,
 915                               &priv_ctx->res_slice_map_buffer,
 916                               NULL);
 917     gen9_hevc_add_gpe_surface(priv_ctx, HEVC_ENC_SURFACE_BRC_INPUT,
 918                               &priv_ctx->res_brc_input_buffer_for_enc_kernels,
 919                               NULL);
 920     gen9_hevc_add_gpe_surface(priv_ctx, HEVC_ENC_SURFACE_LCU_QP,
 921                               &priv_ctx->res_brc_mb_qp_buffer,
 922                               NULL);
 923     gen9_hevc_add_gpe_surface(priv_ctx, HEVC_ENC_SURFACE_ROI,
 924                               &priv_ctx->res_roi_buffer,
 925                               NULL);
 926     gen9_hevc_add_gpe_surface(priv_ctx, HEVC_ENC_SURFACE_RAW_MBSTAT,
 927                               &priv_ctx->res_mb_statistics_buffer,
 928                               NULL);
 929     gen9_hevc_add_gpe_surface(priv_ctx, HEVC_ENC_SURFACE_BRC_DATA,
 930                               &priv_ctx->res_brc_constant_data_buffer,
 931                               NULL);
 932     gen9_hevc_add_gpe_surface(priv_ctx, HEVC_ENC_SURFACE_KERNEL_DEBUG,
 933                               &priv_ctx->res_kernel_debug,
 934                               NULL);
 935     gen9_hevc_add_gpe_surface(priv_ctx, HEVC_ENC_SURFACE_SIMPLIFIED_INTRA,
 936                               &priv_ctx->res_simplest_intra_buffer,
 937                               NULL);
 938     gen9_hevc_add_gpe_surface(priv_ctx, HEVC_ENC_SURFACE_HME_MVP,
 939                               &priv_ctx->s4x_memv_data_buffer,
 940                               NULL);
 941     gen9_hevc_add_gpe_surface(priv_ctx, HEVC_ENC_SURFACE_HME_DIST,
 942                               &priv_ctx->s4x_memv_distortion_buffer,
 943                               NULL);
 944     gen9_hevc_add_gpe_surface(priv_ctx, HEVC_ENC_SURFACE_16x16PU_SAD,
 945                               &priv_ctx->res_sad_16x16_pu_buffer,
 946                               NULL);
 947     gen9_hevc_add_gpe_surface(priv_ctx, HEVC_ENC_SURFACE_VME_8x8,
 948                               &priv_ctx->res_vme_8x8_mode_buffer,
 949                               NULL);
 950     gen9_hevc_add_gpe_surface(priv_ctx, HEVC_ENC_SURFACE_INTRA_MODE,
 951                               &priv_ctx->res_intra_mode_buffer,
 952                               NULL);
 953     gen9_hevc_add_gpe_surface(priv_ctx, HEVC_ENC_SURFACE_INTRA_DIST,
 954                               &priv_ctx->res_intra_distortion_buffer,
 955                               NULL);
 956     gen9_hevc_add_gpe_surface(priv_ctx, HEVC_ENC_SURFACE_MIN_DIST,
 957                               &priv_ctx->res_min_distortion_buffer,
 958                               NULL);
 959     gen9_hevc_add_gpe_surface(priv_ctx, HEVC_ENC_SURFACE_VME_UNI_SIC_DATA,
 960                               &priv_ctx->res_vme_uni_sic_buffer,
 961                               NULL);
 962     gen9_hevc_add_gpe_surface(priv_ctx, HEVC_ENC_SURFACE_CONCURRENT_THREAD,
 963                               &priv_ctx->res_con_corrent_thread_buffer,
 964                               NULL);
 965     gen9_hevc_add_gpe_surface(priv_ctx, HEVC_ENC_SURFACE_MB_MV_INDEX,
 966                               &priv_ctx->res_mv_index_buffer,
 967                               NULL);
 968     gen9_hevc_add_gpe_surface(priv_ctx, HEVC_ENC_SURFACE_MVP_INDEX,
 969                               &priv_ctx->res_mvp_index_buffer,
 970                               NULL);
 971 }
 972
 973 static VAStatus
 974 gen9_hevc_enc_check_parameters(VADriverContextP ctx,
 975                                struct encode_state *encode_state,
 976                                struct intel_encoder_context *encoder_context)
 977 {
 978     VAEncSequenceParameterBufferHEVC *seq_param = NULL;
 979     VAEncPictureParameterBufferHEVC *pic_param = NULL;
 980     VAEncSliceParameterBufferHEVC *slice_param = NULL;
 981     int i = 0, j = 0;
 982
 983     seq_param = (VAEncSequenceParameterBufferHEVC *)encode_state->seq_param_ext->buffer;
 984     pic_param = (VAEncPictureParameterBufferHEVC *)encode_state->pic_param_ext->buffer;
 985
 986     for (i = 0; i < encode_state->num_slice_params_ext; i++) {
 987         slice_param = (VAEncSliceParameterBufferHEVC *)encode_state->slice_params_ext[i]->buffer;
 988
 989         if (slice_param->slice_fields.bits.slice_temporal_mvp_enabled_flag &&
 990             slice_param->slice_fields.bits.collocated_from_l0_flag &&
 991             (pic_param->collocated_ref_pic_index == 0xff ||
 992              pic_param->collocated_ref_pic_index > GEN9_MAX_REF_SURFACES))
 993             slice_param->slice_fields.bits.slice_temporal_mvp_enabled_flag = 0;
 994     }
 995
 996     i = 1 << (seq_param->log2_diff_max_min_luma_coding_block_size +
 997               seq_param->log2_min_luma_coding_block_size_minus3 + 3);
 998     if (i < GEN9_HEVC_ENC_MIN_LCU_SIZE ||
 999         i > GEN9_HEVC_ENC_MAX_LCU_SIZE)
1000         return VA_STATUS_ERROR_ATTR_NOT_SUPPORTED;
1001
1002     //The TU max size in SPS must be the same as the CU max size in SPS
1003     i = seq_param->log2_min_transform_block_size_minus2 +
1004         seq_param->log2_diff_max_min_transform_block_size + 2;
1005     j = seq_param->log2_min_luma_coding_block_size_minus3 +
1006         seq_param->log2_diff_max_min_luma_coding_block_size + 3;
1007
1008     if (i != j)
1009         return VA_STATUS_ERROR_ATTR_NOT_SUPPORTED;
1010
1011     slice_param = (VAEncSliceParameterBufferHEVC *)encode_state->slice_params_ext[0]->buffer;
1012     i = pic_param->pic_init_qp + slice_param->slice_qp_delta;
1013     j = -seq_param->seq_fields.bits.bit_depth_luma_minus8 * 6;
1014     if (i < j || i > 51)
1015         return VA_STATUS_ERROR_INVALID_PARAMETER;
1016
1017     return VA_STATUS_SUCCESS;
1018 }
1019
1020 static void
1021 gen9_hevc_enc_init_seq_parameters(struct gen9_hevc_encoder_context *priv_ctx,
1022                                   struct generic_enc_codec_state *generic_state,
1023                                   struct gen9_hevc_encoder_state *priv_state,
1024                                   VAEncSequenceParameterBufferHEVC *seq_param)
1025 {
1026     int new = 0, m = 0, n = 0;
1027
1028     if (priv_state->picture_width != seq_param->pic_width_in_luma_samples ||
1029         priv_state->picture_height != seq_param->pic_height_in_luma_samples ||
1030         priv_state->bit_depth_luma_minus8 != seq_param->seq_fields.bits.bit_depth_luma_minus8 ||
1031         priv_state->bit_depth_chroma_minus8 != seq_param->seq_fields.bits.bit_depth_chroma_minus8)
1032         new = 1;
1033
1034     if (!new)
1035         return;
1036
1037     priv_state->bit_depth_luma_minus8 = seq_param->seq_fields.bits.bit_depth_luma_minus8;
1038     priv_state->bit_depth_chroma_minus8 = seq_param->seq_fields.bits.bit_depth_chroma_minus8;
1039     priv_state->cu_size = 1 << (seq_param->log2_min_luma_coding_block_size_minus3 + 3);
1040     priv_state->lcu_size = 1 << (seq_param->log2_diff_max_min_luma_coding_block_size +
1041                                  seq_param->log2_min_luma_coding_block_size_minus3 + 3);
1042     priv_state->picture_width = (seq_param->pic_width_in_luma_samples / priv_state->cu_size) * priv_state->cu_size;
1043     priv_state->picture_height = (seq_param->pic_height_in_luma_samples / priv_state->cu_size) * priv_state->cu_size;
1044     priv_state->width_in_lcu  = ALIGN(priv_state->picture_width, priv_state->lcu_size) / priv_state->lcu_size;
1045     priv_state->height_in_lcu = ALIGN(priv_state->picture_height, priv_state->lcu_size) / priv_state->lcu_size;
1046     priv_state->width_in_cu  = ALIGN(priv_state->picture_width, priv_state->cu_size) / priv_state->cu_size;
1047     priv_state->height_in_cu = ALIGN(priv_state->picture_height, priv_state->cu_size) / priv_state->cu_size;
1048     priv_state->width_in_mb  = ALIGN(priv_state->picture_width, 16) / 16;
1049     priv_state->height_in_mb = ALIGN(priv_state->picture_height, 16) / 16;
1050
1051     m = (priv_state->picture_width + GEN9_HEVC_ENC_MIN_LCU_SIZE - 1) / GEN9_HEVC_ENC_MIN_LCU_SIZE;
1052     n = (priv_state->picture_height + GEN9_HEVC_ENC_MIN_LCU_SIZE - 1) / GEN9_HEVC_ENC_MIN_LCU_SIZE;
1053     priv_state->mb_data_offset = ALIGN(m * n * priv_state->pak_obj_size, 0x1000);
1054
1055     m = ALIGN(priv_state->picture_width, GEN9_HEVC_ENC_MAX_LCU_SIZE) / 8;
1056     n = ALIGN(priv_state->picture_height, GEN9_HEVC_ENC_MAX_LCU_SIZE) / 8;
1057     priv_state->mb_code_size = priv_state->mb_data_offset +
1058                                ALIGN(m * n * priv_state->cu_record_size, 0x1000);
1059
1060     priv_state->frame_width_in_max_lcu = ALIGN(priv_state->picture_width, 32);
1061     priv_state->frame_height_in_max_lcu = ALIGN(priv_state->picture_height, 32);
1062     priv_state->frame_width_4x = ALIGN(priv_state->picture_width / 4, 16);
1063     priv_state->frame_height_4x = ALIGN(priv_state->picture_height / 4, 16);
1064     priv_state->frame_width_16x = ALIGN(priv_state->picture_width / 16, 16);
1065     priv_state->frame_height_16x = ALIGN(priv_state->picture_height / 16, 16);
1066     priv_state->frame_width_32x = ALIGN(priv_state->picture_width / 32, 16);
1067     priv_state->frame_height_32x = ALIGN(priv_state->picture_height / 32, 16);
1068
1069     priv_state->downscaled_width_4x_in_mb = priv_state->frame_width_4x / 16;
1070     if (priv_state->bit_depth_luma_minus8) {
1071         priv_state->downscaled_width_4x_in_mb = ALIGN(priv_state->downscaled_width_4x_in_mb * 16, 32) /
1072                                                 16;
1073         priv_state->frame_width_4x = priv_state->downscaled_width_4x_in_mb * 16;
1074     }
1075
1076     priv_state->downscaled_height_4x_in_mb = priv_state->frame_height_4x / 16;
1077     priv_state->downscaled_width_16x_in_mb = priv_state->frame_width_16x / 16;
1078     priv_state->downscaled_height_16x_in_mb = priv_state->frame_height_16x / 16;
1079     priv_state->downscaled_width_32x_in_mb = priv_state->frame_width_32x / 16;
1080     priv_state->downscaled_height_32x_in_mb = priv_state->frame_height_32x / 16;
1081     priv_state->flatness_check_enable = priv_state->flatness_check_supported;
1082     priv_state->widi_first_intra_refresh = 1;
1083
1084     generic_state->hme_supported = GEN9_HEVC_HME_SUPPORTED;
1085     generic_state->b16xme_supported = GEN9_HEVC_16XME_SUPPORTED;
1086     generic_state->b32xme_supported = GEN9_HEVC_32XME_SUPPORTED;
1087     if (generic_state->hme_supported &&
1088         (priv_state->frame_width_4x < GEN9_HEVC_VME_MIN_ALLOWED_SIZE ||
1089          priv_state->frame_height_4x < GEN9_HEVC_VME_MIN_ALLOWED_SIZE)) {
1090         generic_state->b16xme_supported = 0;
1091         generic_state->b32xme_supported = 0;
1092
1093         if (priv_state->frame_width_4x < GEN9_HEVC_VME_MIN_ALLOWED_SIZE) {
1094             priv_state->frame_width_4x = GEN9_HEVC_VME_MIN_ALLOWED_SIZE;
1095             priv_state->downscaled_width_4x_in_mb = priv_state->frame_width_4x / 16;
1096         }
1097
1098         if (priv_state->frame_height_4x < GEN9_HEVC_VME_MIN_ALLOWED_SIZE) {
1099             priv_state->frame_height_4x = GEN9_HEVC_VME_MIN_ALLOWED_SIZE;
1100             priv_state->downscaled_height_4x_in_mb = priv_state->frame_height_4x / 16;
1101         }
1102     } else if (generic_state->b16xme_supported &&
1103                (priv_state->frame_width_16x < GEN9_HEVC_VME_MIN_ALLOWED_SIZE ||
1104                 priv_state->frame_height_16x < GEN9_HEVC_VME_MIN_ALLOWED_SIZE)) {
1105         generic_state->b32xme_supported = 0;
1106
1107         if (priv_state->frame_width_16x < GEN9_HEVC_VME_MIN_ALLOWED_SIZE) {
1108             priv_state->frame_width_16x = GEN9_HEVC_VME_MIN_ALLOWED_SIZE;
1109             priv_state->downscaled_width_16x_in_mb = priv_state->frame_width_16x / 16;
1110         }
1111
1112         if (priv_state->frame_height_16x < GEN9_HEVC_VME_MIN_ALLOWED_SIZE) {
1113             priv_state->frame_height_16x = GEN9_HEVC_VME_MIN_ALLOWED_SIZE;
1114             priv_state->downscaled_height_16x_in_mb = priv_state->frame_height_16x / 16;
1115         }
1116     } else if (generic_state->b32xme_supported &&
1117                (priv_state->frame_width_32x < GEN9_HEVC_VME_MIN_ALLOWED_SIZE ||
1118                 priv_state->frame_height_32x < GEN9_HEVC_VME_MIN_ALLOWED_SIZE)) {
1119         if (priv_state->frame_width_32x < GEN9_HEVC_VME_MIN_ALLOWED_SIZE) {
1120             priv_state->frame_width_32x = GEN9_HEVC_VME_MIN_ALLOWED_SIZE;
1121             priv_state->downscaled_width_32x_in_mb = priv_state->frame_width_32x / 16;
1122         }
1123
1124         if (priv_state->frame_height_32x < GEN9_HEVC_VME_MIN_ALLOWED_SIZE) {
1125             priv_state->frame_height_32x = GEN9_HEVC_VME_MIN_ALLOWED_SIZE;
1126             priv_state->downscaled_height_32x_in_mb = priv_state->frame_height_32x / 16;
1127         }
1128     }
1129
1130     priv_state->gop_ref_dist = seq_param->ip_period;
1131     priv_state->gop_size = seq_param->intra_period;
1132     priv_state->frame_number = 0;
1133
1134     priv_ctx->res_inited = 0;
1135 }
1136
1137 static void
1138 gen9_hevc_enc_init_pic_parameters(struct generic_enc_codec_state *generic_state,
1139                                   struct gen9_hevc_encoder_state *priv_state,
1140                                   VAEncSequenceParameterBufferHEVC *seq_param,
1141                                   VAEncPictureParameterBufferHEVC *pic_param,
1142                                   VAEncSliceParameterBufferHEVC *slice_param)
1143 {
1144     unsigned int log2_max_coding_block_size = 0, raw_ctu_bits = 0;
1145
1146     priv_state->picture_coding_type = slice_param->slice_type;
1147
1148     priv_state->ctu_max_bitsize_allowed = pic_param->ctu_max_bitsize_allowed;
1149     log2_max_coding_block_size  = seq_param->log2_min_luma_coding_block_size_minus3 + 3 +
1150                                   seq_param->log2_diff_max_min_luma_coding_block_size;
1151     raw_ctu_bits = (1 << (2 * log2_max_coding_block_size + 3)) +
1152                    (1 << (2 * log2_max_coding_block_size + 2));
1153     raw_ctu_bits = (5 * raw_ctu_bits / 3);
1154
1155     if (priv_state->ctu_max_bitsize_allowed == 0 ||
1156         priv_state->ctu_max_bitsize_allowed > raw_ctu_bits)
1157         priv_state->ctu_max_bitsize_allowed = raw_ctu_bits;
1158 }
1159
1160 static void
1161 gen9_hevc_enc_init_slice_parameters(VADriverContextP ctx,
1162                                     struct encode_state *encode_state,
1163                                     struct intel_encoder_context *encoder_context)
1164 {
1165     struct encoder_vme_mfc_context *vme_context = NULL;
1166     struct gen9_hevc_encoder_state *priv_state = NULL;
1167     VAEncPictureParameterBufferHEVC *pic_param = NULL;
1168     VAEncSliceParameterBufferHEVC *slice_param = NULL;
1169     int i = 0, j = 0;
1170
1171     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
1172     priv_state = (struct gen9_hevc_encoder_state *)vme_context->private_enc_state;
1173     pic_param = (VAEncPictureParameterBufferHEVC *)encode_state->pic_param_ext->buffer;
1174
1175     priv_state->low_delay = 1;
1176     priv_state->arbitrary_num_mb_in_slice = 0;
1177
1178     for (i = 0; i < encode_state->num_slice_params_ext; i++) {
1179         slice_param = (VAEncSliceParameterBufferHEVC *)encode_state->slice_params_ext[i]->buffer;
1180
1181         if (slice_param->slice_type == HEVC_SLICE_B && priv_state->low_delay) {
1182             for (j = 0; j <= slice_param->num_ref_idx_l0_active_minus1; j++) {
1183                 if (pic_param->decoded_curr_pic.pic_order_cnt <
1184                     slice_param->ref_pic_list0[j].pic_order_cnt)
1185                     priv_state->low_delay = 0;
1186             }
1187
1188             for (j = 0; j <= slice_param->num_ref_idx_l1_active_minus1; j++) {
1189                 if (pic_param->decoded_curr_pic.pic_order_cnt <
1190                     slice_param->ref_pic_list1[j].pic_order_cnt)
1191                     priv_state->low_delay = 0;
1192             }
1193         }
1194
1195         if (!priv_state->arbitrary_num_mb_in_slice &&
1196             (slice_param->num_ctu_in_slice % priv_state->width_in_lcu))
1197             priv_state->arbitrary_num_mb_in_slice = 1;
1198     }
1199 }
1200
1201 static VAStatus
1202 gen9_hevc_enc_init_parameters(VADriverContextP ctx,
1203                               struct encode_state *encode_state,
1204                               struct intel_encoder_context *encoder_context)
1205 {
1206     struct i965_driver_data *i965 = i965_driver_data(ctx);
1207     struct encoder_vme_mfc_context *vme_context = NULL;
1208     struct generic_enc_codec_state *generic_state = NULL;
1209     struct gen9_hevc_encoder_context *priv_ctx = NULL;
1210     struct gen9_hevc_encoder_state *priv_state = NULL;
1211     VAEncPictureParameterBufferHEVC *pic_param = NULL;
1212     VAEncSequenceParameterBufferHEVC *seq_param = NULL;
1213     VAEncSliceParameterBufferHEVC *slice_param = NULL;
1214     struct object_buffer *obj_buffer = NULL;
1215     VAStatus va_status = VA_STATUS_SUCCESS;
1216
1217     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
1218     generic_state = (struct generic_enc_codec_state *)vme_context->generic_enc_state;
1219     priv_ctx = (struct gen9_hevc_encoder_context *)vme_context->private_enc_ctx;
1220     priv_state = (struct gen9_hevc_encoder_state *)vme_context->private_enc_state;
1221     pic_param = (VAEncPictureParameterBufferHEVC *)encode_state->pic_param_ext->buffer;
1222     seq_param = (VAEncSequenceParameterBufferHEVC *)encode_state->seq_param_ext->buffer;
1223     slice_param = (VAEncSliceParameterBufferHEVC *)encode_state->slice_params_ext[0]->buffer;
1224
1225     if (!pic_param || !seq_param || !slice_param)
1226         return VA_STATUS_ERROR_INVALID_PARAMETER;
1227
1228     va_status = gen9_hevc_enc_check_parameters(ctx, encode_state, encoder_context);
1229     if (va_status |= VA_STATUS_SUCCESS)
1230         return va_status;
1231
1232     gen9_hevc_enc_init_seq_parameters(priv_ctx, generic_state, priv_state, seq_param);
1233     gen9_hevc_enc_init_pic_parameters(generic_state, priv_state, seq_param, pic_param, slice_param);
1234     gen9_hevc_enc_init_slice_parameters(ctx, encode_state, encoder_context);
1235
1236     if (priv_state->picture_coding_type == HEVC_SLICE_I) {
1237         generic_state->hme_enabled = 0;
1238         generic_state->b16xme_enabled = 0;
1239         generic_state->b32xme_enabled = 0;
1240     } else {
1241         generic_state->hme_enabled = generic_state->hme_supported;
1242         generic_state->b16xme_enabled = generic_state->b16xme_supported;
1243         generic_state->b32xme_enabled = generic_state->b32xme_supported;
1244     }
1245
1246     obj_buffer = BUFFER(pic_param->coded_buf);
1247     if (!obj_buffer ||
1248         !obj_buffer->buffer_store ||
1249         !obj_buffer->buffer_store->bo) {
1250         va_status = VA_STATUS_ERROR_INVALID_PARAMETER;
1251         goto EXIT;
1252     }
1253     encode_state->coded_buf_object = obj_buffer;
1254     priv_state->status_buffer.bo = obj_buffer->buffer_store->bo;
1255
1256     va_status = gen9_hevc_ensure_surface(ctx, priv_state,
1257                                          encode_state->input_yuv_object, 0);
1258     if (va_status != VA_STATUS_SUCCESS)
1259         goto EXIT;
1260
1261     va_status = gen9_hevc_ensure_surface(ctx, priv_state,
1262                                          encode_state->reconstructed_object, 1);
1263     if (va_status != VA_STATUS_SUCCESS)
1264         goto EXIT;
1265
1266 #if 0
1267     if (encode_state->input_yuv_object->orig_width > priv_state->picture_width)
1268         encode_state->input_yuv_object->orig_width = priv_state->picture_width;
1269
1270     if (encode_state->input_yuv_object->orig_height > priv_state->picture_height)
1271         encode_state->input_yuv_object->orig_height = priv_state->picture_height;
1272
1273
1274     if (encode_state->reconstructed_object->orig_width > priv_state->picture_width)
1275         encode_state->reconstructed_object->orig_width = priv_state->picture_width;
1276
1277     if (encode_state->reconstructed_object->orig_height > priv_state->picture_height)
1278         encode_state->reconstructed_object->orig_height = priv_state->picture_height;
1279 #endif
1280
1281     va_status = gen9_hevc_init_surface_private(ctx, generic_state, priv_state,
1282                                                encode_state->reconstructed_object);
1283     if (va_status != VA_STATUS_SUCCESS)
1284         goto EXIT;
1285
1286     {
1287         struct gen9_hevc_surface_priv *surface_priv = NULL;
1288
1289         surface_priv = (struct gen9_hevc_surface_priv *)encode_state->reconstructed_object->private_data;
1290
1291         surface_priv->qp_value = pic_param->pic_init_qp + slice_param->slice_qp_delta;
1292     }
1293
1294     va_status = gen9_hevc_enc_alloc_resources(ctx, encode_state,
1295                                               encoder_context);
1296     if (va_status != VA_STATUS_SUCCESS)
1297         goto EXIT;
1298
1299     gen9_hevc_init_gpe_surfaces_table(ctx, encode_state,
1300                                       encoder_context);
1301
1302 EXIT:
1303     return va_status;
1304 }
1305
1306 // VME&BRC implementation
1307
1308 static void
1309 gen9_hevc_vme_init_gpe_context(VADriverContextP ctx,
1310                                struct i965_gpe_context *gpe_context,
1311                                unsigned int curbe_size,
1312                                unsigned int inline_data_size)
1313 {
1314     struct i965_driver_data *i965 = i965_driver_data(ctx);
1315
1316     gpe_context->curbe.length = curbe_size;
1317
1318     gpe_context->sampler.entry_size = 0;
1319     gpe_context->sampler.max_entries = 0;
1320
1321     gpe_context->idrt.entry_size = ALIGN(sizeof(struct gen8_interface_descriptor_data), 64);
1322     gpe_context->idrt.max_entries = 1;
1323
1324     gpe_context->surface_state_binding_table.max_entries = MAX_HEVC_KERNELS_ENCODER_SURFACES;
1325     gpe_context->surface_state_binding_table.binding_table_offset = 0;
1326     gpe_context->surface_state_binding_table.surface_state_offset = ALIGN(MAX_HEVC_KERNELS_ENCODER_SURFACES * 4, 64);
1327     gpe_context->surface_state_binding_table.length = ALIGN(MAX_HEVC_KERNELS_ENCODER_SURFACES * 4, 64) + ALIGN(MAX_HEVC_KERNELS_ENCODER_SURFACES * SURFACE_STATE_PADDED_SIZE_GEN9, 64);
1328
1329     if (i965->intel.eu_total > 0)
1330         gpe_context->vfe_state.max_num_threads = 6 * i965->intel.eu_total;
1331     else
1332         gpe_context->vfe_state.max_num_threads = 112;
1333
1334     gpe_context->vfe_state.curbe_allocation_size = MAX(1, ALIGN(gpe_context->curbe.length, 32) >> 5);
1335     gpe_context->vfe_state.urb_entry_size = MAX(1, ALIGN(inline_data_size, 32) >> 5);
1336     gpe_context->vfe_state.num_urb_entries = (MAX_HEVC_KERNELS_URB_SIZE -
1337                                               gpe_context->vfe_state.curbe_allocation_size -
1338                                               ((gpe_context->idrt.entry_size >> 5) *
1339                                                gpe_context->idrt.max_entries)) /
1340                                              gpe_context->vfe_state.urb_entry_size;
1341     gpe_context->vfe_state.num_urb_entries = CLAMP(gpe_context->vfe_state.num_urb_entries, 1, 64);
1342     gpe_context->vfe_state.gpgpu_mode = 0;
1343 }
1344
1345 static void
1346 gen9_hevc_vme_init_scoreboard(struct i965_gpe_context *gpe_context,
1347                               unsigned int mask,
1348                               unsigned int enable,
1349                               unsigned int type)
1350 {
1351     gpe_context->vfe_desc5.scoreboard0.mask = mask;
1352     gpe_context->vfe_desc5.scoreboard0.type = type;
1353     gpe_context->vfe_desc5.scoreboard0.enable = enable;
1354
1355     gpe_context->vfe_desc6.scoreboard1.delta_x0 = 0xF;
1356     gpe_context->vfe_desc6.scoreboard1.delta_y0 = 0x0;
1357
1358     gpe_context->vfe_desc6.scoreboard1.delta_x1 = 0x0;
1359     gpe_context->vfe_desc6.scoreboard1.delta_y1 = 0xF;
1360
1361     gpe_context->vfe_desc6.scoreboard1.delta_x2 = 0x1;
1362     gpe_context->vfe_desc6.scoreboard1.delta_y2 = 0xF;
1363
1364     gpe_context->vfe_desc6.scoreboard1.delta_x3 = 0xF;
1365     gpe_context->vfe_desc6.scoreboard1.delta_y3 = 0xF;
1366
1367     gpe_context->vfe_desc7.scoreboard2.delta_x4 = 0xF;
1368     gpe_context->vfe_desc7.scoreboard2.delta_y4 = 0x1;
1369
1370     gpe_context->vfe_desc7.scoreboard2.delta_x5 = 0x0;
1371     gpe_context->vfe_desc7.scoreboard2.delta_y5 = 0xE;
1372
1373     gpe_context->vfe_desc7.scoreboard2.delta_x6 = 0x1;
1374     gpe_context->vfe_desc7.scoreboard2.delta_y6 = 0xE;
1375
1376     gpe_context->vfe_desc7.scoreboard2.delta_x7 = 0xF;
1377     gpe_context->vfe_desc7.scoreboard2.delta_y7 = 0xE;
1378 }
1379
1380 static void
1381 gen9_hevc_vme_set_scoreboard_26z(struct i965_gpe_context *gpe_context,
1382                                  unsigned int mask,
1383                                  unsigned int enable,
1384                                  unsigned int type)
1385 {
1386     gpe_context->vfe_desc5.scoreboard0.mask = mask;
1387     gpe_context->vfe_desc5.scoreboard0.type = type;
1388     gpe_context->vfe_desc5.scoreboard0.enable = enable;
1389
1390     gpe_context->vfe_desc6.scoreboard1.delta_x0 = -1;
1391     gpe_context->vfe_desc6.scoreboard1.delta_y0 = 3;
1392
1393     gpe_context->vfe_desc6.scoreboard1.delta_x1 = -1;
1394     gpe_context->vfe_desc6.scoreboard1.delta_y1 = 1;
1395
1396     gpe_context->vfe_desc6.scoreboard1.delta_x2 = -1;
1397     gpe_context->vfe_desc6.scoreboard1.delta_y2 = -1;
1398
1399     gpe_context->vfe_desc6.scoreboard1.delta_x3 = 0;
1400     gpe_context->vfe_desc6.scoreboard1.delta_y3 = -1;
1401
1402     gpe_context->vfe_desc7.scoreboard2.delta_x4 = 0;
1403     gpe_context->vfe_desc7.scoreboard2.delta_y4 = -2;
1404
1405     gpe_context->vfe_desc7.scoreboard2.delta_x5 = 0;
1406     gpe_context->vfe_desc7.scoreboard2.delta_y5 = -3;
1407
1408     gpe_context->vfe_desc7.scoreboard2.delta_x6 = 1;
1409     gpe_context->vfe_desc7.scoreboard2.delta_y6 = -2;
1410
1411     gpe_context->vfe_desc7.scoreboard2.delta_x7 = 1;
1412     gpe_context->vfe_desc7.scoreboard2.delta_y7 = -3;
1413 }
1414
1415 static void
1416 gen9_hevc_vme_set_scoreboard_26(struct i965_gpe_context *gpe_context,
1417                                 unsigned int mask,
1418                                 unsigned int enable,
1419                                 unsigned int type)
1420 {
1421     gpe_context->vfe_desc5.scoreboard0.mask = mask;
1422     gpe_context->vfe_desc5.scoreboard0.type = type;
1423     gpe_context->vfe_desc5.scoreboard0.enable = enable;
1424
1425     gpe_context->vfe_desc6.scoreboard1.delta_x0 = -1;
1426     gpe_context->vfe_desc6.scoreboard1.delta_y0 = 0;
1427
1428     gpe_context->vfe_desc6.scoreboard1.delta_x1 = -1;
1429     gpe_context->vfe_desc6.scoreboard1.delta_y1 = -1;
1430
1431     gpe_context->vfe_desc6.scoreboard1.delta_x2 = 0;
1432     gpe_context->vfe_desc6.scoreboard1.delta_y2 = -1;
1433
1434     gpe_context->vfe_desc6.scoreboard1.delta_x3 = 1;
1435     gpe_context->vfe_desc6.scoreboard1.delta_y3 = -1;
1436 }
1437
1438 static void
1439 gen9_hevc_init_object_walker(struct hevc_enc_kernel_walker_parameter *hevc_walker_param,
1440                              struct gpe_media_object_walker_parameter *gpe_param)
1441 {
1442     memset(gpe_param, 0, sizeof(*gpe_param));
1443
1444     gpe_param->use_scoreboard = hevc_walker_param->use_scoreboard;
1445     gpe_param->block_resolution.x = hevc_walker_param->resolution_x;
1446     gpe_param->block_resolution.y = hevc_walker_param->resolution_y;
1447     gpe_param->global_resolution.x = hevc_walker_param->resolution_x;
1448     gpe_param->global_resolution.y = hevc_walker_param->resolution_y;
1449     gpe_param->global_outer_loop_stride.x = hevc_walker_param->resolution_x;
1450     gpe_param->global_outer_loop_stride.y = 0;
1451     gpe_param->global_inner_loop_unit.x = 0;
1452     gpe_param->global_inner_loop_unit.y = hevc_walker_param->resolution_y;
1453     gpe_param->local_loop_exec_count = 0xFFFF;
1454     gpe_param->global_loop_exec_count = 0xFFFF;
1455
1456     if (hevc_walker_param->no_dependency) {
1457         gpe_param->scoreboard_mask = 0;
1458         gpe_param->use_scoreboard = 0;
1459         gpe_param->local_outer_loop_stride.x = 0;
1460         gpe_param->local_outer_loop_stride.y = 1;
1461         gpe_param->local_inner_loop_unit.x = 1;
1462         gpe_param->local_inner_loop_unit.y = 0;
1463         gpe_param->local_end.x = hevc_walker_param->resolution_x - 1;
1464         gpe_param->local_end.y = 0;
1465     }
1466 }
1467
1468 static void
1469 gen9_hevc_init_object_walker_26z(struct gen9_hevc_encoder_state *priv_state,
1470                                  struct i965_gpe_context *gpe_context,
1471                                  struct gpe_media_object_walker_parameter *gpe_param,
1472                                  int split_count,
1473                                  unsigned int max_slight_height,
1474                                  int use_hw_scoreboard,
1475                                  int scoreboard_type)
1476 {
1477     int width = priv_state->width_in_mb;
1478     int height = max_slight_height * 2;
1479     int ts_width = ((width + 3) & 0xfffc) >> 1;
1480     int lcu_width = (width + 1) >> 1;
1481     int lcu_height = (height + 1) >> 1;
1482     int tmp1 = ((lcu_width + 1) >> 1) + ((lcu_width + ((lcu_height - 1) << 1)) + (2 * split_count - 1)) /
1483                (2 * split_count);
1484
1485     gpe_param->use_scoreboard  = use_hw_scoreboard;
1486     gpe_param->scoreboard_mask = 0xFF;
1487     gpe_param->global_resolution.x = ts_width;
1488     gpe_param->global_resolution.y = 4 * tmp1;
1489     gpe_param->global_start.x = 0;
1490     gpe_param->global_start.y = 0;
1491     gpe_param->global_outer_loop_stride.x = ts_width;
1492     gpe_param->global_outer_loop_stride.y = 0;
1493     gpe_param->global_inner_loop_unit.x = 0;
1494     gpe_param->global_inner_loop_unit.y = 4 * tmp1;
1495     gpe_param->block_resolution.x = ts_width;
1496     gpe_param->block_resolution.y = 4 * tmp1;
1497     gpe_param->local_start.x = ts_width;
1498     gpe_param->local_start.y = 0;
1499     gpe_param->local_end.x = 0;
1500     gpe_param->local_end.y = 0;
1501     gpe_param->local_outer_loop_stride.x = 1;
1502     gpe_param->local_outer_loop_stride.y = 0;
1503     gpe_param->local_inner_loop_unit.x = -2;
1504     gpe_param->local_inner_loop_unit.y = 4;
1505     gpe_param->middle_loop_extra_steps = 3;
1506     gpe_param->mid_loop_unit_x = 0;
1507     gpe_param->mid_loop_unit_y = 1;
1508     gpe_param->global_loop_exec_count = 0;
1509     gpe_param->local_loop_exec_count = ((lcu_width + (lcu_height - 1) * 2 + 2 * split_count - 1) /
1510                                         (2 * split_count)) * 2 -
1511                                        1;
1512
1513     gen9_hevc_vme_set_scoreboard_26z(gpe_context, 0xff, use_hw_scoreboard, scoreboard_type);
1514 }
1515
1516 static void
1517 gen9_hevc_init_object_walker_26(struct gen9_hevc_encoder_state *priv_state,
1518                                 struct i965_gpe_context *gpe_context,
1519                                 struct gpe_media_object_walker_parameter *gpe_param,
1520                                 int split_count,
1521                                 unsigned int max_slight_height,
1522                                 int use_hw_scoreboard,
1523                                 int scoreboard_type)
1524 {
1525     int width = priv_state->width_in_mb;
1526     int height = max_slight_height;
1527     int ts_width = (width + 1) & 0xfffe;
1528     int ts_height = (height + 1) & 0xfffe;
1529     int tmp1 = ((ts_width + 1) >> 1) +
1530                ((ts_width + ((ts_height - 1) << 1)) +
1531                 (2 * split_count - 1)) / (2 * split_count);
1532
1533     gpe_param->use_scoreboard = use_hw_scoreboard;
1534     gpe_param->scoreboard_mask = 0x0f;
1535     gpe_param->global_resolution.x = ts_width;
1536     gpe_param->global_resolution.y = tmp1;
1537     gpe_param->global_start.x = 0;
1538     gpe_param->global_start.y = 0;
1539     gpe_param->global_outer_loop_stride.x = ts_width;
1540     gpe_param->global_outer_loop_stride.y = 0;
1541     gpe_param->global_inner_loop_unit.x = 0;
1542     gpe_param->global_inner_loop_unit.y = tmp1;
1543     gpe_param->block_resolution.x = ts_width;
1544     gpe_param->block_resolution.y = tmp1;
1545     gpe_param->local_start.x = ts_width;
1546     gpe_param->local_start.y = 0;
1547     gpe_param->local_end.x = 0;
1548     gpe_param->local_end.y = 0;
1549     gpe_param->local_outer_loop_stride.x = 1;
1550     gpe_param->local_outer_loop_stride.y = 0;
1551     gpe_param->local_inner_loop_unit.x = -2;
1552     gpe_param->local_inner_loop_unit.y = 1;
1553     gpe_param->middle_loop_extra_steps = 0;
1554     gpe_param->mid_loop_unit_x = 0;
1555     gpe_param->mid_loop_unit_y = 0;
1556     gpe_param->global_loop_exec_count = 0;
1557     gpe_param->local_loop_exec_count = (width + (height - 1) * 2 + split_count - 1) /
1558                                        split_count;
1559
1560     gen9_hevc_vme_set_scoreboard_26(gpe_context, 0xff, use_hw_scoreboard, scoreboard_type);
1561 }
1562
1563 static void
1564 gen9_hevc_run_object_walker(VADriverContextP ctx,
1565                             struct intel_encoder_context *encoder_context,
1566                             struct i965_gpe_context *gpe_context,
1567                             struct gpe_media_object_walker_parameter *param,
1568                             int media_state)
1569 {
1570     struct intel_batchbuffer *batch = encoder_context->base.batch;
1571
1572     intel_batchbuffer_start_atomic(batch, 0x1000);
1573
1574     intel_batchbuffer_emit_mi_flush(batch);
1575
1576     gen9_gpe_pipeline_setup(ctx, gpe_context, batch);
1577     gen8_gpe_media_object_walker(ctx, gpe_context, batch, param);
1578     gen8_gpe_media_state_flush(ctx, gpe_context, batch);
1579     gen9_gpe_pipeline_end(ctx, gpe_context, batch);
1580
1581     intel_batchbuffer_end_atomic(batch);
1582
1583     intel_batchbuffer_flush(batch);
1584 }
1585
1586 static void
1587 gen9_hevc_run_object(VADriverContextP ctx,
1588                      struct intel_encoder_context *encoder_context,
1589                      struct i965_gpe_context *gpe_context,
1590                      struct gpe_media_object_parameter *param,
1591                      int media_state)
1592 {
1593     struct intel_batchbuffer *batch = encoder_context->base.batch;
1594
1595     intel_batchbuffer_start_atomic(batch, 0x1000);
1596
1597     intel_batchbuffer_emit_mi_flush(batch);
1598
1599     gen9_gpe_pipeline_setup(ctx, gpe_context, batch);
1600     gen8_gpe_media_object(ctx, gpe_context, batch, param);
1601     gen8_gpe_media_state_flush(ctx, gpe_context, batch);
1602
1603     gen9_gpe_pipeline_end(ctx, gpe_context, batch);
1604
1605     intel_batchbuffer_end_atomic(batch);
1606
1607     intel_batchbuffer_flush(batch);
1608 }
1609
1610 static void
1611 gen9_hevc_get_b_mbenc_default_curbe(enum HEVC_TU_MODE tu_mode,
1612                                     int slice_type,
1613                                     void **curbe_ptr,
1614                                     int *curbe_size)
1615 {
1616     if (tu_mode == HEVC_TU_BEST_SPEED) {
1617         if (slice_type == HEVC_SLICE_I) {
1618             *curbe_size = sizeof(HEVC_ENC_ENCB_TU7_I_CURBE_DATA);
1619             *curbe_ptr = (void *)HEVC_ENC_ENCB_TU7_I_CURBE_DATA;
1620         } else if (slice_type == HEVC_SLICE_P) {
1621             *curbe_size = sizeof(HEVC_ENC_ENCB_TU7_P_CURBE_DATA);
1622             *curbe_ptr = (void *)HEVC_ENC_ENCB_TU7_P_CURBE_DATA;
1623         } else {
1624             *curbe_size = sizeof(HEVC_ENC_ENCB_TU7_B_CURBE_DATA);
1625             *curbe_ptr = (void *)HEVC_ENC_ENCB_TU7_B_CURBE_DATA;
1626         }
1627     } else if (tu_mode == HEVC_TU_RT_SPEED) {
1628         if (slice_type == HEVC_SLICE_P) {
1629             *curbe_size = sizeof(HEVC_ENC_ENCB_TU4_P_CURBE_DATA);
1630             *curbe_ptr = (void *)HEVC_ENC_ENCB_TU4_P_CURBE_DATA;
1631         } else {
1632             *curbe_size = sizeof(HEVC_ENC_ENCB_TU4_B_CURBE_DATA);
1633             *curbe_ptr = (void *)HEVC_ENC_ENCB_TU4_B_CURBE_DATA;
1634         }
1635     } else {
1636         if (slice_type == HEVC_SLICE_P) {
1637             *curbe_size = sizeof(HEVC_ENC_ENCB_TU1_P_CURBE_DATA);
1638             *curbe_ptr = (void *)HEVC_ENC_ENCB_TU1_P_CURBE_DATA;
1639         } else {
1640             *curbe_size = sizeof(HEVC_ENC_ENCB_TU1_B_CURBE_DATA);
1641             *curbe_ptr = (void *)HEVC_ENC_ENCB_TU1_B_CURBE_DATA;
1642         }
1643     }
1644 }
1645
1646 // BRC start
1647
1648 static void
1649 gen9_hevc_brc_prepare(struct encode_state *encode_state,
1650                       struct intel_encoder_context *encoder_context)
1651 {
1652     struct encoder_vme_mfc_context *vme_context = NULL;
1653     struct generic_enc_codec_state *generic_state = NULL;
1654     struct gen9_hevc_encoder_state *priv_state = NULL;
1655     enum HEVC_BRC_METHOD brc_method = HEVC_BRC_CQP;
1656     int internal_tu_mode = encoder_context->quality_level;
1657     int brc_reset = 0;
1658
1659     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
1660     generic_state = (struct generic_enc_codec_state *)vme_context->generic_enc_state;
1661     priv_state = (struct gen9_hevc_encoder_state *)vme_context->private_enc_state;
1662
1663     if (encoder_context->rate_control_mode & VA_RC_CBR)
1664         brc_method = HEVC_BRC_CBR;
1665     else if (encoder_context->rate_control_mode & VA_RC_VBR)
1666         brc_method = HEVC_BRC_VBR;
1667     else if (encoder_context->rate_control_mode & VA_RC_VCM)
1668         brc_method = HEVC_BRC_VCM;
1669
1670     if (internal_tu_mode >= HEVC_TU_RT_SPEED ||
1671         internal_tu_mode == 0)
1672         internal_tu_mode = internal_tu_mode >= HEVC_TU_BEST_SPEED ?
1673                            HEVC_TU_BEST_SPEED : HEVC_TU_RT_SPEED;
1674     else
1675         internal_tu_mode = HEVC_TU_BEST_QUALITY;
1676
1677     brc_reset = priv_state->brc_method != brc_method ||
1678                 priv_state->tu_mode != internal_tu_mode;
1679
1680     if (!generic_state->brc_inited ||
1681         encoder_context->brc.need_reset ||
1682         brc_reset) {
1683         priv_state->tu_mode = internal_tu_mode;
1684         if (priv_state->tu_mode == HEVC_TU_BEST_QUALITY)
1685             priv_state->num_regions_in_slice = 1;
1686         else
1687             priv_state->num_regions_in_slice = 4;
1688
1689         if (brc_method == HEVC_BRC_CQP) {
1690             generic_state->brc_enabled = 0;
1691             generic_state->num_pak_passes = 1;
1692             priv_state->lcu_brc_enabled = 0;
1693         } else {
1694             generic_state->brc_enabled = 1;
1695             generic_state->num_pak_passes = 4;
1696
1697             if (brc_method == HEVC_BRC_VCM ||
1698                 encoder_context->brc.mb_rate_control[0] == 0)
1699                 priv_state->lcu_brc_enabled = (priv_state->tu_mode == HEVC_TU_BEST_QUALITY);
1700             else if (brc_method == HEVC_BRC_ICQ ||
1701                      encoder_context->brc.mb_rate_control[0] == 1)
1702                 priv_state->lcu_brc_enabled = 1;
1703             else
1704                 priv_state->lcu_brc_enabled = 0;
1705
1706             if (brc_method == HEVC_BRC_CBR) {
1707                 priv_state->target_bit_rate_in_kbs =
1708                     ALIGN(encoder_context->brc.bits_per_second[0], HEVC_BRC_KBPS) /
1709                     HEVC_BRC_KBPS;
1710                 priv_state->max_bit_rate_in_kbs = priv_state->target_bit_rate_in_kbs;
1711                 priv_state->min_bit_rate_in_kbs = priv_state->target_bit_rate_in_kbs;
1712             } else {
1713                 if (encoder_context->brc.target_percentage[0] > HEVC_BRC_MIN_TARGET_PERCENTAGE) {
1714                     priv_state->target_bit_rate_in_kbs =
1715                         ALIGN(encoder_context->brc.bits_per_second[0], HEVC_BRC_KBPS) /
1716                         HEVC_BRC_KBPS;
1717                     priv_state->max_bit_rate_in_kbs = priv_state->target_bit_rate_in_kbs;
1718                     priv_state->min_bit_rate_in_kbs = priv_state->target_bit_rate_in_kbs *
1719                                                       (2 * encoder_context->brc.target_percentage[0] - 100) /
1720                                                       100;
1721                     priv_state->target_bit_rate_in_kbs = priv_state->max_bit_rate_in_kbs *
1722                                                          encoder_context->brc.target_percentage[0] / 100;
1723
1724                     brc_reset = 1;
1725                 }
1726             }
1727
1728             if (encoder_context->brc.framerate[0].den)
1729                 priv_state->frames_per_100s = encoder_context->brc.framerate[0].num * 100 /
1730                                               encoder_context->brc.framerate[0].den;
1731
1732             priv_state->init_vbv_buffer_fullness_in_bit =
1733                 encoder_context->brc.hrd_initial_buffer_fullness;
1734             priv_state->vbv_buffer_size_in_bit =
1735                 encoder_context->brc.hrd_buffer_size;
1736         }
1737
1738         priv_state->brc_method = brc_method;
1739         generic_state->brc_need_reset = brc_reset;
1740         encoder_context->brc.need_reset = 0;
1741     }
1742 }
1743
1744 static void
1745 gen9_hevc_brc_init_rest_set_curbe(VADriverContextP ctx,
1746                                   struct encode_state *encode_state,
1747                                   struct intel_encoder_context *encoder_context,
1748                                   struct i965_gpe_context *gpe_context,
1749                                   int reset)
1750 {
1751     struct encoder_vme_mfc_context *vme_context = NULL;
1752     struct gen9_hevc_encoder_state *priv_state = NULL;
1753     gen9_hevc_brc_initreset_curbe_data *cmd = NULL;
1754     VAEncSequenceParameterBufferHEVC *seq_param = NULL;
1755     double input_bits_per_frame = 0;
1756     double bps_ratio = 0;
1757
1758     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
1759     priv_state = (struct gen9_hevc_encoder_state *)vme_context->private_enc_state;
1760     seq_param = (VAEncSequenceParameterBufferHEVC *)encode_state->seq_param_ext->buffer;
1761
1762     cmd = i965_gpe_context_map_curbe(gpe_context);
1763     if (!cmd)
1764         return;
1765
1766     memcpy((void *)cmd, GEN9_HEVC_BRCINIT_CURBE_DATA, sizeof(GEN9_HEVC_BRCINIT_CURBE_DATA));
1767
1768     cmd->dw0.profile_level_max_frame = gen9_hevc_get_profile_level_max_frame(seq_param,
1769                                                                              priv_state->user_max_frame_size,
1770                                                                              priv_state->frames_per_100s);
1771     cmd->dw1.init_buf_full = priv_state->init_vbv_buffer_fullness_in_bit;
1772     cmd->dw2.buf_size = priv_state->vbv_buffer_size_in_bit;
1773     cmd->dw3.targe_bit_rate = priv_state->target_bit_rate_in_kbs * HEVC_BRC_KBPS;
1774     cmd->dw4.maximum_bit_rate = priv_state->max_bit_rate_in_kbs * HEVC_BRC_KBPS;
1775
1776     cmd->dw9.frame_width = priv_state->picture_width;
1777     cmd->dw10.frame_height = priv_state->picture_height;
1778     cmd->dw12.number_slice = encode_state->num_slice_params_ext;
1779     cmd->dw6.frame_rate_m = priv_state->frames_per_100s;
1780     cmd->dw7.frame_rate_d = 100;
1781     cmd->dw8.brc_flag = 0;
1782     cmd->dw8.brc_flag |= (priv_state->lcu_brc_enabled) ? 0 : HEVC_BRCINIT_DISABLE_MBBRC;
1783     cmd->dw25.acqp_buffer = 1;
1784
1785     if (priv_state->brc_method == HEVC_BRC_CBR) {
1786         cmd->dw4.maximum_bit_rate = cmd->dw3.targe_bit_rate;
1787         cmd->dw8.brc_flag |= HEVC_BRCINIT_ISCBR;
1788     } else if (priv_state->brc_method == HEVC_BRC_VBR) {
1789         if (cmd->dw4.maximum_bit_rate < cmd->dw3.targe_bit_rate)
1790             cmd->dw4.maximum_bit_rate = cmd->dw3.targe_bit_rate * 2;
1791         cmd->dw8.brc_flag |= HEVC_BRCINIT_ISVBR;
1792     } else if (priv_state->brc_method == HEVC_BRC_AVBR) {
1793         cmd->dw4.maximum_bit_rate = priv_state->target_bit_rate_in_kbs * HEVC_BRC_KBPS;
1794         cmd->dw8.brc_flag |= HEVC_BRCINIT_ISAVBR;
1795     } else if (priv_state->brc_method == HEVC_BRC_ICQ) {
1796         cmd->dw25.acqp_buffer = priv_state->crf_quality_factor;
1797         cmd->dw8.brc_flag |= HEVC_BRCINIT_ISICQ;
1798     } else if (priv_state->brc_method == HEVC_BRC_VCM) {
1799         cmd->dw4.maximum_bit_rate = priv_state->target_bit_rate_in_kbs * HEVC_BRC_KBPS;
1800         cmd->dw8.brc_flag |= HEVC_BRCINIT_ISVCM;
1801     }
1802
1803     if (priv_state->num_b_in_gop[1] ||
1804         priv_state->num_b_in_gop[2]) {
1805         cmd->dw8.brc_param_a = priv_state->gop_size / priv_state->gop_ref_dist;
1806         cmd->dw9.brc_param_b = cmd->dw8.brc_param_a;
1807         cmd->dw13.brc_param_c = cmd->dw8.brc_param_a * 2;
1808         cmd->dw14.brc_param_d = priv_state->gop_size - cmd->dw8.brc_param_a -
1809                                 cmd->dw9.brc_param_b - cmd->dw13.brc_param_c;
1810
1811         if (!priv_state->num_b_in_gop[2])
1812             cmd->dw14.max_brc_level = 3;
1813         else
1814             cmd->dw14.max_brc_level = 4;
1815     } else {
1816         cmd->dw14.max_brc_level = 1;
1817         cmd->dw8.brc_param_a = priv_state->gop_ref_dist ? (priv_state->gop_size - 1) / priv_state->gop_ref_dist : 0;
1818         cmd->dw9.brc_param_b = priv_state->gop_size - 1 - cmd->dw8.brc_param_a;
1819     }
1820
1821     cmd->dw10.avbr_accuracy = GEN9_HEVC_AVBR_ACCURACY;
1822     cmd->dw11.avbr_convergence = GEN9_HEVC_AVBR_CONVERGENCE;
1823
1824     input_bits_per_frame = (double)(cmd->dw4.maximum_bit_rate) *
1825                            (double)(cmd->dw7.frame_rate_d) /
1826                            (double)(cmd->dw6.frame_rate_m);
1827     if (cmd->dw2.buf_size < (unsigned int)input_bits_per_frame * 4)
1828         cmd->dw2.buf_size = (unsigned int)input_bits_per_frame * 4;
1829
1830     if (cmd->dw1.init_buf_full == 0)
1831         cmd->dw1.init_buf_full = cmd->dw2.buf_size * 7 / 8;
1832
1833     if (cmd->dw1.init_buf_full < (unsigned int)(input_bits_per_frame * 2))
1834         cmd->dw1.init_buf_full = (unsigned int)(input_bits_per_frame * 2);
1835
1836     if (cmd->dw1.init_buf_full > cmd->dw2.buf_size)
1837         cmd->dw1.init_buf_full = cmd->dw2.buf_size;
1838
1839     if (priv_state->brc_method == HEVC_BRC_AVBR) {
1840         cmd->dw2.buf_size = priv_state->target_bit_rate_in_kbs * 2 * HEVC_BRC_KBPS;
1841         cmd->dw1.init_buf_full = (unsigned int)(cmd->dw2.buf_size * 3 / 4);
1842     }
1843
1844     bps_ratio = input_bits_per_frame / (cmd->dw2.buf_size / 30);
1845     bps_ratio = (bps_ratio < 0.1) ? 0.1 : (bps_ratio > 3.5) ? 3.5 : bps_ratio;
1846
1847     cmd->dw19.deviation_threshold0_pbframe = (unsigned int)(-50 * pow(0.90, bps_ratio));
1848     cmd->dw19.deviation_threshold1_pbframe = (unsigned int)(-50 * pow(0.66, bps_ratio));
1849     cmd->dw19.deviation_threshold2_pbframe = (unsigned int)(-50 * pow(0.46, bps_ratio));
1850     cmd->dw19.deviation_threshold3_pbframe = (unsigned int)(-50 * pow(0.3, bps_ratio));
1851
1852     cmd->dw20.deviation_threshold4_pbframe = (unsigned int)(50 *  pow(0.3, bps_ratio));
1853     cmd->dw20.deviation_threshold5_pbframe = (unsigned int)(50 * pow(0.46, bps_ratio));
1854     cmd->dw20.deviation_threshold6_pbframe = (unsigned int)(50 * pow(0.7,  bps_ratio));
1855     cmd->dw20.deviation_threshold7_pbframe = (unsigned int)(50 * pow(0.9,  bps_ratio));
1856
1857     cmd->dw21.deviation_threshold0_vbr_control = (unsigned int)(-50 * pow(0.9, bps_ratio));
1858     cmd->dw21.deviation_threshold1_vbr_control = (unsigned int)(-50 * pow(0.7, bps_ratio));
1859     cmd->dw21.deviation_threshold2_vbr_control = (unsigned int)(-50 * pow(0.5, bps_ratio));
1860     cmd->dw21.deviation_threshold3_vbr_control = (unsigned int)(-50 * pow(0.3, bps_ratio));
1861
1862     cmd->dw22.deviation_threshold4_vbr_control = (unsigned int)(100 * pow(0.4, bps_ratio));
1863     cmd->dw22.deviation_threshold5_vbr_control = (unsigned int)(100 * pow(0.5, bps_ratio));
1864     cmd->dw22.deviation_threshold6_vbr_control = (unsigned int)(100 * pow(0.75, bps_ratio));
1865     cmd->dw22.deviation_threshold7_vbr_control = (unsigned int)(100 * pow(0.9, bps_ratio));
1866
1867     cmd->dw23.deviation_threshold0_iframe = (unsigned int)(-50 * pow(0.8, bps_ratio));
1868     cmd->dw23.deviation_threshold1_iframe = (unsigned int)(-50 * pow(0.6, bps_ratio));
1869     cmd->dw23.deviation_threshold2_iframe = (unsigned int)(-50 * pow(0.34, bps_ratio));
1870     cmd->dw23.deviation_threshold3_iframe = (unsigned int)(-50 * pow(0.2, bps_ratio));
1871
1872     cmd->dw24.deviation_threshold4_iframe = (unsigned int)(50 * pow(0.2,  bps_ratio));
1873     cmd->dw24.deviation_threshold5_iframe = (unsigned int)(50 * pow(0.4,  bps_ratio));
1874     cmd->dw24.deviation_threshold6_iframe = (unsigned int)(50 * pow(0.66, bps_ratio));
1875     cmd->dw24.deviation_threshold7_iframe = (unsigned int)(50 * pow(0.9,  bps_ratio));
1876
1877     if (!reset)
1878         priv_state->brc_init_current_target_buf_full_in_bits = cmd->dw1.init_buf_full;
1879
1880     priv_state->brc_init_reset_buf_size_in_bits = (double)cmd->dw2.buf_size;
1881     priv_state->brc_init_reset_input_bits_per_frame = input_bits_per_frame;
1882
1883     i965_gpe_context_unmap_curbe(gpe_context);
1884 }
1885
1886 static void
1887 gen9_hevc_brc_init_rest_set_surfaces(VADriverContextP ctx,
1888                                      struct intel_encoder_context *encoder_context,
1889                                      struct i965_gpe_context *gpe_context)
1890 {
1891     struct encoder_vme_mfc_context *vme_context = NULL;
1892     struct gen9_hevc_encoder_context *priv_ctx = NULL;
1893     int bti_idx = 0;
1894
1895     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
1896     priv_ctx = (struct gen9_hevc_encoder_context *)vme_context->private_enc_ctx;
1897
1898     gen9_hevc_set_gpe_1d_surface(ctx, priv_ctx, gpe_context,
1899                                  HEVC_ENC_SURFACE_BRC_HISTORY, bti_idx++,
1900                                  0, 0, 0, NULL, NULL);
1901
1902     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
1903                                  HEVC_ENC_SURFACE_BRC_ME_DIST, bti_idx++,
1904                                  0, 1, I965_SURFACEFORMAT_R8_UNORM,
1905                                  &priv_ctx->res_brc_me_dist_buffer, NULL);
1906 }
1907
1908 static void
1909 gen9_hevc_brc_init_reset(VADriverContextP ctx,
1910                          struct encode_state *encode_state,
1911                          struct intel_encoder_context *encoder_context,
1912                          int reset)
1913 {
1914     struct encoder_vme_mfc_context *vme_context = NULL;
1915     struct gen9_hevc_encoder_context *priv_ctx = NULL;
1916
1917     struct i965_gpe_context *gpe_context = NULL;
1918     struct gpe_media_object_parameter param;
1919     int media_state = HEVC_ENC_MEDIA_STATE_BRC_INIT_RESET;
1920     int gpe_idx = reset ? HEVC_BRC_RESET_IDX : HEVC_BRC_INIT_IDX;
1921
1922     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
1923     priv_ctx = (struct gen9_hevc_encoder_context *)vme_context->private_enc_ctx;
1924     gpe_context = &priv_ctx->brc_context.gpe_contexts[gpe_idx];
1925
1926     gen8_gpe_context_init(ctx, gpe_context);
1927     gen9_gpe_reset_binding_table(ctx, gpe_context);
1928     gen9_hevc_brc_init_rest_set_curbe(ctx, encode_state, encoder_context, gpe_context,
1929                                       reset);
1930     gen9_hevc_brc_init_rest_set_surfaces(ctx, encoder_context, gpe_context);
1931     gen8_gpe_setup_interface_data(ctx, gpe_context);
1932
1933     memset(&param, 0, sizeof(param));
1934     gen9_hevc_run_object(ctx, encoder_context, gpe_context, &param,
1935                          media_state);
1936 }
1937
1938 static void
1939 gen9_hevc_brc_intra_dist_set_curbe(VADriverContextP ctx,
1940                                    struct encode_state *encode_state,
1941                                    struct intel_encoder_context *encoder_context,
1942                                    struct i965_gpe_context *gpe_context)
1943 {
1944     struct encoder_vme_mfc_context *vme_context = NULL;
1945     struct gen9_hevc_encoder_state *priv_state = NULL;
1946     gen9_hevc_brc_coarse_intra_curbe_data *cmd = NULL;
1947     int bti_idx = 0;
1948
1949     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
1950     priv_state = (struct gen9_hevc_encoder_state *)vme_context->private_enc_state;
1951
1952     cmd = i965_gpe_context_map_curbe(gpe_context);
1953     if (!cmd)
1954         return;
1955
1956     memset((void *)cmd, 0, sizeof(*cmd));
1957
1958     cmd->dw0.picture_width_in_luma_samples = priv_state->frame_width_4x;
1959     cmd->dw0.picture_height_in_luma_samples = priv_state->frame_height_4x;
1960
1961     cmd->dw1.inter_sad = 2;
1962     cmd->dw1.intra_sad = 2;
1963
1964     cmd->dw8.bti_src_y4 = bti_idx++;
1965     cmd->dw9.bti_intra_dist = bti_idx++;
1966     cmd->dw10.bti_vme_intra = bti_idx++;
1967
1968     i965_gpe_context_unmap_curbe(gpe_context);
1969 }
1970
1971 static void
1972 gen9_hevc_brc_intra_dist_set_surfaces(VADriverContextP ctx,
1973                                       struct encode_state *encode_state,
1974                                       struct intel_encoder_context *encoder_context,
1975                                       struct i965_gpe_context *gpe_context)
1976 {
1977     struct encoder_vme_mfc_context *vme_context = NULL;
1978     struct gen9_hevc_encoder_context *priv_ctx = NULL;
1979     struct object_surface *obj_surface;
1980     struct gen9_hevc_surface_priv *surface_priv;
1981     int bti_idx = 0;
1982
1983     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
1984     priv_ctx = (struct gen9_hevc_encoder_context *)vme_context->private_enc_ctx;
1985
1986     obj_surface = encode_state->reconstructed_object;
1987     surface_priv = (struct gen9_hevc_surface_priv *)(obj_surface->private_data);
1988     obj_surface = surface_priv->scaled_surface_obj[HEVC_SCALED_SURF_4X_ID];
1989
1990     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
1991                                  HEVC_ENC_SURFACE_Y_4X, bti_idx++,
1992                                  0, 1, I965_SURFACEFORMAT_R8_UNORM,
1993                                  NULL, obj_surface);
1994
1995     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
1996                                  HEVC_ENC_SURFACE_BRC_ME_DIST, bti_idx++,
1997                                  0, 1, I965_SURFACEFORMAT_R8_UNORM,
1998                                  &priv_ctx->res_brc_intra_dist_buffer, NULL);
1999
2000     gen9_hevc_set_gpe_adv_surface(ctx, priv_ctx, gpe_context,
2001                                   HEVC_ENC_SURFACE_Y_4X_VME, bti_idx++,
2002                                   obj_surface);
2003 }
2004
2005 static void
2006 gen9_hevc_brc_intra_dist(VADriverContextP ctx,
2007                          struct encode_state *encode_state,
2008                          struct intel_encoder_context *encoder_context)
2009 {
2010     struct encoder_vme_mfc_context *vme_context = NULL;
2011     struct gen9_hevc_encoder_context *priv_ctx = NULL;
2012     struct gen9_hevc_encoder_state *priv_state = NULL;
2013     struct i965_gpe_context *gpe_context = NULL;
2014     struct gpe_media_object_walker_parameter param;
2015     struct hevc_enc_kernel_walker_parameter hevc_walker_param;
2016     int media_state = HEVC_ENC_MEDIA_STATE_ENC_I_FRAME_DIST;
2017     int gpe_idx = HEVC_BRC_COARSE_INTRA_IDX;
2018
2019     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
2020     priv_ctx = (struct gen9_hevc_encoder_context *)vme_context->private_enc_ctx;
2021     priv_state = (struct gen9_hevc_encoder_state *)vme_context->private_enc_state;
2022     gpe_context = &priv_ctx->brc_context.gpe_contexts[gpe_idx];
2023
2024     gen8_gpe_context_init(ctx, gpe_context);
2025     gen9_gpe_reset_binding_table(ctx, gpe_context);
2026     gen9_hevc_brc_intra_dist_set_curbe(ctx, encode_state, encoder_context, gpe_context);
2027     gen9_hevc_brc_intra_dist_set_surfaces(ctx, encode_state, encoder_context, gpe_context);
2028     gen8_gpe_setup_interface_data(ctx, gpe_context);
2029
2030     memset((void *)&hevc_walker_param, 0, sizeof(hevc_walker_param));
2031     hevc_walker_param.resolution_x = priv_state->downscaled_width_4x_in_mb;
2032     hevc_walker_param.resolution_y = priv_state->downscaled_height_4x_in_mb;
2033     hevc_walker_param.no_dependency = 1;
2034     gen9_hevc_init_object_walker(&hevc_walker_param, &param);
2035     gen9_hevc_run_object_walker(ctx, encoder_context, gpe_context, &param,
2036                                 media_state);
2037 }
2038
2039 static GEN9_HEVC_BRC_UPDATE_FRAME_TYPE gen9_hevc_get_brc_frame_type(unsigned int pic_type,
2040                                                                     int low_delay)
2041 {
2042     if (pic_type == HEVC_SLICE_I)
2043         return HEVC_BRC_FTYPE_I;
2044     else if (pic_type == HEVC_SLICE_P)
2045         return HEVC_BRC_FTYPE_P_OR_LB;
2046     else
2047         return low_delay ? HEVC_BRC_FTYPE_P_OR_LB : HEVC_BRC_FTYPE_B;
2048 }
2049
2050 static void
2051 gen9_hevc_brc_update_set_roi_curbe(VADriverContextP ctx,
2052                                    struct encode_state *encode_state,
2053                                    struct intel_encoder_context *encoder_context,
2054                                    gen9_hevc_brc_udpate_curbe_data *cmd)
2055 {
2056     struct encoder_vme_mfc_context *vme_context = NULL;
2057     struct generic_enc_codec_state *generic_state = NULL;
2058     struct gen9_hevc_encoder_state *priv_state = NULL;
2059     struct intel_roi *roi_par = NULL;
2060     unsigned int roi_size = 0, roi_ratio = 0;
2061     int i = 0;
2062
2063     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
2064     generic_state = (struct generic_enc_codec_state *)vme_context->generic_enc_state;
2065     priv_state = (struct gen9_hevc_encoder_state *)vme_context->private_enc_state;
2066
2067     cmd->dw6.cqp_value = 0;
2068     cmd->dw6.roi_flag = 1 | (generic_state->brc_enabled << 1) |
2069                         (priv_state->video_surveillance_flag << 2);
2070
2071     for (i = 0; i < priv_state->num_roi; i++) {
2072         roi_par = &priv_state->roi[i];
2073         roi_size += abs(roi_par->right - roi_par->left) *
2074                     abs(roi_par->bottom - roi_par->top) * 256;
2075     }
2076
2077     if (roi_size)
2078         roi_ratio = MIN(2 * (priv_state->width_in_mb * priv_state->height_in_mb * 256 / roi_size - 1),
2079                         51);
2080
2081     cmd->dw6.roi_ratio = roi_ratio;
2082     cmd->dw7.frame_width_in_lcu = priv_state->frame_width_in_max_lcu;
2083
2084     if (!generic_state->brc_enabled) {
2085         VAEncPictureParameterBufferHEVC *pic_param = NULL;
2086         VAEncSliceParameterBufferHEVC *slice_param = NULL;
2087
2088         pic_param = (VAEncPictureParameterBufferHEVC *)encode_state->pic_param_ext->buffer;
2089         slice_param = (VAEncSliceParameterBufferHEVC *)encode_state->slice_params_ext[0]->buffer;
2090
2091         cmd->dw1.frame_number = priv_state->frame_number;
2092         cmd->dw6.cqp_value = pic_param->pic_init_qp + slice_param->slice_qp_delta;
2093         cmd->dw5.curr_frame_type = gen9_hevc_get_brc_frame_type(priv_state->picture_coding_type,
2094                                                                 priv_state->low_delay);
2095     }
2096 }
2097
2098 static void
2099 gen9_hevc_brc_update_lcu_based_set_roi_parameters(VADriverContextP ctx,
2100                                                   struct encode_state *encode_state,
2101                                                   struct intel_encoder_context *encoder_context,
2102                                                   struct i965_gpe_context *gpe_context)
2103 {
2104     struct encoder_vme_mfc_context *vme_context = NULL;
2105     struct gen9_hevc_encoder_context *priv_ctx = NULL;
2106     struct generic_enc_codec_state *generic_state = NULL;
2107     struct gen9_hevc_encoder_state *priv_state = NULL;
2108     struct intel_roi *roi_par = NULL;
2109     unsigned int width_in_mb_aligned = 0;
2110     unsigned int roi_level, qp_delta;
2111     unsigned int mb_num = 0;
2112     unsigned int *pdata = NULL;
2113     unsigned int out_data = 0;
2114     int i = 0;
2115
2116     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
2117     priv_ctx = (struct gen9_hevc_encoder_context *)vme_context->private_enc_ctx;
2118     generic_state = (struct generic_enc_codec_state *)vme_context->generic_enc_state;
2119     priv_state = (struct gen9_hevc_encoder_state *)vme_context->private_enc_state;
2120
2121     width_in_mb_aligned = ALIGN(priv_state->width_in_mb * 4, 64);
2122     mb_num = priv_state->width_in_mb * priv_state->height_in_mb;
2123
2124     pdata = i965_map_gpe_resource(&priv_ctx->res_roi_buffer);
2125     if (!pdata)
2126         return;
2127
2128     for (i = 0 ; i < mb_num; i++) {
2129         int cur_mb_y = i / priv_state->width_in_mb;
2130         int cur_mb_x = i - cur_mb_y * priv_state->width_in_mb;
2131         int roi_idx = 0;
2132
2133         if (cur_mb_x > 1 && cur_mb_x < (priv_state->width_in_mb - 2) &&
2134             cur_mb_y > 1 && cur_mb_y < (priv_state->width_in_mb - 2)) {
2135             out_data = 0;
2136
2137             for (roi_idx = (priv_state->num_roi - 1); roi_idx >= 0; roi_idx--) {
2138                 roi_par = &priv_state->roi[roi_idx];
2139
2140                 roi_level = qp_delta = 0;
2141                 if (generic_state->brc_enabled)
2142                     roi_level = roi_par->value * 5;
2143                 else
2144                     qp_delta = roi_par->value;
2145
2146 #if 0
2147                 if (roi_level == 0)
2148                     continue;
2149 #endif
2150
2151                 if ((cur_mb_x >= roi_par->left) &&
2152                     (cur_mb_x < roi_par->right) &&
2153                     (cur_mb_y >= roi_par->top) &&
2154                     (cur_mb_y < roi_par->bottom))
2155                     out_data = 15 | (((roi_level) & 0xFF) << 8) | ((qp_delta & 0xFF) << 16);
2156                 else if ((cur_mb_x >= roi_par->left - 1) &&
2157                          (cur_mb_x < roi_par->right + 1) &&
2158                          (cur_mb_y >= roi_par->top - 1) &&
2159                          (cur_mb_y < roi_par->top + 1))
2160                     out_data = 14 | (((roi_level) & 0xFF) << 8) | ((qp_delta & 0xFF) << 16);
2161                 else if ((cur_mb_x >= roi_par->left - 2) &&
2162                          (cur_mb_x < roi_par->right + 2) &&
2163                          (cur_mb_y >= roi_par->top - 2) &&
2164                          (cur_mb_y < roi_par->top + 2))
2165                     out_data = 13 | (((roi_level) & 0xFF) << 8) | ((qp_delta & 0xFF) << 16);
2166                 else if ((cur_mb_x >= roi_par->left - 3) &&
2167                          (cur_mb_x < roi_par->right + 3) &&
2168                          (cur_mb_y >= roi_par->top - 3) &&
2169                          (cur_mb_y < roi_par->top + 3)) {
2170                     if (qp_delta >= 3)
2171                         out_data = 12 | (((roi_level) & 0xFF) << 8) | ((qp_delta & 0xFF) << 16);
2172                 }
2173             }
2174             pdata[(cur_mb_y * (width_in_mb_aligned >> 2)) + cur_mb_x] = out_data;
2175         }
2176     }
2177
2178     i965_unmap_gpe_resource(&priv_ctx->res_roi_buffer);
2179 }
2180
2181 static void
2182 gen9_hevc_brc_update_lcu_based_set_curbe(VADriverContextP ctx,
2183                                          struct encode_state *encode_state,
2184                                          struct intel_encoder_context *encoder_context,
2185                                          struct i965_gpe_context *gpe_context)
2186 {
2187     struct encoder_vme_mfc_context *vme_context = NULL;
2188     struct gen9_hevc_encoder_context *priv_ctx = NULL;
2189     struct generic_enc_codec_state *generic_state = NULL;
2190     struct gen9_hevc_encoder_state *priv_state = NULL;
2191     gen9_hevc_brc_udpate_curbe_data *cmd = NULL, *frame_cmd = NULL;
2192
2193     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
2194     priv_ctx = (struct gen9_hevc_encoder_context *)vme_context->private_enc_ctx;
2195     generic_state = (struct generic_enc_codec_state *)vme_context->generic_enc_state;
2196     priv_state = (struct gen9_hevc_encoder_state *)vme_context->private_enc_state;
2197
2198     cmd = i965_gpe_context_map_curbe(gpe_context);
2199     if (!cmd)
2200         return;
2201
2202     if (generic_state->brc_enabled) {
2203         frame_cmd = i965_gpe_context_map_curbe(&priv_ctx->brc_context.gpe_contexts[HEVC_BRC_FRAME_UPDATE_IDX]);
2204
2205         if (!frame_cmd)
2206             return;
2207
2208         memcpy((void *)cmd, (void *)frame_cmd, sizeof(*cmd));
2209
2210         i965_gpe_context_unmap_curbe(&priv_ctx->brc_context.gpe_contexts[HEVC_BRC_FRAME_UPDATE_IDX]);
2211     } else {
2212         memcpy((void *)cmd, GEN9_HEVC_BRCUPDATE_CURBE_DATA,
2213                sizeof(GEN9_HEVC_BRCUPDATE_CURBE_DATA));
2214
2215         if (priv_state->num_roi)
2216             gen9_hevc_brc_update_set_roi_curbe(ctx, encode_state, encoder_context, cmd);
2217     }
2218
2219     i965_gpe_context_unmap_curbe(gpe_context);
2220 }
2221
2222 static void
2223 gen9_hevc_brc_update_lcu_based_set_surfaces(VADriverContextP ctx,
2224                                             struct encode_state *encode_state,
2225                                             struct intel_encoder_context *encoder_context,
2226                                             struct i965_gpe_context *gpe_context)
2227 {
2228     struct encoder_vme_mfc_context *vme_context = NULL;
2229     struct gen9_hevc_encoder_context *priv_ctx = NULL;
2230     struct gen9_hevc_encoder_state *priv_state = NULL;
2231     int bti_idx = 0;
2232
2233     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
2234     priv_ctx = (struct gen9_hevc_encoder_context *)vme_context->private_enc_ctx;
2235     priv_state = (struct gen9_hevc_encoder_state *)vme_context->private_enc_state;
2236
2237     gen9_hevc_set_gpe_1d_surface(ctx, priv_ctx, gpe_context,
2238                                  HEVC_ENC_SURFACE_BRC_HISTORY, bti_idx++,
2239                                  0, 0, 0, NULL, NULL);
2240
2241     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
2242                                  HEVC_ENC_SURFACE_BRC_ME_DIST, bti_idx++,
2243                                  0, 1, I965_SURFACEFORMAT_R8_UNORM,
2244                                  priv_state->picture_coding_type == HEVC_SLICE_I ?
2245                                  &priv_ctx->res_brc_intra_dist_buffer :
2246                                  &priv_ctx->res_brc_me_dist_buffer,
2247                                  NULL);
2248
2249     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
2250                                  HEVC_ENC_SURFACE_BRC_ME_DIST, bti_idx++,
2251                                  0, 1, I965_SURFACEFORMAT_R8_UNORM,
2252                                  &priv_ctx->res_brc_intra_dist_buffer, NULL);
2253
2254     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
2255                                  HEVC_ENC_SURFACE_HME_MVP, bti_idx++,
2256                                  0, 1, I965_SURFACEFORMAT_R8_UNORM,
2257                                  NULL, NULL);
2258
2259     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
2260                                  HEVC_ENC_SURFACE_LCU_QP, bti_idx++,
2261                                  0, 1, I965_SURFACEFORMAT_R8_UNORM,
2262                                  NULL, NULL);
2263
2264     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
2265                                  HEVC_ENC_SURFACE_ROI, bti_idx++,
2266                                  0, 1, I965_SURFACEFORMAT_R8_UNORM,
2267                                  NULL, NULL);
2268 }
2269
2270 static void
2271 gen9_hevc_brc_update_lcu_based(VADriverContextP ctx,
2272                                struct encode_state *encode_state,
2273                                struct intel_encoder_context *encoder_context)
2274 {
2275     struct encoder_vme_mfc_context *vme_context = NULL;
2276     struct gen9_hevc_encoder_context *priv_ctx = NULL;
2277     struct gen9_hevc_encoder_state *priv_state = NULL;
2278     struct i965_gpe_context *gpe_context = NULL;
2279     struct gpe_media_object_walker_parameter param;
2280     struct hevc_enc_kernel_walker_parameter hevc_walker_param;
2281     int media_state = HEVC_ENC_MEDIA_STATE_HEVC_BRC_LCU_UPDATE;
2282     int gpe_idx = HEVC_BRC_LCU_UPDATE_IDX;
2283
2284     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
2285     priv_ctx = (struct gen9_hevc_encoder_context *)vme_context->private_enc_ctx;
2286     priv_state = (struct gen9_hevc_encoder_state *)vme_context->private_enc_state;
2287     gpe_context = &priv_ctx->brc_context.gpe_contexts[gpe_idx];
2288
2289     gen8_gpe_context_init(ctx, gpe_context);
2290     gen9_gpe_reset_binding_table(ctx, gpe_context);
2291
2292     if (priv_state->num_roi)
2293         gen9_hevc_brc_update_lcu_based_set_roi_parameters(ctx, encode_state, encoder_context,
2294                                                           gpe_context);
2295
2296     gen9_hevc_brc_update_lcu_based_set_curbe(ctx, encode_state, encoder_context, gpe_context);
2297     gen9_hevc_brc_update_lcu_based_set_surfaces(ctx, encode_state, encoder_context, gpe_context);
2298     gen8_gpe_setup_interface_data(ctx, gpe_context);
2299
2300     memset((void *)&hevc_walker_param, 0, sizeof(hevc_walker_param));
2301     hevc_walker_param.resolution_x = ALIGN(priv_state->picture_width, 128) >> 7;
2302     hevc_walker_param.resolution_y = ALIGN(priv_state->picture_height, 128) >> 7;
2303     hevc_walker_param.no_dependency = 1;
2304     gen9_hevc_init_object_walker(&hevc_walker_param, &param);
2305     gen9_hevc_run_object_walker(ctx, encoder_context, gpe_context, &param,
2306                                 media_state);
2307 }
2308
2309 static void
2310 gen9_hevc_add_pic_state(VADriverContextP ctx,
2311                         struct encode_state *encode_state,
2312                         struct intel_encoder_context *encoder_context,
2313                         struct i965_gpe_resource *pic_state_ptr,
2314                         int pic_state_offset,
2315                         int brc_update)
2316 {
2317     struct i965_driver_data *i965 = i965_driver_data(ctx);
2318     struct encoder_vme_mfc_context *pak_context = NULL;
2319     struct gen9_hevc_encoder_state *priv_state = NULL;
2320     VAEncPictureParameterBufferHEVC *pic_param = NULL;
2321     VAEncSequenceParameterBufferHEVC *seq_param = NULL;
2322     unsigned int tmp_data[31], *cmd_ptr = NULL;
2323     int cmd_size = 0;
2324
2325     pak_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
2326     priv_state = (struct gen9_hevc_encoder_state *)pak_context->private_enc_state;
2327
2328     pic_param = (VAEncPictureParameterBufferHEVC *)encode_state->pic_param_ext->buffer;
2329     seq_param = (VAEncSequenceParameterBufferHEVC *)encode_state->seq_param_ext->buffer;
2330
2331     cmd_ptr = tmp_data;
2332     cmd_size = (IS_KBL(i965->intel.device_info) || IS_GLK(i965->intel.device_info)) ? 31 : 19;
2333     memset((void *)tmp_data, 0, 4 * cmd_size);
2334
2335     if (IS_KBL(i965->intel.device_info) || IS_GLK(i965->intel.device_info))
2336         *cmd_ptr++ = HCP_PIC_STATE | (31 - 2);
2337     else
2338         *cmd_ptr++ = HCP_PIC_STATE | (19 - 2);
2339
2340     *cmd_ptr++ = (priv_state->height_in_cu - 1) << 16 |
2341                  0 << 14 |
2342                  (priv_state->width_in_cu - 1);
2343     *cmd_ptr++ = (seq_param->log2_min_transform_block_size_minus2 +
2344                   seq_param->log2_diff_max_min_transform_block_size) << 6 |
2345                  seq_param->log2_min_transform_block_size_minus2 << 4 |
2346                  (seq_param->log2_min_luma_coding_block_size_minus3 +
2347                   seq_param->log2_diff_max_min_luma_coding_block_size) << 2 |
2348                  seq_param->log2_min_luma_coding_block_size_minus3;
2349     *cmd_ptr++ = 0;
2350     *cmd_ptr++ = ((IS_KBL(i965->intel.device_info) || IS_GLK(i965->intel.device_info)) ? 1 : 0) << 27 |
2351                  seq_param->seq_fields.bits.strong_intra_smoothing_enabled_flag << 26 |
2352                  pic_param->pic_fields.bits.transquant_bypass_enabled_flag << 25 |
2353                  ((IS_KBL(i965->intel.device_info) || IS_GLK(i965->intel.device_info)) ? 0 : priv_state->ctu_max_bitsize_allowed > 0) << 24 |
2354                  seq_param->seq_fields.bits.amp_enabled_flag << 23 |
2355                  pic_param->pic_fields.bits.transform_skip_enabled_flag << 22 |
2356                  0 << 21 |
2357                  0 << 20 |
2358                  pic_param->pic_fields.bits.weighted_pred_flag << 19 |
2359                  pic_param->pic_fields.bits.weighted_bipred_flag << 18 |
2360                  0 << 17 |
2361                  pic_param->pic_fields.bits.entropy_coding_sync_enabled_flag << 16 |
2362                  0 << 15 |
2363                  pic_param->pic_fields.bits.sign_data_hiding_enabled_flag << 13 |
2364                  pic_param->log2_parallel_merge_level_minus2 << 10 |
2365                  pic_param->pic_fields.bits.constrained_intra_pred_flag << 9 |
2366                  seq_param->seq_fields.bits.pcm_loop_filter_disabled_flag << 8 |
2367                  (pic_param->diff_cu_qp_delta_depth & 0x03) << 6 |
2368                  pic_param->pic_fields.bits.cu_qp_delta_enabled_flag << 5 |
2369                  0 << 4 |
2370                  seq_param->seq_fields.bits.sample_adaptive_offset_enabled_flag << 3 |
2371                  0;
2372     *cmd_ptr++ = seq_param->seq_fields.bits.bit_depth_luma_minus8 << 27 |
2373                  seq_param->seq_fields.bits.bit_depth_chroma_minus8 << 24 |
2374                  ((IS_KBL(i965->intel.device_info) || IS_GLK(i965->intel.device_info)) ? 0 : 7) << 20 |
2375                  ((IS_KBL(i965->intel.device_info) || IS_GLK(i965->intel.device_info)) ? 0 : 7) << 16 |
2376                  seq_param->max_transform_hierarchy_depth_inter << 13 |
2377                  seq_param->max_transform_hierarchy_depth_intra << 10 |
2378                  (pic_param->pps_cr_qp_offset & 0x1f) << 5 |
2379                  (pic_param->pps_cb_qp_offset & 0x1f);
2380
2381     *cmd_ptr++ = 0 << 29 |
2382                  priv_state->ctu_max_bitsize_allowed;
2383     if (brc_update)
2384         *(cmd_ptr - 1) |= 0 << 31 |
2385                           1 << 26 |
2386                           1 << 25 |
2387                           0 << 24 |
2388                           (pic_state_offset ? 1 : 0) << 16;
2389
2390     *cmd_ptr++ = 0 << 31 |
2391                  0;
2392     *cmd_ptr++ = 0 << 31 |
2393                  0;
2394     *cmd_ptr++ = 0 << 16 |
2395                  0;
2396     *cmd_ptr++ = 0;
2397     *cmd_ptr++ = 0;
2398     *cmd_ptr++ = 0;
2399     *cmd_ptr++ = 0;
2400     *cmd_ptr++ = 0;
2401     *cmd_ptr++ = 0;
2402     *cmd_ptr++ = 0;
2403     *cmd_ptr++ = 0;
2404     *cmd_ptr++ = 0 << 30 |
2405                  0;
2406
2407     if (IS_KBL(i965->intel.device_info) || IS_GLK(i965->intel.device_info)) {
2408         int i = 0;
2409
2410         for (i = 0; i < 12; i++)
2411             *cmd_ptr++ = 0;
2412     }
2413
2414     if (pic_state_ptr) {
2415         char *pdata = i965_map_gpe_resource(pic_state_ptr);
2416
2417         if (!pdata)
2418             return;
2419
2420         memcpy(pdata + pic_state_offset, tmp_data, cmd_size * 4);
2421
2422         pdata += pic_state_offset + cmd_size * 4;
2423
2424         *(unsigned int *)pdata++ = MI_BATCH_BUFFER_END;
2425
2426         i965_unmap_gpe_resource(pic_state_ptr);
2427     } else {
2428         struct intel_batchbuffer *batch = encoder_context->base.batch;
2429
2430         BEGIN_BCS_BATCH(batch, cmd_size);
2431
2432         intel_batchbuffer_data(batch, tmp_data, cmd_size * 4);
2433
2434         ADVANCE_BCS_BATCH(batch);
2435     }
2436 }
2437
2438 static void
2439 gen9_hevc_brc_update_set_pic_states(VADriverContextP ctx,
2440                                     struct encode_state *encode_state,
2441                                     struct intel_encoder_context *encoder_context)
2442 {
2443     struct encoder_vme_mfc_context *vme_context = NULL;
2444     struct gen9_hevc_encoder_context *priv_ctx = NULL;
2445     struct generic_enc_codec_state *generic_state = NULL;
2446     struct gen9_hevc_encoder_state *priv_state = NULL;
2447     int i = 0, offset = 0;
2448
2449     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
2450     priv_ctx = (struct gen9_hevc_encoder_context *)vme_context->private_enc_ctx;
2451     generic_state = (struct generic_enc_codec_state *)vme_context->generic_enc_state;
2452     priv_state = (struct gen9_hevc_encoder_state *)vme_context->private_enc_state;
2453
2454     for (i = 0; i < generic_state->num_pak_passes; i++) {
2455         gen9_hevc_add_pic_state(ctx, encode_state, encoder_context,
2456                                 &priv_ctx->res_brc_pic_states_read_buffer,
2457                                 offset, 1);
2458
2459         offset += priv_state->pic_state_size;
2460     }
2461 }
2462
2463 static void
2464 gen9_hevc_brc_update_set_constant(VADriverContextP ctx,
2465                                   struct encode_state *encode_state,
2466                                   struct intel_encoder_context *encoder_context)
2467 {
2468     struct encoder_vme_mfc_context *vme_context = NULL;
2469     struct gen9_hevc_encoder_context *priv_ctx = NULL;
2470     struct gen9_hevc_encoder_state *priv_state = NULL;
2471     unsigned int width, height, size;
2472     unsigned char *pdata = NULL;
2473     int idx = 0;
2474
2475     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
2476     priv_ctx = (struct gen9_hevc_encoder_context *)vme_context->private_enc_ctx;
2477     priv_state = (struct gen9_hevc_encoder_state *)vme_context->private_enc_state;
2478
2479     pdata = i965_map_gpe_resource(&priv_ctx->res_brc_constant_data_buffer);
2480     if (!pdata)
2481         return;
2482
2483     width = ALIGN(GEN9_HEVC_ENC_BRC_CONSTANT_SURFACE_WIDTH, 64);
2484     height = GEN9_HEVC_ENC_BRC_CONSTANT_SURFACE_HEIGHT;
2485     size = width * height;
2486     memset((void *)pdata, 0, size);
2487
2488     memcpy((void *)pdata, GEN9_HEVC_BRCUPDATE_QP_ADJUST, GEN9_HEVC_BRCUPDATE_QP_ADJUST_SIZE);
2489     pdata += GEN9_HEVC_BRCUPDATE_QP_ADJUST_SIZE;
2490
2491     if (priv_state->picture_coding_type == HEVC_SLICE_I)
2492         memset((void *)pdata, 0, GEN9_HEVC_ENC_SKIP_VAL_SIZE);
2493     else {
2494         gen9_hevc_mbenc_b_mb_enc_curbe_data *curbe_cmd = NULL;
2495         int curbe_size = 0;
2496
2497         gen9_hevc_get_b_mbenc_default_curbe(priv_state->tu_mode,
2498                                             priv_state->picture_coding_type,
2499                                             (void **)&curbe_cmd,
2500                                             &curbe_size);
2501
2502         idx = curbe_cmd->dw3.block_based_skip_enable ? 1 : 0;
2503         memcpy((void *)pdata, GEN9_HEVC_ENC_SKIP_THREAD[idx], sizeof(GEN9_HEVC_ENC_SKIP_THREAD[idx]));
2504     }
2505     pdata += GEN9_HEVC_ENC_SKIP_VAL_SIZE;
2506
2507     memcpy((void *)pdata, GEN9_HEVC_ENC_BRC_LAMBDA_HAAR, sizeof(GEN9_HEVC_ENC_BRC_LAMBDA_HAAR));
2508     pdata += GEN9_HEVC_ENC_BRC_LAMBDA_TABLE_SIZE;
2509
2510     idx = (priv_state->picture_coding_type == HEVC_SLICE_I) ? 0 :
2511           (priv_state->picture_coding_type == HEVC_SLICE_P) ? 1 : 2;
2512     memcpy((void *)pdata, GEN9_HEVC_ENC_BRC_MVCOST_HAAR[idx], sizeof(GEN9_HEVC_ENC_BRC_MVCOST_HAAR[idx]));
2513
2514     i965_unmap_gpe_resource(&priv_ctx->res_brc_constant_data_buffer);
2515 }
2516
2517 static
2518 unsigned int gen9_hevc_get_start_code_offset(unsigned char *ptr,
2519                                              unsigned int size)
2520 {
2521     unsigned int count = 0;
2522
2523     while (count < size && *ptr != 0x01) {
2524         if (*ptr != 0)
2525             break;
2526
2527         count++;
2528         ptr++;
2529     }
2530
2531     return count + 1;
2532 }
2533
2534 static
2535 unsigned int gen9_hevc_get_emulation_num(unsigned char *ptr,
2536                                          unsigned int size)
2537 {
2538     unsigned int emulation_num = 0;
2539     unsigned int header_offset = 0;
2540     unsigned int zero_count = 0;
2541     int i = 0;
2542
2543     header_offset = gen9_hevc_get_start_code_offset(ptr, size);
2544     ptr += header_offset;
2545
2546     for (i = 0 ; i < (size - header_offset); i++, ptr++) {
2547         if (zero_count == 2 && !(*ptr & 0xFC)) {
2548             zero_count = 0;
2549             emulation_num++;
2550         }
2551
2552         if (*ptr == 0x00)
2553             zero_count++;
2554         else
2555             zero_count = 0;
2556     }
2557
2558     return emulation_num;
2559 }
2560
2561 #define HEVC_ENC_START_CODE_NAL_OFFSET                  (2)
2562
2563 static unsigned int
2564 gen9_hevc_get_pic_header_size(struct encode_state *encode_state)
2565 {
2566     VAEncPackedHeaderParameterBuffer *param = NULL;
2567     unsigned int header_begin = 0;
2568     unsigned int accum_size = 0;
2569     unsigned char *header_data = NULL;
2570     unsigned int length_in_bytes = 0;
2571     int packed_type = 0;
2572     int idx = 0, count = 0, idx_offset = 0;
2573     int i = 0, slice_idx = 0, start_index = 0;
2574
2575     for (i = 0; i < 4; i++) {
2576         idx_offset = 0;
2577         switch (i) {
2578         case 0:
2579             packed_type = VAEncPackedHeaderHEVC_VPS;
2580             break;
2581         case 1:
2582             packed_type = VAEncPackedHeaderHEVC_VPS;
2583             idx_offset = 1;
2584             break;
2585         case 2:
2586             packed_type = VAEncPackedHeaderHEVC_PPS;
2587             break;
2588         case 3:
2589             packed_type = VAEncPackedHeaderHEVC_SEI;
2590             break;
2591         default:
2592             break;
2593         }
2594
2595         idx = va_enc_packed_type_to_idx(packed_type) + idx_offset;
2596         if (encode_state->packed_header_data[idx]) {
2597             param = (VAEncPackedHeaderParameterBuffer *)encode_state->packed_header_param[idx]->buffer;
2598             header_data = (unsigned char *)encode_state->packed_header_data[idx]->buffer;
2599             length_in_bytes = (param->bit_length + 7) / 8;
2600
2601             header_begin = gen9_hevc_get_start_code_offset(header_data, length_in_bytes) +
2602                            HEVC_ENC_START_CODE_NAL_OFFSET;
2603
2604             accum_size += length_in_bytes;
2605             if (!param->has_emulation_bytes)
2606                 accum_size += gen9_hevc_get_emulation_num(header_data,
2607                                                           length_in_bytes);
2608         }
2609     }
2610
2611     for (slice_idx = 0; slice_idx < encode_state->num_slice_params_ext; slice_idx++) {
2612         count = encode_state->slice_rawdata_count[slice_idx];
2613         start_index = encode_state->slice_rawdata_index[slice_idx] &
2614                       SLICE_PACKED_DATA_INDEX_MASK;
2615
2616         for (i = 0; i < count; i++) {
2617             param = (VAEncPackedHeaderParameterBuffer *)
2618                     (encode_state->packed_header_params_ext[start_index + i]->buffer);
2619
2620             if (param->type == VAEncPackedHeaderSlice)
2621                 continue;
2622
2623             header_data = (unsigned char *)encode_state->packed_header_data[start_index]->buffer;
2624             length_in_bytes = (param->bit_length + 7) / 8;
2625
2626             accum_size += length_in_bytes;
2627             if (!param->has_emulation_bytes)
2628                 accum_size += gen9_hevc_get_emulation_num(header_data,
2629                                                           length_in_bytes);
2630         }
2631     }
2632
2633     header_begin = MIN(header_begin, accum_size);
2634
2635     return ((accum_size - header_begin) * 8);
2636 }
2637
2638 static void
2639 gen9_hevc_brc_update_set_curbe(VADriverContextP ctx,
2640                                struct encode_state *encode_state,
2641                                struct intel_encoder_context *encoder_context,
2642                                struct i965_gpe_context *gpe_context)
2643 {
2644     struct encoder_vme_mfc_context *vme_context = NULL;
2645     struct generic_enc_codec_state *generic_state = NULL;
2646     struct gen9_hevc_encoder_state *priv_state = NULL;
2647     gen9_hevc_brc_udpate_curbe_data *cmd = NULL;
2648
2649     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
2650     generic_state = (struct generic_enc_codec_state *)vme_context->generic_enc_state;
2651     priv_state = (struct gen9_hevc_encoder_state *)vme_context->private_enc_state;
2652
2653     cmd = i965_gpe_context_map_curbe(gpe_context);
2654     if (!cmd)
2655         return;
2656
2657     memcpy((void *)cmd, GEN9_HEVC_BRCUPDATE_CURBE_DATA,
2658            sizeof(GEN9_HEVC_BRCUPDATE_CURBE_DATA));
2659
2660     cmd->dw5.target_size_flag = 0;
2661     if (priv_state->brc_init_current_target_buf_full_in_bits >
2662         (double)priv_state->brc_init_reset_buf_size_in_bits) {
2663         priv_state->brc_init_current_target_buf_full_in_bits -=
2664             (double)priv_state->brc_init_reset_buf_size_in_bits;
2665         cmd->dw5.target_size_flag = 1;
2666     }
2667
2668     if (priv_state->num_skip_frames) {
2669         cmd->dw6.num_skipped_frames = priv_state->num_skip_frames;
2670         cmd->dw15.size_of_skipped_frames = priv_state->size_skip_frames;
2671
2672         priv_state->brc_init_current_target_buf_full_in_bits +=
2673             priv_state->brc_init_reset_input_bits_per_frame * priv_state->num_skip_frames;
2674     }
2675
2676     cmd->dw0.target_size = (unsigned int)priv_state->brc_init_current_target_buf_full_in_bits;
2677     cmd->dw1.frame_number = priv_state->frame_number;
2678     cmd->dw2.picture_header_size = gen9_hevc_get_pic_header_size(encode_state);
2679
2680     cmd->dw5.brc_flag = 0;
2681     cmd->dw5.curr_frame_type = gen9_hevc_get_brc_frame_type(priv_state->picture_coding_type,
2682                                                             priv_state->low_delay);
2683
2684     cmd->dw5.max_num_paks = generic_state->num_pak_passes;
2685     cmd->dw14.parallel_mode = priv_state->parallel_brc;
2686
2687     priv_state->brc_init_current_target_buf_full_in_bits +=
2688         priv_state->brc_init_reset_input_bits_per_frame;
2689
2690     if (priv_state->brc_method == HEVC_BRC_AVBR) {
2691         cmd->dw3.start_gadj_frame0 = (unsigned int)((10 * GEN9_HEVC_AVBR_CONVERGENCE) / (double)150);
2692         cmd->dw3.start_gadj_frame1 = (unsigned int)((50 * GEN9_HEVC_AVBR_CONVERGENCE) / (double)150);
2693         cmd->dw4.start_gadj_frame2 = (unsigned int)((100 * GEN9_HEVC_AVBR_CONVERGENCE) / (double)150);
2694         cmd->dw4.start_gadj_frame3 = (unsigned int)((150 * GEN9_HEVC_AVBR_CONVERGENCE) / (double)150);
2695         cmd->dw11.g_rate_ratio_threshold0 = (unsigned int)((100 - (GEN9_HEVC_AVBR_ACCURACY / (double)30) * (100 - 40)));
2696         cmd->dw11.g_rate_ratio_threshold1 = (unsigned int)((100 - (GEN9_HEVC_AVBR_ACCURACY / (double)30) * (100 - 75)));
2697         cmd->dw12.g_rate_ratio_threshold2 = (unsigned int)((100 - (GEN9_HEVC_AVBR_ACCURACY / (double)30) * (100 - 97)));
2698         cmd->dw12.g_rate_ratio_threshold3 = (unsigned int)((100 + (GEN9_HEVC_AVBR_ACCURACY / (double)30) * (103 - 100)));
2699         cmd->dw12.g_rate_ratio_threshold4 = (unsigned int)((100 + (GEN9_HEVC_AVBR_ACCURACY / (double)30) * (125 - 100)));
2700         cmd->dw12.g_rate_ratio_threshold5 = (unsigned int)((100 + (GEN9_HEVC_AVBR_ACCURACY / (double)30) * (160 - 100)));
2701     } else {
2702         cmd->dw3.start_gadj_frame0 = 10;
2703         cmd->dw3.start_gadj_frame1 = 50;
2704         cmd->dw4.start_gadj_frame2 = 100;
2705         cmd->dw4.start_gadj_frame3 = 150;
2706         cmd->dw11.g_rate_ratio_threshold0 = 40;
2707         cmd->dw11.g_rate_ratio_threshold1 = 75;
2708         cmd->dw12.g_rate_ratio_threshold2 = 97;
2709         cmd->dw12.g_rate_ratio_threshold3 = 103;
2710         cmd->dw12.g_rate_ratio_threshold4 = 125;
2711         cmd->dw12.g_rate_ratio_threshold5 = 160;
2712     }
2713
2714     i965_gpe_context_unmap_curbe(gpe_context);
2715 }
2716
2717 static void
2718 gen9_hevc_brc_update_set_surfaces(VADriverContextP ctx,
2719                                   struct intel_encoder_context *encoder_context,
2720                                   struct i965_gpe_context *gpe_context)
2721 {
2722     struct encoder_vme_mfc_context *vme_context = NULL;
2723     struct gen9_hevc_encoder_context *priv_ctx = NULL;
2724     struct gen9_hevc_encoder_state *priv_state = NULL;
2725     int bti_idx = 0;
2726
2727     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
2728     priv_ctx = (struct gen9_hevc_encoder_context *)vme_context->private_enc_ctx;
2729     priv_state = (struct gen9_hevc_encoder_state *)vme_context->private_enc_state;
2730
2731     gen9_hevc_set_gpe_1d_surface(ctx, priv_ctx, gpe_context,
2732                                  HEVC_ENC_SURFACE_BRC_HISTORY, bti_idx++,
2733                                  0, 0, 0, NULL, NULL);
2734
2735     gen9_hevc_set_gpe_1d_surface(ctx, priv_ctx, gpe_context,
2736                                  HEVC_ENC_SURFACE_BRC_PAST_PAK_INFO, bti_idx++,
2737                                  0, 0, 0, NULL, NULL);
2738
2739     gen9_hevc_set_gpe_1d_surface(ctx, priv_ctx, gpe_context,
2740                                  HEVC_ENC_SURFACE_BRC_HCP_PIC_STATE, bti_idx++,
2741                                  0, 0, 0, &priv_ctx->res_brc_pic_states_read_buffer,
2742                                  NULL);
2743
2744     gen9_hevc_set_gpe_1d_surface(ctx, priv_ctx, gpe_context,
2745                                  HEVC_ENC_SURFACE_BRC_HCP_PIC_STATE, bti_idx++,
2746                                  0, 0, 0, &priv_ctx->res_brc_pic_states_write_buffer,
2747                                  NULL);
2748
2749     gen9_hevc_set_gpe_1d_surface(ctx, priv_ctx, gpe_context,
2750                                  HEVC_ENC_SURFACE_BRC_INPUT, bti_idx++,
2751                                  0, 0, 0, NULL, NULL);
2752
2753     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
2754                                  HEVC_ENC_SURFACE_BRC_ME_DIST, bti_idx++,
2755                                  0, 1, I965_SURFACEFORMAT_R8_UNORM,
2756                                  priv_state->picture_coding_type == HEVC_SLICE_I ?
2757                                  &priv_ctx->res_brc_intra_dist_buffer :
2758                                  &priv_ctx->res_brc_me_dist_buffer,
2759                                  NULL);
2760
2761     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
2762                                  HEVC_ENC_SURFACE_BRC_DATA, bti_idx++,
2763                                  0, 1, I965_SURFACEFORMAT_R8_UNORM,
2764                                  NULL, NULL);
2765 }
2766
2767 static void
2768 gen9_hevc_brc_update(VADriverContextP ctx,
2769                      struct encode_state *encode_state,
2770                      struct intel_encoder_context *encoder_context)
2771 {
2772     struct encoder_vme_mfc_context *vme_context = NULL;
2773     struct gen9_hevc_encoder_context *priv_ctx = NULL;
2774     struct gen9_hevc_encoder_state *priv_state = NULL;
2775     struct i965_gpe_context *gpe_context = NULL;
2776     struct gpe_media_object_parameter param;
2777     int media_state = HEVC_ENC_MEDIA_STATE_BRC_UPDATE;
2778     int gpe_idx = HEVC_BRC_FRAME_UPDATE_IDX;
2779
2780     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
2781     priv_ctx = (struct gen9_hevc_encoder_context *)vme_context->private_enc_ctx;
2782     priv_state = (struct gen9_hevc_encoder_state *)vme_context->private_enc_state;
2783     gpe_context = &priv_ctx->brc_context.gpe_contexts[gpe_idx];
2784
2785     gen8_gpe_context_init(ctx, gpe_context);
2786     gen9_gpe_reset_binding_table(ctx, gpe_context);
2787
2788     gen9_hevc_brc_update_set_pic_states(ctx, encode_state, encoder_context);
2789     gen9_hevc_brc_update_set_constant(ctx, encode_state, encoder_context);
2790     gen9_hevc_brc_update_set_curbe(ctx, encode_state, encoder_context, gpe_context);
2791     gen9_hevc_brc_update_set_surfaces(ctx, encoder_context, gpe_context);
2792
2793     gen8_gpe_setup_interface_data(ctx, gpe_context);
2794
2795     memset(&param, 0, sizeof(param));
2796     gen9_hevc_run_object(ctx, encoder_context, gpe_context, &param,
2797                          media_state);
2798
2799     if (priv_state->lcu_brc_enabled ||
2800         priv_state->num_roi)
2801         gen9_hevc_brc_update_lcu_based(ctx, encode_state, encoder_context);
2802 }
2803
2804 // Depth converstion for 10bits
2805
2806 static void
2807 gen9_hevc_frame_depth_conversion_set_curbe(VADriverContextP ctx,
2808                                            struct encode_state *encode_state,
2809                                            struct intel_encoder_context *encoder_context,
2810                                            struct i965_gpe_context *gpe_context,
2811                                            GEN9_HEVC_DOWNSCALE_STAGE scale_stage)
2812 {
2813     struct encoder_vme_mfc_context *vme_context = NULL;
2814     struct gen9_hevc_encoder_state *priv_state = NULL;
2815     gen95_hevc_mbenc_ds_combined_curbe_data *cmd = NULL;
2816     int bti_idx = 0;
2817
2818     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
2819     priv_state = (struct gen9_hevc_encoder_state *)vme_context->private_enc_state;
2820
2821     cmd = i965_gpe_context_map_curbe(gpe_context);
2822     if (!cmd)
2823         return;
2824
2825     memset((void *)cmd, 0, sizeof(*cmd));
2826
2827     cmd->dw0.pak_bitdepth_chroma = 10;
2828     cmd->dw0.pak_bitdepth_luma = 10;
2829     cmd->dw0.enc_bitdepth_chroma = 8;
2830     cmd->dw0.enc_bitdepth_luma = 8;
2831     cmd->dw0.rounding_value = 1;
2832     cmd->dw1.pic_format = 0;
2833     cmd->dw1.pic_convert_flag = 1;
2834     cmd->dw1.pic_down_scale = scale_stage;
2835     cmd->dw1.pic_mb_stat_output_cntrl = 0;
2836     cmd->dw2.orig_pic_width = priv_state->picture_width;
2837     cmd->dw2.orig_pic_height = priv_state->picture_height;
2838
2839     cmd->dw3.bti_surface_p010 = bti_idx++;
2840     bti_idx++;
2841     cmd->dw4.bti_surface_nv12 = bti_idx++;
2842     bti_idx++;
2843     cmd->dw5.bti_src_y_4xdownscaled = bti_idx++;
2844     cmd->dw6.bti_surf_mbstate = bti_idx++;
2845     cmd->dw7.bit_src_y_2xdownscaled = bti_idx++;
2846
2847     i965_gpe_context_unmap_curbe(gpe_context);
2848 }
2849
2850 static void
2851 gen9_hevc_frame_depth_conversion_set_surfaces(VADriverContextP ctx,
2852                                               struct encode_state *encode_state,
2853                                               struct intel_encoder_context *encoder_context,
2854                                               struct i965_gpe_context *gpe_context,
2855                                               struct object_surface *src_surface,
2856                                               struct object_surface *dst_surface)
2857 {
2858     struct encoder_vme_mfc_context *vme_context = NULL;
2859     struct gen9_hevc_encoder_context *priv_ctx = NULL;
2860     struct gen9_hevc_surface_priv *surface_priv = NULL;
2861     int bti_idx = 0;
2862
2863     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
2864     priv_ctx = (struct gen9_hevc_encoder_context *)vme_context->private_enc_ctx;
2865     surface_priv = (struct gen9_hevc_surface_priv *)dst_surface->private_data;
2866
2867     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
2868                                  HEVC_ENC_SURFACE_RAW_10bit_Y_UV, bti_idx++,
2869                                  1, 1, I965_SURFACEFORMAT_R8_UNORM,
2870                                  NULL, src_surface);
2871     bti_idx++;
2872
2873     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
2874                                  HEVC_ENC_SURFACE_RAW_FC_8bit_Y_UV, bti_idx++,
2875                                  1, 1, I965_SURFACEFORMAT_R8_UNORM,
2876                                  NULL,
2877                                  surface_priv->surface_obj_nv12);
2878     bti_idx++;
2879
2880     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
2881                                  HEVC_ENC_SURFACE_Y_4X, bti_idx++,
2882                                  0, 1, I965_SURFACEFORMAT_R32_UNORM,
2883                                  NULL,
2884                                  surface_priv->scaled_surface_obj[HEVC_SCALED_SURF_4X_ID]);
2885
2886     gen9_hevc_set_gpe_1d_surface(ctx, priv_ctx, gpe_context,
2887                                  HEVC_ENC_SURFACE_RAW_MBSTAT, bti_idx++,
2888                                  0, 0, 0, NULL, NULL);
2889
2890     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
2891                                  HEVC_ENC_SURFACE_Y_2X, bti_idx++,
2892                                  0, 1, I965_SURFACEFORMAT_R32_UNORM,
2893                                  NULL, NULL);
2894 }
2895
2896 static void
2897 gen9_hevc_frame_depth_conversion(VADriverContextP ctx,
2898                                  struct encode_state *encode_state,
2899                                  struct intel_encoder_context *encoder_context,
2900                                  struct object_surface *src_surface,
2901                                  struct object_surface *dst_surface,
2902                                  GEN9_HEVC_DOWNSCALE_STAGE scale_stage)
2903 {
2904     struct encoder_vme_mfc_context *vme_context = NULL;
2905     struct gen9_hevc_encoder_context *priv_ctx = NULL;
2906     struct gen9_hevc_encoder_state *priv_state = NULL;
2907     struct gpe_media_object_walker_parameter param;
2908     struct hevc_enc_kernel_walker_parameter hevc_walker_param;
2909     struct i965_gpe_context *gpe_context = NULL;
2910     int media_state = HEVC_ENC_MEDIA_STATE_DS_COMBINED;
2911
2912     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
2913     priv_ctx = (struct gen9_hevc_encoder_context *)vme_context->private_enc_ctx;
2914     priv_state = (struct gen9_hevc_encoder_state *)vme_context->private_enc_state;
2915
2916     gpe_context = &priv_ctx->mbenc_context.gpe_contexts[HEVC_MBENC_DS_COMBINED_IDX];
2917
2918     gen8_gpe_context_init(ctx, gpe_context);
2919     gen9_gpe_reset_binding_table(ctx, gpe_context);
2920     gen9_hevc_frame_depth_conversion_set_curbe(ctx, encode_state, encoder_context, gpe_context,
2921                                                scale_stage);
2922     gen9_hevc_frame_depth_conversion_set_surfaces(ctx, encode_state, encoder_context, gpe_context,
2923                                                   src_surface, dst_surface);
2924     gen8_gpe_setup_interface_data(ctx, gpe_context);
2925
2926     memset((void *)&hevc_walker_param, 0, sizeof(hevc_walker_param));
2927     hevc_walker_param.resolution_x = ALIGN(priv_state->picture_width >> 2, 32) >> 3;
2928     hevc_walker_param.resolution_y = ALIGN(priv_state->picture_height >> 2, 32) >> 3;
2929     hevc_walker_param.no_dependency = 1;
2930     gen9_hevc_init_object_walker(&hevc_walker_param, &param);
2931     gen9_hevc_run_object_walker(ctx, encoder_context, gpe_context, &param,
2932                                 media_state);
2933 }
2934
2935 static void
2936 gen9_hevc_ref_frame_depth_conversion(VADriverContextP ctx,
2937                                      struct encode_state *encode_state,
2938                                      struct intel_encoder_context *encoder_context)
2939 {
2940     struct i965_driver_data *i965 = i965_driver_data(ctx);
2941     struct gen9_hevc_surface_priv *surface_priv = NULL;
2942     VAEncSliceParameterBufferHEVC *slice_param = NULL;
2943     struct object_surface *obj_surface = NULL;
2944     int i = 0;
2945
2946     slice_param = (VAEncSliceParameterBufferHEVC *)encode_state->slice_params_ext[0]->buffer;
2947
2948     for (i = 0; i < slice_param->num_ref_idx_l0_active_minus1 + 1; i++) {
2949         obj_surface = SURFACE(slice_param->ref_pic_list0[i].picture_id);
2950         if (obj_surface) {
2951             surface_priv = (struct gen9_hevc_surface_priv *)obj_surface->private_data;
2952
2953             if (!surface_priv->surface_nv12_valid) {
2954                 gen9_hevc_frame_depth_conversion(ctx, encode_state, encoder_context,
2955                                                  obj_surface, obj_surface,
2956                                                  HEVC_ENC_DS_DISABLED);
2957
2958                 surface_priv->surface_reff = surface_priv->surface_obj_nv12;
2959                 surface_priv->surface_nv12_valid = 1;
2960             }
2961         }
2962     }
2963
2964     for (i = 0; i < slice_param->num_ref_idx_l1_active_minus1 + 1; i++) {
2965         obj_surface = SURFACE(slice_param->ref_pic_list1[i].picture_id);
2966         if (obj_surface) {
2967             surface_priv = (struct gen9_hevc_surface_priv *)obj_surface->private_data;
2968
2969             if (!surface_priv->surface_nv12_valid) {
2970                 gen9_hevc_frame_depth_conversion(ctx, encode_state, encoder_context,
2971                                                  obj_surface, obj_surface,
2972                                                  HEVC_ENC_DS_DISABLED);
2973
2974                 surface_priv->surface_reff = surface_priv->surface_obj_nv12;
2975                 surface_priv->surface_nv12_valid = 1;
2976             }
2977         }
2978     }
2979 }
2980
2981 // Scaling implementation
2982
2983 static void
2984 gen9_hevc_scaling_set_curbe_2x(struct i965_gpe_context *gpe_context,
2985                                struct gen9_hevc_scaling_parameter *scaling_param)
2986 {
2987     gen9_hevc_scaling2x_curbe_data *cmd;
2988
2989     cmd = i965_gpe_context_map_curbe(gpe_context);
2990     if (!cmd)
2991         return;
2992
2993     memset((void *)cmd, 0, sizeof(*cmd));
2994
2995     cmd->dw0.input_picture_width  = scaling_param->input_frame_width;
2996     cmd->dw0.input_picture_height = scaling_param->input_frame_height;
2997
2998     cmd->dw8.input_y_bti = GEN9_HEVC_SCALING_FRAME_SRC_Y_INDEX;
2999     cmd->dw9.output_y_bti = GEN9_HEVC_SCALING_FRAME_DST_Y_INDEX;
3000
3001     i965_gpe_context_unmap_curbe(gpe_context);
3002 }
3003
3004 static void
3005 gen9_hevc_scaling_set_curbe_4x(struct i965_gpe_context *gpe_context,
3006                                struct gen9_hevc_scaling_parameter *scaling_param)
3007 {
3008     gen9_hevc_scaling4x_curbe_data *cmd;
3009
3010     cmd = i965_gpe_context_map_curbe(gpe_context);
3011     if (!cmd)
3012         return;
3013
3014     memset((void *)cmd, 0, sizeof(*cmd));
3015
3016     cmd->dw0.input_picture_width  = scaling_param->input_frame_width;
3017     cmd->dw0.input_picture_height = scaling_param->input_frame_height;
3018
3019     cmd->dw1.input_y_bti = GEN9_HEVC_SCALING_FRAME_SRC_Y_INDEX;
3020     cmd->dw2.output_y_bti = GEN9_HEVC_SCALING_FRAME_DST_Y_INDEX;
3021
3022     cmd->dw5.flatness_threshold = 0;
3023     cmd->dw6.enable_mb_flatness_check = scaling_param->enable_mb_flatness_check;
3024     cmd->dw7.enable_mb_variance_output = scaling_param->enable_mb_variance_output;
3025     cmd->dw8.enable_mb_pixel_average_output = scaling_param->enable_mb_pixel_average_output;
3026
3027     if (cmd->dw6.enable_mb_flatness_check ||
3028         cmd->dw7.enable_mb_variance_output ||
3029         cmd->dw8.enable_mb_pixel_average_output)
3030         cmd->dw10.mbv_proc_stat_bti = GEN9_HEVC_SCALING_FRAME_MBVPROCSTATS_DST_INDEX;
3031
3032     i965_gpe_context_unmap_curbe(gpe_context);
3033 }
3034
3035 static void
3036 gen9_hevc_scaling_set_curbe(struct gen9_hevc_scaling_parameter *scaling_param,
3037                             struct i965_gpe_context *gpe_context)
3038 {
3039     if (scaling_param->use_32x_scaling)
3040         gen9_hevc_scaling_set_curbe_2x(gpe_context, scaling_param);
3041     else
3042         gen9_hevc_scaling_set_curbe_4x(gpe_context, scaling_param);
3043 }
3044
3045 static void
3046 gen9_hevc_scaling_set_surfaces(VADriverContextP ctx,
3047                                struct encode_state *encode_state,
3048                                struct intel_encoder_context *encoder_context,
3049                                struct gen9_hevc_scaling_parameter *scaling_param,
3050                                struct i965_gpe_context *gpe_context)
3051 {
3052     unsigned int surface_format;
3053
3054     if (scaling_param->scaling_out_use_32unorm_surf_fmt)
3055         surface_format = I965_SURFACEFORMAT_R32_UNORM;
3056     else if (scaling_param->scaling_out_use_16unorm_surf_fmt)
3057         surface_format = I965_SURFACEFORMAT_R16_UNORM;
3058     else
3059         surface_format = I965_SURFACEFORMAT_R8_UNORM;
3060
3061     gen9_add_2d_gpe_surface(ctx, gpe_context,
3062                             scaling_param->input_surface,
3063                             0, 1, surface_format,
3064                             GEN9_HEVC_SCALING_FRAME_SRC_Y_INDEX);
3065
3066     gen9_add_2d_gpe_surface(ctx, gpe_context,
3067                             scaling_param->output_surface,
3068                             0, 1, surface_format,
3069                             GEN9_HEVC_SCALING_FRAME_DST_Y_INDEX);
3070
3071     if ((scaling_param->enable_mb_flatness_check ||
3072          scaling_param->enable_mb_variance_output ||
3073          scaling_param->enable_mb_pixel_average_output) &&
3074         scaling_param->use_4x_scaling) {
3075         gen9_add_buffer_2d_gpe_surface(ctx, gpe_context,
3076                                        scaling_param->pres_mbv_proc_stat_buffer,
3077                                        1,
3078                                        I965_SURFACEFORMAT_R8_UNORM,
3079                                        GEN9_HEVC_SCALING_FRAME_MBVPROCSTATS_DST_INDEX);
3080     }
3081 }
3082
3083 static void
3084 gen9_hevc_kernel_scaling(VADriverContextP ctx,
3085                          struct encode_state *encode_state,
3086                          struct intel_encoder_context *encoder_context,
3087                          enum HEVC_HME_TYPE hme_type)
3088 {
3089     struct encoder_vme_mfc_context *vme_context = NULL;
3090     struct gen9_hevc_encoder_context *priv_ctx = NULL;
3091     struct gen9_hevc_encoder_state *priv_state = NULL;
3092     struct gen9_hevc_surface_priv *surface_priv = NULL;
3093     struct gen9_hevc_scaling_parameter scaling_param;
3094     struct gpe_media_object_walker_parameter param;
3095     unsigned int downscaled_width_in_mb, downscaled_height_in_mb;
3096     struct hevc_enc_kernel_walker_parameter hevc_walker_param;
3097     struct i965_gpe_context *gpe_context = NULL;
3098     int gpe_idx = 0, media_state = 0;;
3099
3100     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
3101     priv_ctx = (struct gen9_hevc_encoder_context *)vme_context->private_enc_ctx;
3102     priv_state = (struct gen9_hevc_encoder_state *)vme_context->private_enc_state;
3103
3104     surface_priv = (struct gen9_hevc_surface_priv *)encode_state->reconstructed_object->private_data;
3105
3106     memset((void *)&scaling_param, 0, sizeof(scaling_param));
3107     switch (hme_type) {
3108     case HEVC_HME_4X:
3109         media_state = HEVC_ENC_MEDIA_STATE_4X_SCALING;
3110         gpe_idx = HEVC_ENC_SCALING_4X;
3111         downscaled_width_in_mb = priv_state->downscaled_width_4x_in_mb;
3112         downscaled_height_in_mb = priv_state->downscaled_height_4x_in_mb;
3113
3114         scaling_param.input_surface = encode_state->input_yuv_object;
3115         scaling_param.input_frame_width = priv_state->picture_width;
3116         scaling_param.input_frame_height = priv_state->picture_height;
3117
3118         scaling_param.output_surface = surface_priv->scaled_surface_obj[HEVC_SCALED_SURF_4X_ID];
3119         scaling_param.output_frame_width = priv_state->frame_width_4x;
3120         scaling_param.output_frame_height = priv_state->frame_height_4x;
3121
3122         scaling_param.enable_mb_flatness_check = priv_state->flatness_check_enable;
3123         scaling_param.enable_mb_variance_output = 0;
3124         scaling_param.enable_mb_pixel_average_output = 0;
3125         scaling_param.pres_mbv_proc_stat_buffer = &(priv_ctx->res_flatness_check_surface);
3126
3127         scaling_param.blk8x8_stat_enabled = 0;
3128         scaling_param.use_4x_scaling  = 1;
3129         scaling_param.use_16x_scaling = 0;
3130         scaling_param.use_32x_scaling = 0;
3131         break;
3132     case HEVC_HME_16X:
3133         media_state = HEVC_ENC_MEDIA_STATE_16X_SCALING;
3134         gpe_idx = HEVC_ENC_SCALING_16X;
3135         downscaled_width_in_mb = priv_state->downscaled_width_16x_in_mb;
3136         downscaled_height_in_mb = priv_state->downscaled_height_16x_in_mb;
3137
3138         scaling_param.input_surface = surface_priv->scaled_surface_obj[HEVC_SCALED_SURF_4X_ID];
3139         scaling_param.input_frame_width = priv_state->frame_width_4x;
3140         scaling_param.input_frame_height = priv_state->frame_height_4x;
3141
3142         scaling_param.output_surface = surface_priv->scaled_surface_obj[HEVC_SCALED_SURF_16X_ID];
3143         scaling_param.output_frame_width = priv_state->frame_width_16x;
3144         scaling_param.output_frame_height = priv_state->frame_height_16x;
3145
3146         scaling_param.enable_mb_flatness_check = 0;
3147         scaling_param.enable_mb_variance_output = 0;
3148         scaling_param.enable_mb_pixel_average_output = 0;
3149
3150         scaling_param.blk8x8_stat_enabled = 0;
3151         scaling_param.use_4x_scaling  = 0;
3152         scaling_param.use_16x_scaling = 1;
3153         scaling_param.use_32x_scaling = 0;
3154         break;
3155     case HEVC_HME_32X:
3156         media_state = HEVC_ENC_MEDIA_STATE_32X_SCALING;
3157         gpe_idx = HEVC_ENC_SCALING_32X;
3158         downscaled_width_in_mb = priv_state->downscaled_width_32x_in_mb;
3159         downscaled_height_in_mb = priv_state->downscaled_height_32x_in_mb;
3160
3161         scaling_param.input_surface = surface_priv->scaled_surface_obj[HEVC_SCALED_SURF_16X_ID];
3162         scaling_param.input_frame_width = priv_state->frame_width_16x;
3163         scaling_param.input_frame_height = priv_state->frame_height_16x;
3164
3165         scaling_param.output_surface = surface_priv->scaled_surface_obj[HEVC_SCALED_SURF_32X_ID];
3166         scaling_param.output_frame_width = priv_state->frame_width_32x;
3167         scaling_param.output_frame_height = priv_state->frame_height_32x;
3168
3169         scaling_param.enable_mb_flatness_check = 0;
3170         scaling_param.enable_mb_variance_output = 0;
3171         scaling_param.enable_mb_pixel_average_output = 0;
3172
3173         scaling_param.blk8x8_stat_enabled = 0;
3174         scaling_param.use_4x_scaling  = 0;
3175         scaling_param.use_16x_scaling = 0;
3176         scaling_param.use_32x_scaling = 1;
3177         break;
3178     default:
3179         return;
3180     }
3181
3182     gpe_context = &priv_ctx->scaling_context.gpe_contexts[gpe_idx];
3183     gen8_gpe_context_init(ctx, gpe_context);
3184     gen9_gpe_reset_binding_table(ctx, gpe_context);
3185     gen9_hevc_scaling_set_curbe(&scaling_param, gpe_context);
3186
3187     if (hme_type == HEVC_HME_32X) {
3188         scaling_param.scaling_out_use_16unorm_surf_fmt = 1;
3189         scaling_param.scaling_out_use_32unorm_surf_fmt = 0;
3190     } else {
3191         scaling_param.scaling_out_use_16unorm_surf_fmt = 0;
3192         scaling_param.scaling_out_use_32unorm_surf_fmt = 1;
3193     }
3194
3195     gen9_hevc_scaling_set_surfaces(ctx, encode_state, encoder_context, &scaling_param,
3196                                    gpe_context);
3197     gen8_gpe_setup_interface_data(ctx, gpe_context);
3198
3199     memset((void *)&hevc_walker_param, 0, sizeof(hevc_walker_param));
3200     if (hme_type == HEVC_HME_32X) {
3201         hevc_walker_param.resolution_x = downscaled_width_in_mb;
3202         hevc_walker_param.resolution_y = downscaled_height_in_mb;
3203     } else {
3204         hevc_walker_param.resolution_x = downscaled_width_in_mb * 2;
3205         hevc_walker_param.resolution_y = downscaled_height_in_mb * 2;
3206     }
3207     hevc_walker_param.no_dependency = 1;
3208     gen9_hevc_init_object_walker(&hevc_walker_param, &param);
3209     gen9_hevc_run_object_walker(ctx, encoder_context, gpe_context, &param,
3210                                 media_state);
3211 }
3212
3213 static void
3214 gen9_hevc_hme_scaling(VADriverContextP ctx,
3215                       struct encode_state *encode_state,
3216                       struct intel_encoder_context *encoder_context)
3217 {
3218     struct encoder_vme_mfc_context *vme_context = NULL;
3219     struct generic_enc_codec_state *generic_state = NULL;
3220     struct gen9_hevc_encoder_state *priv_state = NULL;
3221
3222     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
3223     generic_state = (struct generic_enc_codec_state *)vme_context->generic_enc_state;
3224     priv_state = (struct gen9_hevc_encoder_state *)vme_context->private_enc_state;
3225
3226     if (priv_state->bit_depth_luma_minus8)
3227         gen9_hevc_frame_depth_conversion(ctx, encode_state, encoder_context,
3228                                          encode_state->input_yuv_object,
3229                                          encode_state->reconstructed_object,
3230                                          HEVC_ENC_2xDS_4xDS_STAGE);
3231     else
3232         gen9_hevc_kernel_scaling(ctx, encode_state, encoder_context, HEVC_HME_4X);
3233
3234     if (generic_state->b16xme_supported) {
3235         gen9_hevc_kernel_scaling(ctx, encode_state, encoder_context, HEVC_HME_16X);
3236
3237         if (generic_state->b32xme_supported)
3238             gen9_hevc_kernel_scaling(ctx, encode_state, encoder_context, HEVC_HME_32X);
3239     }
3240 }
3241
3242 // ME implementation
3243
3244 static void
3245 gen9_hevc_me_set_curbe(VADriverContextP ctx,
3246                        struct encode_state *encode_state,
3247                        struct intel_encoder_context *encoder_context,
3248                        enum HEVC_HME_TYPE hme_type,
3249                        struct i965_gpe_context *gpe_context)
3250 {
3251     struct encoder_vme_mfc_context *vme_context = NULL;
3252     struct generic_enc_codec_state *generic_state = NULL;
3253     struct gen9_hevc_encoder_state *priv_state = NULL;
3254     gen9_hevc_me_curbe_data *cmd = NULL;
3255     VAEncPictureParameterBufferHEVC *pic_param = NULL;
3256     VAEncSliceParameterBufferHEVC *slice_param = NULL;
3257     unsigned int use_mv_from_prev_step = 0;
3258     unsigned int write_distortions = 0;
3259     unsigned int slice_qp = 0;
3260     unsigned int me_method = 0;
3261     unsigned int mv_shift_factor = 0, prev_mv_read_pos_factor = 0;
3262     unsigned int downscaled_width_in_mb, downscaled_height_in_mb;
3263
3264     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
3265     generic_state = (struct generic_enc_codec_state *)vme_context->generic_enc_state;
3266     priv_state = (struct gen9_hevc_encoder_state *)vme_context->private_enc_state;
3267     pic_param = (VAEncPictureParameterBufferHEVC *)encode_state->pic_param_ext->buffer;
3268     slice_param = (VAEncSliceParameterBufferHEVC *)encode_state->slice_params_ext[0]->buffer;
3269
3270     switch (hme_type) {
3271     case HEVC_HME_4X :
3272         use_mv_from_prev_step = (generic_state->b16xme_enabled) ? 1 : 0;;
3273         write_distortions = 1;
3274         mv_shift_factor = 2;
3275         prev_mv_read_pos_factor = 0;
3276         downscaled_width_in_mb = ALIGN(priv_state->picture_width / 4, 16) / 16;
3277         downscaled_height_in_mb = ALIGN(priv_state->picture_height / 4, 16) / 16;
3278         break;
3279     case HEVC_HME_16X :
3280         use_mv_from_prev_step = (generic_state->b32xme_enabled) ? 1 : 0;
3281         write_distortions = 0;
3282         mv_shift_factor = 2;
3283         prev_mv_read_pos_factor = 1;
3284         downscaled_width_in_mb = ALIGN(priv_state->picture_width / 16, 16) / 16;
3285         downscaled_height_in_mb = ALIGN(priv_state->picture_height / 16, 16) / 16;
3286         break;
3287     case HEVC_HME_32X :
3288         use_mv_from_prev_step = 0;
3289         write_distortions = 0;
3290         mv_shift_factor = 1;
3291         prev_mv_read_pos_factor = 0;
3292         downscaled_width_in_mb = ALIGN(priv_state->picture_width / 32, 16) / 16;
3293         downscaled_height_in_mb = ALIGN(priv_state->picture_height / 32, 16) / 16;
3294         break;
3295     default:
3296         return;
3297     }
3298
3299     me_method = GEN9_HEVC_ME_METHOD[priv_state->tu_mode];
3300     slice_qp = pic_param->pic_init_qp + slice_param->slice_qp_delta;
3301
3302     cmd = i965_gpe_context_map_curbe(gpe_context);
3303     if (!cmd)
3304         return;
3305
3306     memcpy((void *)cmd, GEN9_HEVC_ME_INIT_CURBE_DATA, sizeof(gen9_hevc_me_curbe_data));
3307
3308     cmd->dw3.sub_pel_mode = 3;
3309     cmd->dw4.picture_height_minus1 = downscaled_height_in_mb - 1;
3310     cmd->dw4.picture_width = downscaled_width_in_mb;
3311     cmd->dw5.qp_prime_y = slice_qp;
3312     cmd->dw6.use_mv_from_prev_step = use_mv_from_prev_step;
3313     cmd->dw6.write_distortions = write_distortions;
3314     cmd->dw6.super_combine_dist = GEN9_HEVC_SUPER_COMBINE_DIST[priv_state->tu_mode];
3315     cmd->dw6.max_vmvr = 512;
3316
3317     if (priv_state->picture_coding_type != HEVC_SLICE_I) {
3318         cmd->dw13.num_ref_idx_l0_minus1 = slice_param->num_ref_idx_l0_active_minus1;
3319         if (priv_state->picture_coding_type == HEVC_SLICE_B) {
3320             cmd->dw1.bi_weight = 32;
3321             cmd->dw13.num_ref_idx_l1_minus1 = slice_param->num_ref_idx_l1_active_minus1;
3322         }
3323     }
3324
3325     cmd->dw15.prev_mv_read_pos_factor = prev_mv_read_pos_factor;
3326     cmd->dw15.mv_shift_factor = mv_shift_factor;
3327
3328     memcpy(&cmd->dw16, table_enc_search_path[GEN9_HEVC_ENC_MEMETHOD_TABLE][me_method], 14 * sizeof(unsigned int));
3329
3330     cmd->dw32._4x_memv_output_data_surf_index = GEN9_HEVC_ME_MV_DATA_SURFACE_INDEX;
3331     cmd->dw33._16x_32x_memv_input_data_surf_index = (hme_type == HEVC_HME_32X) ?
3332                                                     GEN9_HEVC_ME_32X_MV_DATA_SURFACE_INDEX : GEN9_HEVC_ME_16X_MV_DATA_SURFACE_INDEX;
3333     cmd->dw34._4x_me_output_dist_surf_index = GEN9_HEVC_ME_DISTORTION_SURFACE_INDEX;
3334     cmd->dw35._4x_me_output_brc_dist_surf_index = GEN9_HEVC_ME_BRC_DISTORTION_INDEX;
3335     cmd->dw36.vme_fwd_inter_pred_surf_index = GEN9_HEVC_ME_CURR_FOR_FWD_REF_INDEX;
3336     cmd->dw37.vme_bdw_inter_pred_surf_index = GEN9_HEVC_ME_CURR_FOR_BWD_REF_INDEX;
3337
3338     i965_gpe_context_unmap_curbe(gpe_context);
3339 }
3340
3341 static void
3342 gen9_hevc_me_set_surfaces(VADriverContextP ctx,
3343                           struct encode_state *encode_state,
3344                           struct intel_encoder_context *encoder_context,
3345                           enum HEVC_HME_TYPE hme_type,
3346                           struct i965_gpe_context *gpe_context)
3347 {
3348     struct i965_driver_data *i965 = i965_driver_data(ctx);
3349     struct encoder_vme_mfc_context *vme_context = NULL;
3350     struct gen9_hevc_encoder_context *priv_ctx = NULL;
3351     struct generic_enc_codec_state *generic_state = NULL;
3352     VAEncSliceParameterBufferHEVC *slice_param = NULL;
3353     struct gen9_hevc_surface_priv *surface_priv = NULL;
3354     struct object_surface *obj_surface = NULL;
3355     int scaled_surf_id = VA_INVALID_SURFACE, surface_id = VA_INVALID_SURFACE;
3356     int i = 0;
3357
3358     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
3359     priv_ctx = (struct gen9_hevc_encoder_context *)vme_context->private_enc_ctx;
3360     generic_state = (struct generic_enc_codec_state *)vme_context->generic_enc_state;
3361     slice_param = (VAEncSliceParameterBufferHEVC *)encode_state->slice_params_ext[0]->buffer;
3362
3363     switch (hme_type) {
3364     case HEVC_HME_4X:
3365         scaled_surf_id = HEVC_SCALED_SURF_4X_ID;
3366         gen9_add_buffer_2d_gpe_surface(ctx, gpe_context,
3367                                        &priv_ctx->s4x_memv_data_buffer,
3368                                        1,
3369                                        I965_SURFACEFORMAT_R8_UNORM,
3370                                        GEN9_HEVC_ME_MV_DATA_SURFACE_INDEX);
3371
3372         if (generic_state->b16xme_enabled)
3373             gen9_add_buffer_2d_gpe_surface(ctx, gpe_context,
3374                                            &priv_ctx->s16x_memv_data_buffer,
3375                                            1,
3376                                            I965_SURFACEFORMAT_R8_UNORM,
3377                                            GEN9_HEVC_ME_16X_MV_DATA_SURFACE_INDEX);
3378
3379         gen9_add_buffer_2d_gpe_surface(ctx, gpe_context,
3380                                        &priv_ctx->res_brc_me_dist_buffer,
3381                                        1,
3382                                        I965_SURFACEFORMAT_R8_UNORM,
3383                                        GEN9_HEVC_ME_BRC_DISTORTION_INDEX);
3384
3385         gen9_add_buffer_2d_gpe_surface(ctx, gpe_context,
3386                                        &priv_ctx->s4x_memv_distortion_buffer,
3387                                        1,
3388                                        I965_SURFACEFORMAT_R8_UNORM,
3389                                        GEN9_HEVC_ME_DISTORTION_SURFACE_INDEX);
3390         break;
3391     case HEVC_HME_16X:
3392         scaled_surf_id = HEVC_SCALED_SURF_16X_ID;
3393
3394         gen9_add_buffer_2d_gpe_surface(ctx, gpe_context,
3395                                        &priv_ctx->s16x_memv_data_buffer,
3396                                        1,
3397                                        I965_SURFACEFORMAT_R8_UNORM,
3398                                        GEN9_HEVC_ME_MV_DATA_SURFACE_INDEX);
3399
3400         if (generic_state->b32xme_enabled)
3401             gen9_add_buffer_2d_gpe_surface(ctx, gpe_context,
3402                                            &priv_ctx->s32x_memv_data_buffer,
3403                                            1,
3404                                            I965_SURFACEFORMAT_R8_UNORM,
3405                                            GEN9_HEVC_ME_32X_MV_DATA_SURFACE_INDEX);
3406         else
3407             gen9_add_buffer_2d_gpe_surface(ctx, gpe_context,
3408                                            &priv_ctx->s16x_memv_data_buffer,
3409                                            1,
3410                                            I965_SURFACEFORMAT_R8_UNORM,
3411                                            GEN9_HEVC_ME_16X_MV_DATA_SURFACE_INDEX);
3412         break;
3413     case HEVC_HME_32X:
3414         scaled_surf_id = HEVC_SCALED_SURF_32X_ID;
3415         gen9_add_buffer_2d_gpe_surface(ctx, gpe_context,
3416                                        &priv_ctx->s32x_memv_data_buffer,
3417                                        1,
3418                                        I965_SURFACEFORMAT_R8_UNORM,
3419                                        GEN9_HEVC_ME_MV_DATA_SURFACE_INDEX);
3420         break;
3421     default:
3422         return;
3423     }
3424
3425     obj_surface = encode_state->reconstructed_object;
3426     surface_priv = (struct gen9_hevc_surface_priv *)obj_surface->private_data;
3427     gen9_add_adv_gpe_surface(ctx, gpe_context,
3428                              surface_priv->scaled_surface_obj[scaled_surf_id],
3429                              GEN9_HEVC_ME_CURR_FOR_FWD_REF_INDEX);
3430
3431     for (i = 0; i < slice_param->num_ref_idx_l0_active_minus1 + 1; i++) {
3432         surface_id = slice_param->ref_pic_list0[i].picture_id;
3433         obj_surface = SURFACE(surface_id);
3434         if (!obj_surface || !obj_surface->private_data)
3435             break;
3436
3437         surface_priv = (struct gen9_hevc_surface_priv *)obj_surface->private_data;
3438         gen9_add_adv_gpe_surface(ctx, gpe_context,
3439                                  surface_priv->scaled_surface_obj[scaled_surf_id],
3440                                  GEN9_HEVC_ME_CURR_FOR_FWD_REF_INDEX + i * 2 + 1);
3441     }
3442
3443     obj_surface = encode_state->reconstructed_object;
3444     surface_priv = (struct gen9_hevc_surface_priv *)obj_surface->private_data;
3445     gen9_add_adv_gpe_surface(ctx, gpe_context,
3446                              surface_priv->scaled_surface_obj[scaled_surf_id],
3447                              GEN9_HEVC_ME_CURR_FOR_BWD_REF_INDEX);
3448
3449     for (i = 0; i < slice_param->num_ref_idx_l1_active_minus1 + 1; i++) {
3450         surface_id = slice_param->ref_pic_list1[i].picture_id;
3451         obj_surface = SURFACE(surface_id);
3452         if (!obj_surface || !obj_surface->private_data)
3453             break;
3454
3455         surface_priv = (struct gen9_hevc_surface_priv *)obj_surface->private_data;
3456         gen9_add_adv_gpe_surface(ctx, gpe_context,
3457                                  surface_priv->scaled_surface_obj[scaled_surf_id],
3458                                  GEN9_HEVC_ME_CURR_FOR_BWD_REF_INDEX + i * 2 + 1);
3459     }
3460 }
3461
3462 static void
3463 gen9_hevc_kernel_me(VADriverContextP ctx,
3464                     struct encode_state *encode_state,
3465                     struct intel_encoder_context *encoder_context,
3466                     enum HEVC_HME_TYPE hme_type)
3467 {
3468     struct encoder_vme_mfc_context *vme_context = NULL;
3469     struct gen9_hevc_encoder_context *priv_ctx = NULL;
3470     struct gen9_hevc_encoder_state *priv_state = NULL;
3471     struct gpe_media_object_walker_parameter param;
3472     struct hevc_enc_kernel_walker_parameter hevc_walker_param;
3473     unsigned int downscaled_width_in_mb, downscaled_height_in_mb;
3474     struct i965_gpe_context *gpe_context;
3475     int media_state = 0;
3476
3477     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
3478     priv_ctx = (struct gen9_hevc_encoder_context *)vme_context->private_enc_ctx;
3479     priv_state = (struct gen9_hevc_encoder_state *)vme_context->private_enc_state;
3480
3481     switch (hme_type) {
3482     case HEVC_HME_4X:
3483         media_state = HEVC_ENC_MEDIA_STATE_4X_ME;
3484         downscaled_width_in_mb = priv_state->downscaled_width_4x_in_mb;
3485         downscaled_height_in_mb = priv_state->downscaled_height_4x_in_mb;
3486         break;
3487     case HEVC_HME_16X:
3488         media_state = HEVC_ENC_MEDIA_STATE_16X_ME;
3489         downscaled_width_in_mb = priv_state->downscaled_width_16x_in_mb;
3490         downscaled_height_in_mb = priv_state->downscaled_height_16x_in_mb;
3491         break;
3492     case HEVC_HME_32X:
3493         media_state = HEVC_ENC_MEDIA_STATE_32X_ME;
3494         downscaled_width_in_mb = priv_state->downscaled_width_32x_in_mb;
3495         downscaled_height_in_mb = priv_state->downscaled_height_32x_in_mb;
3496         break;
3497     default:
3498         return;
3499     }
3500
3501     if (priv_state->picture_coding_type == HEVC_SLICE_P)
3502         gpe_context = &priv_ctx->me_context.gpe_context[hme_type][HEVC_ENC_ME_P];
3503     else
3504         gpe_context = &priv_ctx->me_context.gpe_context[hme_type][HEVC_ENC_ME_B];
3505
3506     gen8_gpe_context_init(ctx, gpe_context);
3507     gen9_gpe_reset_binding_table(ctx, gpe_context);
3508     gen9_hevc_me_set_curbe(ctx, encode_state, encoder_context, hme_type, gpe_context);
3509     gen9_hevc_me_set_surfaces(ctx, encode_state, encoder_context, hme_type, gpe_context);
3510     gen8_gpe_setup_interface_data(ctx, gpe_context);
3511
3512     memset((void *)&hevc_walker_param, 0, sizeof(hevc_walker_param));
3513     hevc_walker_param.resolution_x = downscaled_width_in_mb;
3514     hevc_walker_param.resolution_y = downscaled_height_in_mb;
3515     hevc_walker_param.no_dependency = 1;
3516     gen9_hevc_init_object_walker(&hevc_walker_param, &param);
3517
3518     gen9_hevc_run_object_walker(ctx, encoder_context, gpe_context, &param,
3519                                 media_state);
3520 }
3521
3522 static void
3523 gen9_hevc_hme_encode_me(VADriverContextP ctx,
3524                         struct encode_state *encode_state,
3525                         struct intel_encoder_context *encoder_context)
3526 {
3527     struct encoder_vme_mfc_context *vme_context = NULL;
3528     struct generic_enc_codec_state *generic_state = NULL;
3529
3530     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
3531     generic_state = (struct generic_enc_codec_state *)vme_context->generic_enc_state;
3532
3533     if (generic_state->b16xme_enabled) {
3534         if (generic_state->b32xme_enabled)
3535             gen9_hevc_kernel_me(ctx, encode_state, encoder_context, HEVC_HME_32X);
3536
3537         gen9_hevc_kernel_me(ctx, encode_state, encoder_context, HEVC_HME_16X);
3538     }
3539
3540     gen9_hevc_kernel_me(ctx, encode_state, encoder_context, HEVC_HME_4X);
3541 }
3542
3543 // MBENC kernels setting start
3544
3545 static unsigned char
3546 map_44_lut_value(unsigned int value,
3547                  unsigned char max)
3548 {
3549     unsigned int max_cost = 0;
3550     int data = 0;
3551     unsigned char ret = 0;
3552
3553     if (value == 0)
3554         return 0;
3555
3556     max_cost = ((max & 15) << (max >> 4));
3557     if (value >= max_cost)
3558         return max;
3559
3560     data = (int)(log((double)value) / log(2.)) - 3;
3561     if (data < 0)
3562         data = 0;
3563
3564     ret = (unsigned char)((data << 4) +
3565                           (int)((value + (data == 0 ? 0 : (1 << (data - 1)))) >> data));
3566     ret = (ret & 0xf) == 0 ? (ret | 8) : ret;
3567
3568     return ret;
3569 }
3570
3571 static void
3572 gen9_hevc_mbenc_set_costs(struct gen9_hevc_encoder_context *priv_ctx,
3573                           int slice_type,
3574                           int intra_trans_type,
3575                           unsigned int slice_qp,
3576                           unsigned char *mode_cost,
3577                           unsigned char *mv_cost,
3578                           unsigned char *mode_cost_sp,
3579                           unsigned int *simplest_intra_inter_threshold)
3580 {
3581     float had_bias = intra_trans_type == HEVC_ENC_INTRA_TRANS_HADAMARD ?
3582                      1.67f : 2.0f;
3583     double lambda_md, lambda_me;
3584
3585     lambda_md = priv_ctx->lambda_md_table[slice_type][slice_qp];
3586     lambda_me = priv_ctx->lambda_md_table[slice_type][slice_qp];
3587
3588     if (mode_cost) {
3589         mode_cost[0] = map_44_lut_value((unsigned int)(lambda_md * GEN9_HEVC_ENC_Mode_COST[slice_type][0] * had_bias), 0x6f);
3590         mode_cost[1] = map_44_lut_value((unsigned int)(lambda_md * GEN9_HEVC_ENC_Mode_COST[slice_type][1] * had_bias), 0x8f);
3591         mode_cost[2] = map_44_lut_value((unsigned int)(lambda_md * GEN9_HEVC_ENC_Mode_COST[slice_type][2] * had_bias), 0x8f);
3592         mode_cost[3] = map_44_lut_value((unsigned int)(lambda_md * GEN9_HEVC_ENC_Mode_COST[slice_type][3] * had_bias), 0x8f);
3593         mode_cost[4] = map_44_lut_value((unsigned int)(lambda_md * GEN9_HEVC_ENC_Mode_COST[slice_type][4] * had_bias), 0x8f);
3594         mode_cost[5] = map_44_lut_value((unsigned int)(lambda_md * GEN9_HEVC_ENC_Mode_COST[slice_type][5] * had_bias), 0x6f);
3595         mode_cost[6] = map_44_lut_value((unsigned int)(lambda_md * GEN9_HEVC_ENC_Mode_COST[slice_type][6] * had_bias), 0x6f);
3596         mode_cost[7] = map_44_lut_value((unsigned int)(lambda_md * GEN9_HEVC_ENC_Mode_COST[slice_type][7] * had_bias), 0x6f);
3597         mode_cost[8] = map_44_lut_value((unsigned int)(lambda_md * GEN9_HEVC_ENC_Mode_COST[slice_type][8] * had_bias), 0x8f);
3598         mode_cost[9] = map_44_lut_value((unsigned int)(lambda_md * GEN9_HEVC_ENC_Mode_COST[slice_type][9] * had_bias), 0x6f);
3599         mode_cost[10] = map_44_lut_value((unsigned int)(lambda_md * GEN9_HEVC_ENC_Mode_COST[slice_type][10] * had_bias), 0x6f);
3600         mode_cost[11] = map_44_lut_value((unsigned int)(lambda_md * GEN9_HEVC_ENC_Mode_COST[slice_type][11] * had_bias), 0x6f);
3601     }
3602
3603     if (mv_cost) {
3604         mv_cost[0] = map_44_lut_value((unsigned int)(lambda_me * GEN9_HEVC_ENC_MV_COST[slice_type][0] * had_bias), 0x6f);
3605         mv_cost[1] = map_44_lut_value((unsigned int)(lambda_me * GEN9_HEVC_ENC_MV_COST[slice_type][1] * had_bias), 0x6f);
3606         mv_cost[2] = map_44_lut_value((unsigned int)(lambda_me * GEN9_HEVC_ENC_MV_COST[slice_type][2] * had_bias), 0x6f);
3607         mv_cost[3] = map_44_lut_value((unsigned int)(lambda_me * GEN9_HEVC_ENC_MV_COST[slice_type][3] * had_bias), 0x6f);
3608         mv_cost[4] = map_44_lut_value((unsigned int)(lambda_me * GEN9_HEVC_ENC_MV_COST[slice_type][4] * had_bias), 0x6f);
3609         mv_cost[5] = map_44_lut_value((unsigned int)(lambda_me * GEN9_HEVC_ENC_MV_COST[slice_type][5] * had_bias), 0x6f);
3610         mv_cost[6] = map_44_lut_value((unsigned int)(lambda_me * GEN9_HEVC_ENC_MV_COST[slice_type][6] * had_bias), 0x6f);
3611         mv_cost[7] = map_44_lut_value((unsigned int)(lambda_me * GEN9_HEVC_ENC_MV_COST[slice_type][7] * had_bias), 0x6f);
3612     }
3613
3614     if (mode_cost_sp)
3615         *mode_cost_sp = map_44_lut_value((unsigned int)(lambda_md * 45 * had_bias), 0x8f);
3616
3617     if (simplest_intra_inter_threshold) {
3618         lambda_md *= had_bias;
3619         *simplest_intra_inter_threshold = 0;
3620         if (GEN9_HEVC_ENC_Mode_COST[slice_type][1] < GEN9_HEVC_ENC_Mode_COST[slice_type][3])
3621             *simplest_intra_inter_threshold = (unsigned int)(lambda_md *
3622                                                              (GEN9_HEVC_ENC_Mode_COST[slice_type][3] - GEN9_HEVC_ENC_Mode_COST[slice_type][1]) + 0.5);
3623     }
3624 }
3625
3626 static void
3627 gen9_hevc_set_lambda_tables(struct gen9_hevc_encoder_context *priv_ctx,
3628                             int slice_type,
3629                             int intra_trans_type)
3630 {
3631     if (slice_type != HEVC_SLICE_I) {
3632         if (priv_ctx->lambda_table_inited)
3633             return;
3634
3635         memcpy((void *)&priv_ctx->lambda_me_table[slice_type], &GEN9_HEVC_ENC_QP_LAMBDA_ME[slice_type],
3636                sizeof(GEN9_HEVC_ENC_QP_LAMBDA_ME[slice_type]));
3637         memcpy((void *)&priv_ctx->lambda_md_table[slice_type], &GEN9_HEVC_ENC_QP_LAMBDA_ME[slice_type],
3638                sizeof(GEN9_HEVC_ENC_QP_LAMBDA_ME[slice_type]));
3639     } else if (intra_trans_type != priv_ctx->lambda_intra_trans_type ||
3640                !priv_ctx->lambda_table_inited) {
3641         double temp = 0.0;
3642         double lambda = 0.0;
3643         int qp = 0;
3644
3645         for (qp = 0; qp < 52; qp++) {
3646             temp = (double)qp - 12;
3647             lambda = 0.85 * pow(2.0, temp / 3.0);
3648
3649             if ((intra_trans_type != HEVC_ENC_INTRA_TRANS_HAAR) &&
3650                 (intra_trans_type != HEVC_ENC_INTRA_TRANS_HADAMARD))
3651                 lambda *= 0.95;
3652
3653             priv_ctx->lambda_md_table[slice_type][qp] =
3654                 priv_ctx->lambda_me_table[slice_type][qp] = sqrt(lambda);
3655         }
3656
3657         priv_ctx->lambda_intra_trans_type = intra_trans_type;
3658     }
3659 }
3660
3661 static void
3662 gen9_hevc_lambda_tables_init(struct gen9_hevc_encoder_context *priv_ctx)
3663 {
3664     gen9_hevc_set_lambda_tables(priv_ctx, HEVC_SLICE_B, HEVC_ENC_INTRA_TRANS_HAAR);
3665     gen9_hevc_set_lambda_tables(priv_ctx, HEVC_SLICE_P, HEVC_ENC_INTRA_TRANS_HAAR);
3666     gen9_hevc_set_lambda_tables(priv_ctx, HEVC_SLICE_I, HEVC_ENC_INTRA_TRANS_HAAR);
3667
3668     priv_ctx->lambda_table_inited = 1;
3669 }
3670
3671 static void
3672 gen9_hevc_8x8_b_pak_set_curbe(VADriverContextP ctx,
3673                               struct encode_state *encode_state,
3674                               struct intel_encoder_context *encoder_context,
3675                               struct i965_gpe_context *gpe_context)
3676 {
3677     struct i965_driver_data *i965 = i965_driver_data(ctx);
3678     struct encoder_vme_mfc_context *vme_context = NULL;
3679     struct generic_enc_codec_state *generic_state = NULL;
3680     struct gen9_hevc_encoder_state *priv_state = NULL;
3681     gen9_hevc_mbenc_b_pak_curbe_data *cmd = NULL;
3682     VAEncPictureParameterBufferHEVC *pic_param = NULL;
3683     VAEncSliceParameterBufferHEVC *slice_param = NULL;
3684     unsigned int slice_qp = 0;
3685     int bti_idx = 0;
3686
3687     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
3688     generic_state = (struct generic_enc_codec_state *)vme_context->generic_enc_state;
3689     priv_state = (struct gen9_hevc_encoder_state *)vme_context->private_enc_state;
3690     pic_param = (VAEncPictureParameterBufferHEVC *)encode_state->pic_param_ext->buffer;
3691     slice_param = (VAEncSliceParameterBufferHEVC *)encode_state->slice_params_ext[0]->buffer;
3692
3693     slice_qp = pic_param->pic_init_qp + slice_param->slice_qp_delta;
3694     cmd = i965_gpe_context_map_curbe(gpe_context);
3695     if (!cmd)
3696         return;
3697
3698     memset((void *)cmd, 0, sizeof(*cmd));
3699
3700     cmd->dw0.frame_width = ALIGN(priv_state->picture_width, 16);
3701     cmd->dw0.frame_height = ALIGN(priv_state->picture_height, 16);
3702     cmd->dw1.max_vmvr = 511 * 4;
3703     cmd->dw1.qp = slice_qp;
3704     cmd->dw2.brc_enable = generic_state->brc_enabled;
3705     cmd->dw2.lcu_brc_enable = priv_state->lcu_brc_enabled;
3706     cmd->dw2.screen_content = !!pic_param->pic_fields.bits.screen_content_flag;
3707     cmd->dw2.slice_type = priv_state->picture_coding_type;
3708     cmd->dw2.roi_enable = (priv_state->num_roi > 0);
3709     cmd->dw2.fast_surveillance_flag = priv_state->picture_coding_type == HEVC_SLICE_I ?
3710                                       0 : priv_state->video_surveillance_flag;
3711     cmd->dw2.kbl_control_flag = (IS_KBL(i965->intel.device_info) || IS_GLK(i965->intel.device_info));
3712     cmd->dw2.enable_rolling_intra = priv_state->rolling_intra_refresh;
3713     cmd->dw3.widi_intra_refresh_qp_delta = priv_state->widi_intra_refresh_qp_delta;
3714     cmd->dw3.widi_intra_refresh_mb_num = priv_state->widi_intra_insertion_location;
3715     cmd->dw3.widi_intra_refresh_unit_in_mb = priv_state->widi_intra_insertion_size;
3716
3717     cmd->dw16.bti_cu_record = bti_idx++;
3718     cmd->dw17.bti_pak_obj = bti_idx++;
3719     cmd->dw18.bti_slice_map = bti_idx++;
3720     cmd->dw19.bti_brc_input = bti_idx++;
3721     cmd->dw20.bti_lcu_qp = bti_idx++;
3722     cmd->dw21.bti_brc_data = bti_idx++;
3723     cmd->dw22.bti_mb_data = bti_idx++;
3724     cmd->dw23.bti_mvp_surface = bti_idx++;
3725     cmd->dw24.bti_debug = bti_idx++;
3726
3727     i965_gpe_context_unmap_curbe(gpe_context);
3728 }
3729
3730 static void
3731 gen9_hevc_8x8_b_pak_set_surfaces(VADriverContextP ctx,
3732                                  struct encode_state *encode_state,
3733                                  struct intel_encoder_context *encoder_context,
3734                                  struct i965_gpe_context *gpe_context)
3735 {
3736     struct encoder_vme_mfc_context *vme_context = NULL;
3737     struct gen9_hevc_encoder_context *priv_ctx = NULL;
3738     struct gen9_hevc_encoder_state *priv_state = NULL;
3739     int bti_idx = 0;
3740     int size = 0;
3741
3742     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
3743     priv_ctx = (struct gen9_hevc_encoder_context *)vme_context->private_enc_ctx;
3744     priv_state = (struct gen9_hevc_encoder_state *)vme_context->private_enc_state;
3745
3746     size = priv_state->width_in_cu * priv_state->height_in_cu *
3747            priv_state->cu_record_size;
3748     gen9_hevc_set_gpe_1d_surface(ctx, priv_ctx, gpe_context,
3749                                  HEVC_ENC_SURFACE_CU_RECORD, bti_idx++,
3750                                  0, size, priv_state->mb_data_offset, NULL, NULL);
3751
3752     size = priv_state->mb_data_offset;
3753     gen9_hevc_set_gpe_1d_surface(ctx, priv_ctx, gpe_context,
3754                                  HEVC_ENC_SURFACE_HCP_PAK, bti_idx++,
3755                                  0, size, 0, NULL, NULL);
3756
3757     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
3758                                  HEVC_ENC_SURFACE_SLICE_MAP, bti_idx++,
3759                                  0, 1, I965_SURFACEFORMAT_R8_UNORM,
3760                                  NULL, NULL);
3761
3762     gen9_hevc_set_gpe_1d_surface(ctx, priv_ctx, gpe_context,
3763                                  HEVC_ENC_SURFACE_BRC_INPUT, bti_idx++,
3764                                  0, 0, 0, NULL, NULL);
3765
3766     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
3767                                  HEVC_ENC_SURFACE_LCU_QP, bti_idx++,
3768                                  0, 1, I965_SURFACEFORMAT_R8_UNORM,
3769                                  NULL, NULL);
3770
3771     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
3772                                  HEVC_ENC_SURFACE_BRC_DATA, bti_idx++,
3773                                  0, 1, I965_SURFACEFORMAT_R8_UNORM,
3774                                  NULL, NULL);
3775
3776     gen9_hevc_set_gpe_1d_surface(ctx, priv_ctx, gpe_context,
3777                                  HEVC_ENC_SURFACE_MB_MV_INDEX, bti_idx++,
3778                                  0, 0, 0, NULL, NULL);
3779
3780     gen9_hevc_set_gpe_1d_surface(ctx, priv_ctx, gpe_context,
3781                                  HEVC_ENC_SURFACE_MVP_INDEX, bti_idx++,
3782                                  0, 0, 0, NULL, NULL);
3783
3784     gen9_hevc_set_gpe_1d_surface(ctx, priv_ctx, gpe_context,
3785                                  HEVC_ENC_SURFACE_KERNEL_DEBUG, bti_idx++,
3786                                  0, 0, 0, NULL, NULL);
3787 }
3788
3789 static void
3790 gen9_hevc_8x8_b_pak(VADriverContextP ctx,
3791                     struct encode_state *encode_state,
3792                     struct intel_encoder_context *encoder_context)
3793 {
3794     struct encoder_vme_mfc_context *vme_context = NULL;
3795     struct gen9_hevc_encoder_context *priv_ctx = NULL;
3796     struct gen9_hevc_encoder_state *priv_state = NULL;
3797     struct gpe_media_object_walker_parameter param;
3798     struct hevc_enc_kernel_walker_parameter hevc_walker_param;
3799     struct i965_gpe_context *gpe_context = NULL;
3800     int media_state = HEVC_ENC_MEDIA_STATE_HEVC_B_PAK;
3801
3802     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
3803     priv_ctx = (struct gen9_hevc_encoder_context *)vme_context->private_enc_ctx;
3804     priv_state = (struct gen9_hevc_encoder_state *)vme_context->private_enc_state;
3805
3806     gpe_context = &priv_ctx->mbenc_context.gpe_contexts[HEVC_MBENC_BPAK_IDX];
3807
3808     gen8_gpe_context_init(ctx, gpe_context);
3809     gen9_gpe_reset_binding_table(ctx, gpe_context);
3810     gen9_hevc_8x8_b_pak_set_curbe(ctx, encode_state, encoder_context, gpe_context);
3811     gen9_hevc_8x8_b_pak_set_surfaces(ctx, encode_state, encoder_context, gpe_context);
3812     gen8_gpe_setup_interface_data(ctx, gpe_context);
3813
3814     memset((void *)&hevc_walker_param, 0, sizeof(hevc_walker_param));
3815     hevc_walker_param.resolution_x = ALIGN(priv_state->picture_width, 32) >> 5;
3816     hevc_walker_param.resolution_y = ALIGN(priv_state->picture_height, 32) >> 5;
3817     hevc_walker_param.no_dependency = 1;
3818     gen9_hevc_init_object_walker(&hevc_walker_param, &param);
3819     gen9_hevc_run_object_walker(ctx, encoder_context, gpe_context, &param,
3820                                 media_state);
3821 }
3822
3823 static const unsigned char ftq_25i[27] = {
3824     0, 0, 0, 0,
3825     1, 3, 6, 8, 11,
3826     13, 16, 19, 22, 26,
3827     30, 34, 39, 44, 50,
3828     56, 62, 69, 77, 85,
3829     94, 104, 115
3830 };
3831
3832 static void
3833 gen9_hevc_set_forward_coeff_thd(unsigned char *pcoeff,
3834                                 int qp)
3835 {
3836     int idx = (qp + 1) >> 1;
3837
3838     memset((void *)pcoeff, ftq_25i[idx], 7);
3839 }
3840
3841 static int
3842 gen9_hevc_get_qp_from_ref_list(VADriverContextP ctx,
3843                                VAEncSliceParameterBufferHEVC *slice_param,
3844                                int list_idx,
3845                                int ref_frame_idx)
3846 {
3847     struct i965_driver_data *i965 = i965_driver_data(ctx);
3848     struct gen9_hevc_surface_priv *hevc_priv_surface = NULL;
3849     struct object_surface *obj_surface = NULL;
3850     VASurfaceID surface_id;
3851
3852     if (list_idx == 0) {
3853         if (ref_frame_idx < slice_param->num_ref_idx_l0_active_minus1 + 1)
3854             surface_id = slice_param->ref_pic_list0[ref_frame_idx].picture_id;
3855         else
3856             goto FAIL;
3857     } else {
3858         if (ref_frame_idx < slice_param->num_ref_idx_l1_active_minus1 + 1)
3859             surface_id = slice_param->ref_pic_list1[ref_frame_idx].picture_id;
3860         else
3861             goto FAIL;
3862     }
3863
3864     obj_surface = SURFACE(surface_id);
3865     if (obj_surface && obj_surface->private_data) {
3866         hevc_priv_surface = obj_surface->private_data;
3867         return hevc_priv_surface->qp_value;
3868     }
3869
3870 FAIL:
3871     return 0;
3872 }
3873
3874 static short
3875 gen9_hevc_get_poc_diff_from_ref_list(VAEncPictureParameterBufferHEVC *pic_param,
3876                                      VAEncSliceParameterBufferHEVC *slice_param,
3877                                      int list_idx,
3878                                      int ref_frame_idx)
3879 {
3880     short poc_diff = 0;
3881
3882     if (list_idx == 0 &&
3883         ref_frame_idx < slice_param->num_ref_idx_l0_active_minus1 + 1)
3884         poc_diff = pic_param->decoded_curr_pic.pic_order_cnt -
3885                    slice_param->ref_pic_list0[ref_frame_idx].pic_order_cnt;
3886     else if (list_idx == 1 &&
3887              ref_frame_idx < slice_param->num_ref_idx_l1_active_minus1 + 1)
3888         poc_diff = pic_param->decoded_curr_pic.pic_order_cnt -
3889                    slice_param->ref_pic_list1[ref_frame_idx].pic_order_cnt;
3890
3891     return CLAMP(-128, 127, poc_diff);
3892 }
3893
3894 static void
3895 gen9_hevc_set_control_region(VADriverContextP ctx,
3896                              struct encode_state *encode_state,
3897                              struct intel_encoder_context *encoder_context,
3898                              struct i965_gpe_context *gpe_context,
3899                              struct gen9_hevc_walking_pattern_parameter *param)
3900 {
3901     struct encoder_vme_mfc_context *vme_context = NULL;
3902     struct gen9_hevc_encoder_context *priv_ctx = NULL;
3903     struct gen9_hevc_encoder_state *priv_state = NULL;
3904     VAEncSliceParameterBufferHEVC *slice_param = NULL;
3905     gen9_hevc_mbenc_control_region *p_region = NULL;
3906     unsigned int slice, num_regions, height, num_slices, num_units_in_region;
3907     unsigned int frame_width_in_units, frame_height_in_units;
3908     unsigned short region_start_table[64];
3909     unsigned int offset_to_the_region_start[16];
3910     unsigned short temp_data[32][32];
3911     int is_arbitrary_slices = 0;
3912     int slice_start_y[NUM_SLICES + 1];
3913     int max_height;
3914     int k = 0, i = 0;
3915
3916     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
3917     priv_ctx = (struct gen9_hevc_encoder_context *)vme_context->private_enc_ctx;
3918     priv_state = (struct gen9_hevc_encoder_state *)vme_context->private_enc_state;
3919
3920     memset(slice_start_y, 0, sizeof(int) * (NUM_SLICES + 1));
3921     memset(region_start_table, 0, sizeof(region_start_table));
3922     memset(temp_data, 0, sizeof(temp_data));
3923     memset(offset_to_the_region_start, 0, sizeof(offset_to_the_region_start));
3924
3925     if (priv_state->num_regions_in_slice < 1)
3926         priv_state->num_regions_in_slice = 1;
3927
3928     if (priv_state->num_regions_in_slice > 16)
3929         priv_state->num_regions_in_slice = 16;
3930
3931     if (priv_state->walking_pattern_26) {
3932         frame_width_in_units = ALIGN(priv_state->picture_width, 16) / 16;
3933         frame_height_in_units = ALIGN(priv_state->picture_height, 16) / 16;
3934     } else {
3935         frame_width_in_units = ALIGN(priv_state->picture_width, 32) / 32;
3936         frame_height_in_units = ALIGN(priv_state->picture_height, 32) / 32;
3937     }
3938
3939     for (slice = 0; slice < encode_state->num_slice_params_ext; slice++) {
3940         slice_param = (VAEncSliceParameterBufferHEVC *)encode_state->slice_params_ext[slice]->buffer;
3941         if (slice_param->slice_segment_address %
3942             ALIGN(priv_state->picture_width, 32)) {
3943             is_arbitrary_slices = 1;
3944         } else {
3945             slice_start_y[slice] = slice_param->slice_segment_address /
3946                                    ALIGN(priv_state->picture_width, 32);
3947
3948             if (priv_state->walking_pattern_26) {
3949                 slice_start_y[slice] *= 2;
3950             }
3951         }
3952     }
3953
3954     slice_start_y[encode_state->num_slice_params_ext] = frame_height_in_units;
3955
3956     region_start_table[0] = 0;
3957     region_start_table[GEN9_HEVC_ENC_REGION_START_Y_OFFSET + 0] = 0;
3958     num_regions = 1;
3959
3960     if (is_arbitrary_slices) {
3961         height = frame_height_in_units;
3962         num_slices = 1;
3963         max_height = height;
3964         if (priv_state->num_regions_in_slice > 1) {
3965             num_units_in_region =
3966                 (frame_width_in_units + 2 * (frame_height_in_units - 1) + priv_state->num_regions_in_slice - 1) / priv_state->num_regions_in_slice;
3967
3968             num_regions = priv_state->num_regions_in_slice;
3969
3970             for (i = 1; i < priv_state->num_regions_in_slice; i++) {
3971                 unsigned int front = i * num_units_in_region;
3972
3973                 if (front < frame_width_in_units)
3974                     region_start_table[i] = (unsigned int)front;
3975                 else if (((front - frame_width_in_units + 1) & 1) == 0)
3976                     region_start_table[i] = (unsigned int)frame_width_in_units - 1;
3977                 else
3978                     region_start_table[i] = (unsigned int)frame_width_in_units - 2;
3979
3980                 region_start_table[GEN9_HEVC_ENC_REGION_START_Y_OFFSET + i] = (unsigned int)((front - region_start_table[i]) >> 1);
3981             }
3982         }
3983     } else {
3984         int start_y;
3985         int slice_is_merged = 0;
3986
3987         max_height = 0;
3988         num_slices = encode_state->num_slice_params_ext;
3989
3990         for (slice = 0; slice < num_slices; slice++) {
3991             int sliceHeight = slice_start_y[slice + 1] - slice_start_y[slice];
3992
3993             if (sliceHeight > max_height)
3994                 max_height = sliceHeight;
3995         }
3996
3997         while (!slice_is_merged) {
3998             int newNumSlices = 1;
3999
4000             start_y = 0;
4001
4002             for (slice = 1; slice < num_slices; slice++) {
4003                 if ((slice_start_y[slice + 1] - start_y) <= max_height)
4004                     slice_start_y[slice] = -1;
4005                 else
4006                     start_y = slice_start_y[slice];
4007             }
4008
4009             for (slice = 1; slice < num_slices; slice++) {
4010                 if (slice_start_y[slice] > 0) {
4011                     slice_start_y[newNumSlices] = slice_start_y[slice];
4012                     newNumSlices++;
4013                 }
4014             }
4015
4016             num_slices = newNumSlices;
4017             slice_start_y[num_slices] = frame_height_in_units;
4018
4019             if (num_slices * priv_state->num_regions_in_slice <= GEN9_HEVC_MEDIA_WALKER_MAX_COLORS)
4020                 slice_is_merged = 1;
4021             else {
4022                 int num = 1;
4023
4024                 max_height = frame_height_in_units;
4025
4026                 for (slice = 0; slice < num_slices - 1; slice++) {
4027                     if ((slice_start_y[slice + 2] - slice_start_y[slice]) <= max_height) {
4028                         max_height = slice_start_y[slice + 2] - slice_start_y[slice];
4029                         num = slice + 1;
4030                     }
4031                 }
4032
4033                 for (slice = num; slice < num_slices; slice++)
4034                     slice_start_y[slice] = slice_start_y[slice + 1];
4035
4036                 num_slices--;
4037             }
4038         }
4039
4040         num_units_in_region = (frame_width_in_units + 2 * (max_height - 1) + priv_state->num_regions_in_slice - 1) /
4041                               priv_state->num_regions_in_slice;
4042         num_regions = num_slices * priv_state->num_regions_in_slice;
4043
4044         for (slice = 0; slice < num_slices; slice++) {
4045             region_start_table[slice * priv_state->num_regions_in_slice] = 0;
4046             region_start_table[GEN9_HEVC_ENC_REGION_START_Y_OFFSET + (slice * priv_state->num_regions_in_slice)] = (unsigned int)slice_start_y[slice];
4047
4048             for (i = 1; i < priv_state->num_regions_in_slice; i++) {
4049                 int front = i * num_units_in_region;
4050
4051                 if ((unsigned int)front < frame_width_in_units)
4052                     region_start_table[slice * priv_state->num_regions_in_slice + i] = (unsigned int)front;
4053                 else if (((front - frame_width_in_units + 1) & 1) == 0)
4054                     region_start_table[slice * priv_state->num_regions_in_slice + i] = (unsigned int)frame_width_in_units - 1;
4055                 else
4056                     region_start_table[slice * priv_state->num_regions_in_slice + i] = (unsigned int)frame_width_in_units - 2;
4057
4058                 region_start_table[GEN9_HEVC_ENC_REGION_START_Y_OFFSET + (slice * priv_state->num_regions_in_slice + i)] = (unsigned int)slice_start_y[slice] +
4059                                                                                                                            ((front - region_start_table[i]) >> 1);
4060             }
4061         }
4062         height = max_height;
4063     }
4064
4065     for (k = 0; k < num_slices; k++) {
4066         int i;
4067         int nearest_reg = 0;
4068         int min_delta = priv_state->picture_height;
4069
4070         if (priv_state->walking_pattern_26) {
4071             int cur_lcu_pel_y = region_start_table[GEN9_HEVC_ENC_REGION_START_Y_OFFSET + (k * priv_state->num_regions_in_slice)] << 4;
4072             int ts_width = priv_state->picture_width >> 4;
4073             int ts_height = height;
4074             int offset_y = -((ts_width + 1) >> 1);
4075             int offset_delta = ((ts_width + ((ts_height - 1) << 1)) + (priv_state->num_regions_in_slice - 1)) / (priv_state->num_regions_in_slice);
4076
4077             for (i = 0; i < (int)num_regions; i++) {
4078                 if (region_start_table[i] == 0) {
4079                     int delta = cur_lcu_pel_y - (region_start_table[GEN9_HEVC_ENC_REGION_START_Y_OFFSET + i] << 4);
4080
4081                     if (delta >= 0) {
4082                         if (delta < min_delta) {
4083                             min_delta = delta;
4084                             nearest_reg = i;
4085                         }
4086                     }
4087                 }
4088
4089                 offset_to_the_region_start[k] = 2 * region_start_table[GEN9_HEVC_ENC_REGION_START_Y_OFFSET + nearest_reg];
4090             }
4091
4092             for (i = 0; i < priv_state->num_regions_in_slice; i++) {
4093                 int tmp_y = region_start_table[GEN9_HEVC_ENC_REGION_START_Y_OFFSET + (nearest_reg + priv_state->num_regions_in_slice)];
4094
4095                 temp_data[k * priv_state->num_regions_in_slice + i][0] = region_start_table[nearest_reg + i];
4096                 temp_data[k * priv_state->num_regions_in_slice + i][1] = region_start_table[GEN9_HEVC_ENC_REGION_START_Y_OFFSET + (nearest_reg + i)];
4097                 temp_data[k * priv_state->num_regions_in_slice + i][2] = region_start_table[GEN9_HEVC_ENC_REGION_START_Y_OFFSET + nearest_reg];
4098                 temp_data[k * priv_state->num_regions_in_slice + i][3] = (unsigned int)((tmp_y != 0) ? tmp_y : (priv_state->picture_height) >> 4);
4099                 temp_data[k * priv_state->num_regions_in_slice + i][4] = offset_to_the_region_start[k] & 0x0FFFF;
4100                 temp_data[k * priv_state->num_regions_in_slice + i][5] = 0;
4101                 temp_data[k * priv_state->num_regions_in_slice + i][6] = 0;
4102                 temp_data[k * priv_state->num_regions_in_slice + i][7] = (unsigned int)(offset_y + region_start_table[GEN9_HEVC_ENC_REGION_START_Y_OFFSET + nearest_reg] + ((i * offset_delta) >> 1));
4103             }
4104         } else {
4105             int cur_lcu_pel_y = region_start_table[GEN9_HEVC_ENC_REGION_START_Y_OFFSET + (k * priv_state->num_regions_in_slice)] << 5;
4106             int ts_width = (priv_state->picture_width + 16) >> 5;
4107             int ts_height = height;
4108             int offset_y = -4 * ((ts_width + 1) >> 1);
4109             int offset_delta = ((ts_width + ((ts_height - 1) << 1)) + (priv_state->num_regions_in_slice - 1)) / (priv_state->num_regions_in_slice);
4110
4111             for (i = 0; i < (int)num_regions; i++) {
4112                 if (region_start_table[i] == 0) {
4113                     int delta = cur_lcu_pel_y - (region_start_table[GEN9_HEVC_ENC_REGION_START_Y_OFFSET + i] << 5);
4114
4115                     if (delta >= 0) {
4116                         if (delta < min_delta) {
4117                             min_delta = delta;
4118                             nearest_reg = i;
4119                         }
4120                     }
4121                 }
4122
4123                 offset_to_the_region_start[k] = 2 * region_start_table[GEN9_HEVC_ENC_REGION_START_Y_OFFSET + nearest_reg];
4124             }
4125
4126             for (i = 0; i < priv_state->num_regions_in_slice; i++) {
4127                 int tmp_y = 2 * region_start_table[GEN9_HEVC_ENC_REGION_START_Y_OFFSET + (nearest_reg + priv_state->num_regions_in_slice)];
4128
4129                 temp_data[k * priv_state->num_regions_in_slice + i][0] = region_start_table[nearest_reg + i];
4130                 temp_data[k * priv_state->num_regions_in_slice + i][1] = 2 * region_start_table[GEN9_HEVC_ENC_REGION_START_Y_OFFSET + (nearest_reg + i)];
4131                 temp_data[k * priv_state->num_regions_in_slice + i][2] = 2 * region_start_table[GEN9_HEVC_ENC_REGION_START_Y_OFFSET + nearest_reg];
4132                 temp_data[k * priv_state->num_regions_in_slice + i][3] = (unsigned int)((tmp_y != 0) ? tmp_y : (priv_state->picture_height) >> 4);
4133                 temp_data[k * priv_state->num_regions_in_slice + i][4] = offset_to_the_region_start[k] & 0x0FFFF;
4134                 temp_data[k * priv_state->num_regions_in_slice + i][5] = 0;
4135                 temp_data[k * priv_state->num_regions_in_slice + i][6] = 0;
4136                 temp_data[k * priv_state->num_regions_in_slice + i][7] = (unsigned int)(offset_y + 4 * region_start_table[GEN9_HEVC_ENC_REGION_START_Y_OFFSET + nearest_reg] + (4 * ((i * offset_delta) >> 1)));
4137             }
4138         }
4139     }
4140
4141     if (priv_state->walking_pattern_26)
4142         gen9_hevc_init_object_walker_26(priv_state, gpe_context, &param->gpe_param,
4143                                         priv_state->num_regions_in_slice, max_height,
4144                                         priv_state->use_hw_scoreboard,
4145                                         priv_state->use_hw_non_stalling_scoreborad);
4146     else
4147         gen9_hevc_init_object_walker_26z(priv_state, gpe_context, &param->gpe_param,
4148                                          priv_state->num_regions_in_slice, max_height,
4149                                          priv_state->use_hw_scoreboard,
4150                                          priv_state->use_hw_non_stalling_scoreborad);
4151
4152     p_region = (gen9_hevc_mbenc_control_region *)i965_map_gpe_resource(&priv_ctx->res_con_corrent_thread_buffer);
4153     if (!p_region)
4154         return;
4155
4156     memset((void *)p_region, 0, sizeof(*p_region) * GEN9_HEVC_ENC_CONCURRENT_SURFACE_HEIGHT);
4157
4158     for (i = 0; i < 1024 ; i += 64)
4159         memcpy(((unsigned char *)p_region) + i, (unsigned char *)temp_data[i / 64], 32);
4160
4161     param->max_height_in_region = priv_state->walking_pattern_26 ? max_height : max_height * 2;;
4162     param->num_region = num_regions;
4163     param->num_units_in_region = (frame_width_in_units + 2 * (max_height - 1) + priv_state->num_regions_in_slice - 1) /
4164                                  priv_state->num_regions_in_slice;
4165
4166     i965_unmap_gpe_resource(&priv_ctx->res_con_corrent_thread_buffer);
4167 }
4168
4169 static const char hevc_qpc_table[22] = {
4170     29, 30, 31, 32, 32, 33, 34, 34, 35, 35, 36, 36, 37, 37, 37, 38, 38, 38, 39, 39, 39, 39
4171 };
4172
4173 static void
4174 gen9_hevc_8x8_b_mbenc_set_curbe(VADriverContextP ctx,
4175                                 struct encode_state *encode_state,
4176                                 struct intel_encoder_context *encoder_context,
4177                                 struct i965_gpe_context *gpe_context,
4178                                 struct gen9_hevc_walking_pattern_parameter *param)
4179 {
4180     struct encoder_vme_mfc_context *vme_context = NULL;
4181     struct gen9_hevc_encoder_context *priv_ctx = NULL;
4182     struct generic_enc_codec_state *generic_state = NULL;
4183     struct gen9_hevc_encoder_state *priv_state = NULL;
4184     gen9_hevc_mbenc_b_mb_enc_curbe_data *cmd = NULL;
4185     VAEncSequenceParameterBufferHEVC *seq_param = NULL;
4186     VAEncPictureParameterBufferHEVC *pic_param = NULL;
4187     VAEncSliceParameterBufferHEVC *slice_param = NULL;
4188     unsigned int slice_qp = 0, slice_type = HEVC_SLICE_I;
4189     unsigned char mode_cost[12], mv_cost[8], mode_cost_sp;
4190     unsigned char forward_trans_thd[7];
4191     unsigned int simplest_intra_inter_threshold;
4192     int transform_8x8_mode_flag = 1;
4193     int qp_bd_offset_c, q_pi, qp_c;
4194     void *default_curbe_ptr = NULL;
4195     int default_curbe_size = 0;
4196     int max_sp_len = 57;
4197     int bti_idx = 0;
4198
4199     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
4200     priv_ctx = (struct gen9_hevc_encoder_context *)vme_context->private_enc_ctx;
4201     generic_state = (struct generic_enc_codec_state *)vme_context->generic_enc_state;
4202     priv_state = (struct gen9_hevc_encoder_state *)vme_context->private_enc_state;
4203     seq_param = (VAEncSequenceParameterBufferHEVC *)encode_state->seq_param_ext->buffer;
4204     pic_param = (VAEncPictureParameterBufferHEVC *)encode_state->pic_param_ext->buffer;
4205     slice_param = (VAEncSliceParameterBufferHEVC *)encode_state->slice_params_ext[0]->buffer;
4206
4207     slice_qp = pic_param->pic_init_qp + slice_param->slice_qp_delta;
4208     slice_type = priv_state->picture_coding_type;
4209
4210     if (priv_state->tu_mode == HEVC_TU_BEST_SPEED) {
4211         gen9_hevc_set_lambda_tables(priv_ctx, slice_type,
4212                                     HEVC_ENC_INTRA_TRANS_HAAR);
4213
4214         max_sp_len = 25;
4215     }
4216
4217     gen9_hevc_mbenc_set_costs(priv_ctx, slice_type, HEVC_ENC_INTRA_TRANS_REGULAR, slice_qp,
4218                               mode_cost, mv_cost, &mode_cost_sp,
4219                               &simplest_intra_inter_threshold);
4220
4221     gen9_hevc_set_forward_coeff_thd(forward_trans_thd, slice_qp);
4222
4223     gen9_hevc_get_b_mbenc_default_curbe(priv_state->tu_mode,
4224                                         slice_type, &default_curbe_ptr,
4225                                         &default_curbe_size);
4226
4227     cmd = i965_gpe_context_map_curbe(gpe_context);
4228     if (!cmd)
4229         return;
4230
4231     memcpy((void *)cmd, default_curbe_ptr, default_curbe_size);
4232
4233     cmd->dw0.adaptive_en = 1;
4234     cmd->dw0.t_8x8_flag_for_inter_en = transform_8x8_mode_flag;
4235     cmd->dw2.pic_width = priv_state->width_in_mb;
4236     cmd->dw2.len_sp = max_sp_len;
4237     cmd->dw3.src_access = 0;
4238     cmd->dw3.ref_access = 0;
4239     cmd->dw3.ft_enable = (HEVC_ENC_FTQ_BASED_SKIP[priv_state->tu_mode] >> 1) & 0x01;
4240     cmd->dw4.pic_height_minus1 = priv_state->height_in_mb - 1;
4241     cmd->dw4.hme_enable = !!generic_state->hme_enabled;
4242     cmd->dw4.slice_type = slice_type;
4243     cmd->dw4.use_actual_ref_qp_value = 0;
4244     cmd->dw6.frame_width = priv_state->width_in_mb * 16;
4245     cmd->dw6.frame_height = priv_state->height_in_mb * 16;
4246     cmd->dw7.intra_part_mask = 3;
4247     cmd->dw8.mode0_cost = mode_cost[0];
4248     cmd->dw8.mode1_cost = mode_cost[1];
4249     cmd->dw8.mode2_cost = mode_cost[2];
4250     cmd->dw8.mode3_cost = mode_cost[3];
4251     cmd->dw9.mode4_cost = mode_cost[4];
4252     cmd->dw9.mode5_cost = mode_cost[5];
4253     cmd->dw9.mode6_cost = mode_cost[6];
4254     cmd->dw9.mode7_cost = mode_cost[7];
4255     cmd->dw10.mode8_cost = mode_cost[8];
4256     cmd->dw10.mode9_cost = mode_cost[9];
4257     cmd->dw10.ref_id_cost = mode_cost[10];
4258     cmd->dw10.chroma_intra_mode_cost = mode_cost[11];
4259     cmd->dw11.mv0_cost = mv_cost[0];
4260     cmd->dw11.mv1_cost = mv_cost[1];
4261     cmd->dw11.mv2_cost = mv_cost[2];
4262     cmd->dw11.mv3_cost = mv_cost[3];
4263     cmd->dw12.mv4_cost = mv_cost[4];
4264     cmd->dw12.mv5_cost = mv_cost[5];
4265     cmd->dw12.mv6_cost = mv_cost[6];
4266     cmd->dw12.mv7_cost = mv_cost[7];
4267
4268     cmd->dw13.qp_prime_y = slice_qp;
4269     qp_bd_offset_c = 6 * priv_state->bit_depth_chroma_minus8;
4270     q_pi = CLAMP(-qp_bd_offset_c, 51, slice_qp + pic_param->pps_cb_qp_offset);
4271     qp_c = (q_pi < 30) ? q_pi : hevc_qpc_table[q_pi - 30];
4272     cmd->dw13.qp_prime_cb = qp_c;
4273     q_pi = CLAMP(-qp_bd_offset_c, 51, slice_qp + pic_param->pps_cr_qp_offset);
4274     qp_c = (q_pi < 30) ? q_pi : hevc_qpc_table[q_pi - 30];
4275     cmd->dw13.qp_prime_cr = qp_c;
4276
4277     cmd->dw14.sic_fwd_trans_coeff_thread_0 = forward_trans_thd[0];
4278     cmd->dw14.sic_fwd_trans_coeff_thread_1 = forward_trans_thd[1];
4279     cmd->dw14.sic_fwd_trans_coeff_thread_2 = forward_trans_thd[2];
4280     cmd->dw15.sic_fwd_trans_coeff_thread_3 = forward_trans_thd[3];
4281     cmd->dw15.sic_fwd_trans_coeff_thread_4 = forward_trans_thd[4];
4282     cmd->dw15.sic_fwd_trans_coeff_thread_5 = forward_trans_thd[5];
4283     cmd->dw15.sic_fwd_trans_coeff_thread_6 = forward_trans_thd[6];
4284     cmd->dw32.skip_val =
4285         HEVC_ENC_SKIPVAL_B[cmd->dw3.block_based_skip_enable][transform_8x8_mode_flag][slice_qp];
4286
4287     if (priv_state->picture_coding_type == HEVC_SLICE_I)
4288         *(float *)&cmd->dw34.lambda_me = 0.0;
4289     else
4290         *(float *)&cmd->dw34.lambda_me = (float)priv_ctx->lambda_me_table[slice_type][slice_qp];
4291
4292     cmd->dw35.mode_cost_sp = mode_cost_sp;
4293     cmd->dw35.simp_intra_inter_threashold = simplest_intra_inter_threshold;
4294     cmd->dw36.num_refidx_l0_minus_one = slice_param->num_ref_idx_l0_active_minus1;
4295     cmd->dw36.num_refidx_l1_minus_one = slice_param->num_ref_idx_l1_active_minus1;
4296     cmd->dw36.brc_enable = !!generic_state->brc_enabled;
4297     cmd->dw36.lcu_brc_enable = !!priv_state->lcu_brc_enabled;
4298     cmd->dw36.power_saving = priv_state->power_saving;
4299     cmd->dw36.roi_enable = (priv_state->num_roi > 0);
4300     cmd->dw36.fast_surveillance_flag = priv_state->picture_coding_type == HEVC_SLICE_I ?
4301                                        0 : priv_state->video_surveillance_flag;
4302
4303     if (priv_state->picture_coding_type != HEVC_SLICE_I) {
4304         cmd->dw37.actual_qp_refid0_list0 = gen9_hevc_get_qp_from_ref_list(ctx, slice_param, 0, 0);
4305         cmd->dw37.actual_qp_refid1_list0 = gen9_hevc_get_qp_from_ref_list(ctx, slice_param, 0, 1);
4306         cmd->dw37.actual_qp_refid2_list0 = gen9_hevc_get_qp_from_ref_list(ctx, slice_param, 0, 2);
4307         cmd->dw37.actual_qp_refid3_list0 = gen9_hevc_get_qp_from_ref_list(ctx, slice_param, 0, 3);
4308
4309         if (priv_state->picture_coding_type == HEVC_SLICE_B) {
4310             cmd->dw39.actual_qp_refid0_list1 = gen9_hevc_get_qp_from_ref_list(ctx, slice_param, 1, 0);
4311             cmd->dw39.actual_qp_refid1_list1 = gen9_hevc_get_qp_from_ref_list(ctx, slice_param, 1, 1);
4312         }
4313     }
4314
4315     cmd->dw44.max_vmvr = 511 * 4;
4316     cmd->dw44.max_num_merge_candidates = slice_param->max_num_merge_cand;
4317
4318     if (priv_state->picture_coding_type != HEVC_SLICE_I) {
4319         cmd->dw44.max_num_ref_list0 = cmd->dw36.num_refidx_l0_minus_one + 1;
4320         cmd->dw44.max_num_ref_list1 = cmd->dw36.num_refidx_l1_minus_one + 1;
4321         cmd->dw45.temporal_mvp_enable_flag = slice_param->slice_fields.bits.slice_temporal_mvp_enabled_flag;
4322         cmd->dw45.hme_combine_len_pslice = 8;
4323         if (priv_state->picture_coding_type == HEVC_SLICE_B)
4324             cmd->dw45.hme_combine_len_bslice = 8;
4325     }
4326
4327     cmd->dw45.log2_parallel_merge_level = pic_param->log2_parallel_merge_level_minus2 + 2;
4328     cmd->dw46.log2_min_tu_size = seq_param->log2_min_transform_block_size_minus2 + 2;
4329     cmd->dw46.log2_max_tu_size = cmd->dw46.log2_min_tu_size + seq_param->log2_diff_max_min_transform_block_size;
4330     cmd->dw46.log2_min_cu_size = seq_param->log2_min_luma_coding_block_size_minus3 + 3;
4331     cmd->dw46.log2_max_cu_size = cmd->dw46.log2_min_cu_size + seq_param->log2_diff_max_min_luma_coding_block_size;
4332     cmd->dw47.num_regions_in_slice = priv_state->num_regions_in_slice;
4333     cmd->dw47.type_of_walking_pattern = priv_state->walking_pattern_26;
4334     cmd->dw47.chroma_flatness_check_flag = (priv_state->tu_mode == HEVC_TU_BEST_SPEED) ? 0 : 1;
4335     cmd->dw47.enable_intra_early_exit = (priv_state->tu_mode == HEVC_TU_RT_SPEED);
4336     cmd->dw47.skip_intra_krn_flag = (priv_state->tu_mode == HEVC_TU_BEST_SPEED);
4337     cmd->dw47.collocated_from_l0_flag = slice_param->slice_fields.bits.collocated_from_l0_flag;
4338     cmd->dw47.is_low_delay = priv_state->low_delay;
4339     cmd->dw47.screen_content_flag = !!pic_param->pic_fields.bits.screen_content_flag;
4340     cmd->dw47.multi_slice_flag = (encode_state->num_slice_params_ext > 1);
4341     cmd->dw47.arbitary_slice_flag = priv_state->arbitrary_num_mb_in_slice;
4342     cmd->dw47.num_region_minus1 = param->num_region - 1;
4343
4344     if (priv_state->picture_coding_type != HEVC_SLICE_I) {
4345         cmd->dw48.current_td_l0_0 = gen9_hevc_get_poc_diff_from_ref_list(pic_param, slice_param, 0, 0);
4346         cmd->dw48.current_td_l0_1 = gen9_hevc_get_poc_diff_from_ref_list(pic_param, slice_param, 0, 1);
4347         cmd->dw49.current_td_l0_2 = gen9_hevc_get_poc_diff_from_ref_list(pic_param, slice_param, 0, 2);
4348         cmd->dw49.current_td_l0_3 = gen9_hevc_get_poc_diff_from_ref_list(pic_param, slice_param, 0, 3);
4349
4350         if (priv_state->picture_coding_type == HEVC_SLICE_B) {
4351             cmd->dw50.current_td_l1_0 = gen9_hevc_get_poc_diff_from_ref_list(pic_param, slice_param, 1, 0);
4352             cmd->dw50.current_td_l1_1 = gen9_hevc_get_poc_diff_from_ref_list(pic_param, slice_param, 1, 1);
4353         }
4354     }
4355
4356     cmd->dw52.num_of_units_in_region = param->num_units_in_region;
4357     cmd->dw52.max_height_in_region = param->max_height_in_region;
4358
4359     if (priv_state->rolling_intra_refresh) {
4360         cmd->dw35.widi_intra_refresh_en = 1;
4361         cmd->dw35.widi_first_intra_refresh = priv_state->widi_first_intra_refresh;
4362         cmd->dw35.half_update_mixed_lcu = 0;
4363         cmd->dw35.enable_rolling_intra = 1;
4364         cmd->dw38.widi_num_frame_in_gob = priv_state->widi_frame_num_in_gob;
4365         cmd->dw38.widi_num_intra_refresh_off_frames = priv_state->widi_frame_num_without_intra_refresh;
4366         cmd->dw51.widi_intra_refresh_qp_delta = priv_state->widi_intra_refresh_qp_delta;
4367         cmd->dw51.widi_intra_refresh_mb_num = priv_state->widi_intra_insertion_location;
4368         cmd->dw51.widi_intra_refresh_unit_in_mb = priv_state->widi_intra_insertion_size;
4369         cmd->dw53.widi_intra_refresh_ref_height = 40;
4370         cmd->dw53.widi_intra_refresh_ref_width = 48;
4371
4372         priv_state->widi_first_intra_refresh = 0;
4373         priv_state->widi_frame_num_without_intra_refresh = 0;
4374     } else if (priv_state->picture_coding_type != HEVC_SLICE_I)
4375         priv_state->widi_frame_num_without_intra_refresh++;
4376
4377     cmd->dw56.bti_cu_record = bti_idx++;
4378     cmd->dw57.bti_pak_cmd = bti_idx++;
4379     cmd->dw58.bti_src_y = bti_idx++;
4380     bti_idx++;
4381     cmd->dw59.bti_intra_dist = bti_idx++;
4382     cmd->dw60.bti_min_dist = bti_idx++;
4383     cmd->dw61.bti_hme_mv_pred_fwd_bwd_surf_index = bti_idx++;
4384     cmd->dw62.bti_hme_dist_surf_index = bti_idx++;
4385     cmd->dw63.bti_slice_map = bti_idx++;
4386     cmd->dw64.bti_vme_saved_uni_sic = bti_idx++;
4387     cmd->dw65.bti_simplest_intra = bti_idx++;
4388     cmd->dw66.bti_collocated_refframe = bti_idx++;
4389     cmd->dw67.bti_reserved = bti_idx++;
4390     cmd->dw68.bti_brc_input = bti_idx++;
4391     cmd->dw69.bti_lcu_qp = bti_idx++;
4392     cmd->dw70.bti_brc_data = bti_idx++;
4393     cmd->dw71.bti_vme_inter_prediction_surf_index = bti_idx++;
4394     bti_idx += 16;
4395
4396     if (priv_state->picture_coding_type == HEVC_SLICE_P) {
4397         cmd->dw72.bti_concurrent_thread_map = bti_idx++;
4398         cmd->dw73.bti_mb_data_cur_frame = bti_idx++;
4399         cmd->dw74.bti_mvp_cur_frame = bti_idx++;
4400         cmd->dw75.bti_debug = bti_idx++;
4401     } else {
4402         cmd->dw72.bti_vme_inter_prediction_b_surf_index = bti_idx++;
4403         bti_idx += 8;
4404
4405         cmd->dw73.bti_concurrent_thread_map = bti_idx++;
4406         cmd->dw74.bti_mb_data_cur_frame = bti_idx++;
4407         cmd->dw75.bti_mvp_cur_frame = bti_idx++;
4408         cmd->dw76.bti_debug = bti_idx++;
4409     }
4410
4411     i965_gpe_context_unmap_curbe(gpe_context);
4412 }
4413
4414 static void
4415 gen9_hevc_8x8_b_mbenc_set_surfaces(VADriverContextP ctx,
4416                                    struct encode_state *encode_state,
4417                                    struct intel_encoder_context *encoder_context,
4418                                    struct i965_gpe_context *gpe_context)
4419 {
4420     struct i965_driver_data *i965 = i965_driver_data(ctx);
4421     struct encoder_vme_mfc_context *vme_context = NULL;
4422     struct gen9_hevc_encoder_context *priv_ctx = NULL;
4423     struct gen9_hevc_encoder_state *priv_state = NULL;
4424     VAEncSliceParameterBufferHEVC *slice_param = NULL;
4425     VAEncPictureParameterBufferHEVC *pic_param = NULL;
4426     struct gen9_hevc_surface_priv *surface_priv = NULL;
4427     struct object_surface *obj_surface = NULL;
4428     dri_bo *collocated_mv_temporal_bo = NULL;
4429     int bti_idx = 0;
4430     int size = 0;
4431     int i = 0;
4432
4433     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
4434     priv_ctx = (struct gen9_hevc_encoder_context *)vme_context->private_enc_ctx;
4435     priv_state = (struct gen9_hevc_encoder_state *)vme_context->private_enc_state;
4436     pic_param = (VAEncPictureParameterBufferHEVC *)encode_state->pic_param_ext->buffer;
4437     slice_param = (VAEncSliceParameterBufferHEVC *)encode_state->slice_params_ext[0]->buffer;
4438
4439     if (priv_state->picture_coding_type != HEVC_SLICE_I &&
4440         slice_param->slice_fields.bits.slice_temporal_mvp_enabled_flag &&
4441         slice_param->slice_fields.bits.collocated_from_l0_flag) {
4442         if (pic_param->collocated_ref_pic_index != 0xff &&
4443             pic_param->collocated_ref_pic_index < GEN9_MAX_REF_SURFACES) {
4444             VASurfaceID idx = VA_INVALID_SURFACE;
4445
4446             idx = pic_param->reference_frames[pic_param->collocated_ref_pic_index].picture_id;
4447             obj_surface = SURFACE(idx);
4448             if (obj_surface) {
4449                 surface_priv = (struct gen9_hevc_surface_priv *)obj_surface->private_data;
4450                 if (surface_priv)
4451                     collocated_mv_temporal_bo = surface_priv->motion_vector_temporal_bo;
4452             }
4453         }
4454     }
4455
4456     size = priv_state->width_in_cu * priv_state->height_in_cu *
4457            priv_state->cu_record_size;
4458     gen9_hevc_set_gpe_1d_surface(ctx, priv_ctx, gpe_context,
4459                                  HEVC_ENC_SURFACE_CU_RECORD, bti_idx++,
4460                                  0, size, priv_state->mb_data_offset, NULL, NULL);
4461
4462     size = priv_state->mb_data_offset;
4463     gen9_hevc_set_gpe_1d_surface(ctx, priv_ctx, gpe_context,
4464                                  HEVC_ENC_SURFACE_HCP_PAK, bti_idx++,
4465                                  0, size, 0, NULL, NULL);
4466
4467     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
4468                                  HEVC_ENC_SURFACE_RAW_Y_UV, bti_idx++,
4469                                  1, 1, I965_SURFACEFORMAT_R8_UNORM,
4470                                  NULL, NULL);
4471     bti_idx++;
4472
4473     gen9_hevc_set_gpe_1d_surface(ctx, priv_ctx, gpe_context,
4474                                  HEVC_ENC_SURFACE_INTRA_DIST, bti_idx++,
4475                                  0, 0, 0, NULL, NULL);
4476
4477     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
4478                                  HEVC_ENC_SURFACE_MIN_DIST, bti_idx++,
4479                                  0, 1, I965_SURFACEFORMAT_R8_UNORM,
4480                                  NULL, NULL);
4481
4482     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
4483                                  HEVC_ENC_SURFACE_HME_MVP, bti_idx++,
4484                                  0, 1, I965_SURFACEFORMAT_R8_UNORM,
4485                                  NULL, NULL);
4486
4487     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
4488                                  HEVC_ENC_SURFACE_HME_DIST, bti_idx++,
4489                                  0, 1, I965_SURFACEFORMAT_R8_UNORM,
4490                                  NULL, NULL);
4491
4492     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
4493                                  HEVC_ENC_SURFACE_SLICE_MAP, bti_idx++,
4494                                  0, 1, I965_SURFACEFORMAT_R8_UNORM,
4495                                  NULL, NULL);
4496
4497     gen9_hevc_set_gpe_1d_surface(ctx, priv_ctx, gpe_context,
4498                                  HEVC_ENC_SURFACE_VME_UNI_SIC_DATA, bti_idx++,
4499                                  0, 0, 0, NULL, NULL);
4500
4501     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
4502                                  HEVC_ENC_SURFACE_SIMPLIFIED_INTRA, bti_idx++,
4503                                  0, 1, I965_SURFACEFORMAT_R8_UNORM,
4504                                  NULL, NULL);
4505
4506     if (collocated_mv_temporal_bo)
4507         gen9_hevc_set_gpe_1d_surface(ctx, priv_ctx, gpe_context,
4508                                      HEVC_ENC_SURFACE_COL_MB_MV, bti_idx++,
4509                                      0, 0, 0, NULL, collocated_mv_temporal_bo);
4510     else
4511         bti_idx++;
4512
4513     bti_idx++;
4514
4515     gen9_hevc_set_gpe_1d_surface(ctx, priv_ctx, gpe_context,
4516                                  HEVC_ENC_SURFACE_BRC_INPUT, bti_idx++,
4517                                  0, 0, 0, NULL, NULL);
4518
4519     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
4520                                  HEVC_ENC_SURFACE_LCU_QP, bti_idx++,
4521                                  0, 1, I965_SURFACEFORMAT_R8_UNORM,
4522                                  NULL, NULL);
4523
4524     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
4525                                  HEVC_ENC_SURFACE_BRC_DATA, bti_idx++,
4526                                  0, 1, I965_SURFACEFORMAT_R8_UNORM,
4527                                  NULL, NULL);
4528
4529     gen9_hevc_set_gpe_adv_surface(ctx, priv_ctx, gpe_context,
4530                                   HEVC_ENC_SURFACE_RAW_VME, bti_idx++,
4531                                   NULL);
4532
4533     for (i = 0; i < 8; i++) {
4534         if (i <= slice_param->num_ref_idx_l0_active_minus1)
4535             obj_surface = SURFACE(slice_param->ref_pic_list0[i].picture_id);
4536         else
4537             obj_surface = NULL;
4538
4539         if (obj_surface) {
4540             surface_priv = (struct gen9_hevc_surface_priv *)obj_surface->private_data;
4541
4542             gen9_hevc_set_gpe_adv_surface(ctx, priv_ctx, gpe_context,
4543                                           HEVC_ENC_SURFACE_REF_FRAME_VME, bti_idx++,
4544                                           surface_priv->surface_reff);
4545         } else
4546             bti_idx++;
4547
4548         if (i <= slice_param->num_ref_idx_l1_active_minus1)
4549             obj_surface = SURFACE(slice_param->ref_pic_list1[i].picture_id);
4550         else
4551             obj_surface = NULL;
4552
4553         if (obj_surface) {
4554             surface_priv = (struct gen9_hevc_surface_priv *)obj_surface->private_data;
4555
4556             gen9_hevc_set_gpe_adv_surface(ctx, priv_ctx, gpe_context,
4557                                           HEVC_ENC_SURFACE_REF_FRAME_VME, bti_idx++,
4558                                           surface_priv->surface_reff);
4559         } else
4560             bti_idx++;
4561     }
4562
4563     if (priv_state->picture_coding_type == HEVC_SLICE_B) {
4564         gen9_hevc_set_gpe_adv_surface(ctx, priv_ctx, gpe_context,
4565                                       HEVC_ENC_SURFACE_RAW_VME, bti_idx++,
4566                                       NULL);
4567
4568         for (i = 0; i < 4; i++) {
4569             if (i <= slice_param->num_ref_idx_l1_active_minus1)
4570                 obj_surface = SURFACE(slice_param->ref_pic_list1[i].picture_id);
4571             else
4572                 obj_surface = NULL;
4573
4574             if (obj_surface) {
4575                 surface_priv = (struct gen9_hevc_surface_priv *)obj_surface->private_data;
4576
4577                 gen9_hevc_set_gpe_adv_surface(ctx, priv_ctx, gpe_context,
4578                                               HEVC_ENC_SURFACE_REF_FRAME_VME, bti_idx++,
4579                                               surface_priv->surface_reff);
4580             } else
4581                 bti_idx++;
4582
4583             bti_idx++;
4584         }
4585     }
4586
4587     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
4588                                  HEVC_ENC_SURFACE_CONCURRENT_THREAD, bti_idx++,
4589                                  0, 1, I965_SURFACEFORMAT_R8_UNORM,
4590                                  NULL, NULL);
4591
4592     gen9_hevc_set_gpe_1d_surface(ctx, priv_ctx, gpe_context,
4593                                  HEVC_ENC_SURFACE_MB_MV_INDEX, bti_idx++,
4594                                  0, 0, 0, NULL, NULL);
4595
4596     gen9_hevc_set_gpe_1d_surface(ctx, priv_ctx, gpe_context,
4597                                  HEVC_ENC_SURFACE_MVP_INDEX, bti_idx++,
4598                                  0, 0, 0, NULL, NULL);
4599
4600     gen9_hevc_set_gpe_1d_surface(ctx, priv_ctx, gpe_context,
4601                                  HEVC_ENC_SURFACE_KERNEL_DEBUG, bti_idx++,
4602                                  0, 0, 0, NULL, NULL);
4603 }
4604
4605 static void
4606 gen9_hevc_8x8_b_mbenc(VADriverContextP ctx,
4607                       struct encode_state *encode_state,
4608                       struct intel_encoder_context *encoder_context)
4609 {
4610     struct encoder_vme_mfc_context *vme_context = NULL;
4611     struct gen9_hevc_encoder_context *priv_ctx = NULL;
4612     struct gen9_hevc_encoder_state *priv_state = NULL;
4613     struct gen9_hevc_walking_pattern_parameter param;
4614     struct i965_gpe_context *gpe_context = NULL;
4615     int media_state = HEVC_ENC_MEDIA_STATE_HEVC_B_MBENC;
4616     int gpe_idx = 0;
4617
4618     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
4619     priv_ctx = (struct gen9_hevc_encoder_context *)vme_context->private_enc_ctx;
4620     priv_state = (struct gen9_hevc_encoder_state *)vme_context->private_enc_state;
4621
4622     if (priv_state->picture_coding_type == HEVC_SLICE_P)
4623         gpe_idx = priv_state->rolling_intra_refresh ? HEVC_MBENC_P_WIDI_IDX : HEVC_MBENC_PENC_IDX;
4624     else
4625         gpe_idx = priv_state->rolling_intra_refresh ? HEVC_MBENC_BENC_IDX : HEVC_MBENC_MBENC_WIDI_IDX;
4626
4627     gpe_context = &priv_ctx->mbenc_context.gpe_contexts[gpe_idx];
4628
4629     gen8_gpe_context_init(ctx, gpe_context);
4630     gen9_gpe_reset_binding_table(ctx, gpe_context);
4631
4632     memset((void *)&param, 0, sizeof(param));
4633     gen9_hevc_set_control_region(ctx, encode_state, encoder_context, gpe_context, &param);
4634
4635     gen9_hevc_8x8_b_mbenc_set_curbe(ctx, encode_state, encoder_context, gpe_context, &param);
4636     gen9_hevc_8x8_b_mbenc_set_surfaces(ctx, encode_state, encoder_context, gpe_context);
4637     gen8_gpe_setup_interface_data(ctx, gpe_context);
4638
4639     param.gpe_param.color_count_minus1 = param.num_region - 1;
4640     gen9_hevc_run_object_walker(ctx, encoder_context, gpe_context, &param.gpe_param,
4641                                 media_state);
4642 }
4643
4644 static void
4645 gen9_hevc_8x8_pu_fmode_set_curbe(VADriverContextP ctx,
4646                                  struct encode_state *encode_state,
4647                                  struct intel_encoder_context *encoder_context,
4648                                  struct i965_gpe_context *gpe_context)
4649 {
4650     struct i965_driver_data *i965 = i965_driver_data(ctx);
4651     struct encoder_vme_mfc_context *vme_context = NULL;
4652     struct gen9_hevc_encoder_context *priv_ctx = NULL;
4653     struct generic_enc_codec_state *generic_state = NULL;
4654     struct gen9_hevc_encoder_state *priv_state = NULL;
4655     gen9_hevc_mbenc_8x8_pu_fmode_curbe_data *cmd = NULL;
4656     VAEncSequenceParameterBufferHEVC *seq_param = NULL;
4657     VAEncPictureParameterBufferHEVC *pic_param = NULL;
4658     VAEncSliceParameterBufferHEVC *slice_param = NULL;
4659     unsigned int slice_qp = 0;
4660     int bti_idx = 0;
4661
4662     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
4663     priv_ctx = (struct gen9_hevc_encoder_context *)vme_context->private_enc_ctx;
4664     generic_state = (struct generic_enc_codec_state *)vme_context->generic_enc_state;
4665     priv_state = (struct gen9_hevc_encoder_state *)vme_context->private_enc_state;
4666     seq_param = (VAEncSequenceParameterBufferHEVC *)encode_state->seq_param_ext->buffer;
4667     pic_param = (VAEncPictureParameterBufferHEVC *)encode_state->pic_param_ext->buffer;
4668     slice_param = (VAEncSliceParameterBufferHEVC *)encode_state->slice_params_ext[0]->buffer;
4669
4670     slice_qp = pic_param->pic_init_qp + slice_param->slice_qp_delta;
4671
4672     cmd = i965_gpe_context_map_curbe(gpe_context);
4673     if (!cmd)
4674         return;
4675
4676     memset((void *)cmd, 0, sizeof(*cmd));
4677
4678     cmd->dw0.frame_width = ALIGN(priv_state->picture_width, 16);
4679     cmd->dw0.frame_height = ALIGN(priv_state->picture_height, 16);
4680     cmd->dw1.slice_type = priv_state->picture_coding_type;
4681     cmd->dw1.pu_type = 2;
4682     cmd->dw1.pak_reording_flag = priv_state->picture_coding_type == HEVC_SLICE_I ? 1 : 0;
4683
4684     if (seq_param->log2_min_luma_coding_block_size_minus3 + 3 +
4685         seq_param->log2_diff_max_min_luma_coding_block_size == 6)
4686         cmd->dw1.lcu_type = 0;
4687     else
4688         cmd->dw1.lcu_type = 1;
4689
4690     cmd->dw1.screen_content_flag = !!pic_param->pic_fields.bits.screen_content_flag;
4691     cmd->dw1.enable_intra_early_exit = priv_state->tu_mode == HEVC_TU_RT_SPEED ?
4692                                        (priv_state->picture_coding_type == HEVC_SLICE_I ? 0 : 1) : 0;
4693     cmd->dw1.enable_debug_dump = 0;
4694     cmd->dw1.brc_enable = generic_state->brc_enabled;
4695     cmd->dw1.lcu_brc_enable = priv_state->lcu_brc_enabled;
4696     cmd->dw1.roi_enable = (priv_state->num_roi > 0);
4697     cmd->dw1.fast_surveillance_flag = priv_state->picture_coding_type == HEVC_SLICE_I ?
4698                                       0 : priv_state->video_surveillance_flag;
4699     cmd->dw1.enable_rolling_intra = priv_state->rolling_intra_refresh;
4700     cmd->dw1.widi_intra_refresh_en = priv_state->rolling_intra_refresh;
4701     cmd->dw1.half_update_mixed_lcu = 0;
4702     cmd->dw2.luma_lambda = priv_state->fixed_point_lambda_for_luma;
4703
4704     if (priv_state->picture_coding_type != HEVC_SLICE_I) {
4705         double lambda_md;
4706         float had_bias = 2.0f;
4707
4708         lambda_md = priv_ctx->lambda_md_table[cmd->dw1.slice_type][slice_qp];
4709         lambda_md = lambda_md * had_bias;
4710         cmd->dw3.lambda_for_dist_calculation = (unsigned int)(lambda_md * (1 << 10));
4711     }
4712
4713     cmd->dw4.mode_cost_for_8x8_pu_tu8 = 0;
4714     cmd->dw5.mode_cost_for_8x8_pu_tu4 = 0;
4715     cmd->dw6.satd_16x16_pu_threshold = MAX(200 * ((int)slice_qp - 12), 0);
4716     cmd->dw6.bias_factor_toward_8x8 = pic_param->pic_fields.bits.screen_content_flag ? 1024 : 1126 + 102;
4717     cmd->dw7.qp = slice_qp;
4718     cmd->dw7.qp_for_inter = 0;
4719     cmd->dw8.simplified_flag_for_inter = 0;
4720     cmd->dw8.kbl_control_flag = (IS_KBL(i965->intel.device_info) || IS_GLK(i965->intel.device_info));
4721     cmd->dw9.widi_intra_refresh_qp_delta = priv_state->widi_intra_refresh_qp_delta;
4722     cmd->dw9.widi_intra_refresh_mb_num = priv_state->widi_intra_insertion_location;
4723     cmd->dw9.widi_intra_refresh_unit_in_mb = priv_state->widi_intra_insertion_size;
4724
4725     cmd->dw16.bti_pak_object = bti_idx++;
4726     cmd->dw17.bti_vme_8x8_mode = bti_idx++;
4727     cmd->dw18.bti_intra_mode = bti_idx++;
4728     cmd->dw19.bti_pak_command = bti_idx++;
4729     cmd->dw20.bti_slice_map = bti_idx++;
4730     cmd->dw21.bti_intra_dist = bti_idx++;
4731     cmd->dw22.bti_brc_input = bti_idx++;
4732     cmd->dw23.bti_simplest_intra = bti_idx++;
4733     cmd->dw24.bti_lcu_qp_surface = bti_idx++;
4734     cmd->dw25.bti_brc_data = bti_idx++;
4735     cmd->dw26.bti_debug = bti_idx++;
4736
4737     i965_gpe_context_unmap_curbe(gpe_context);
4738 }
4739
4740 static void
4741 gen9_hevc_8x8_pu_fmode_set_surfaces(VADriverContextP ctx,
4742                                     struct encode_state *encode_state,
4743                                     struct intel_encoder_context *encoder_context,
4744                                     struct i965_gpe_context *gpe_context)
4745 {
4746     struct encoder_vme_mfc_context *vme_context = NULL;
4747     struct gen9_hevc_encoder_context *priv_ctx = NULL;
4748     struct gen9_hevc_encoder_state *priv_state = NULL;
4749     int bti_idx = 0;
4750     int size = 0;
4751
4752     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
4753     priv_ctx = (struct gen9_hevc_encoder_context *)vme_context->private_enc_ctx;
4754     priv_state = (struct gen9_hevc_encoder_state *)vme_context->private_enc_state;
4755
4756     size = priv_state->width_in_cu * priv_state->height_in_cu *
4757            priv_state->cu_record_size;
4758     gen9_hevc_set_gpe_1d_surface(ctx, priv_ctx, gpe_context,
4759                                  HEVC_ENC_SURFACE_CU_RECORD, bti_idx++,
4760                                  0, size, priv_state->mb_data_offset, NULL, NULL);
4761
4762     gen9_hevc_set_gpe_1d_surface(ctx, priv_ctx, gpe_context,
4763                                  HEVC_ENC_SURFACE_VME_8x8, bti_idx++,
4764                                  0, 0, 0, NULL, NULL);
4765
4766     gen9_hevc_set_gpe_1d_surface(ctx, priv_ctx, gpe_context,
4767                                  HEVC_ENC_SURFACE_INTRA_MODE, bti_idx++,
4768                                  0, 0, 0, NULL, NULL);
4769
4770     size = priv_state->mb_data_offset;
4771     gen9_hevc_set_gpe_1d_surface(ctx, priv_ctx, gpe_context,
4772                                  HEVC_ENC_SURFACE_HCP_PAK, bti_idx++,
4773                                  0, size, 0, NULL, NULL);
4774
4775     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
4776                                  HEVC_ENC_SURFACE_SLICE_MAP, bti_idx++,
4777                                  0, 1, I965_SURFACEFORMAT_R8_UNORM,
4778                                  NULL, NULL);
4779
4780     gen9_hevc_set_gpe_1d_surface(ctx, priv_ctx, gpe_context,
4781                                  HEVC_ENC_SURFACE_INTRA_DIST, bti_idx++,
4782                                  0, 0, 0, NULL, NULL);
4783
4784     gen9_hevc_set_gpe_1d_surface(ctx, priv_ctx, gpe_context,
4785                                  HEVC_ENC_SURFACE_BRC_INPUT, bti_idx++,
4786                                  0, 0, 0, NULL, NULL);
4787
4788     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
4789                                  HEVC_ENC_SURFACE_SIMPLIFIED_INTRA, bti_idx++,
4790                                  0, 1, I965_SURFACEFORMAT_R8_UNORM,
4791                                  NULL, NULL);
4792
4793     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
4794                                  HEVC_ENC_SURFACE_LCU_QP, bti_idx++,
4795                                  0, 1, I965_SURFACEFORMAT_R8_UNORM,
4796                                  NULL, NULL);
4797
4798     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
4799                                  HEVC_ENC_SURFACE_BRC_DATA, bti_idx++,
4800                                  0, 1, I965_SURFACEFORMAT_R8_UNORM,
4801                                  NULL, NULL);
4802
4803     gen9_hevc_set_gpe_1d_surface(ctx, priv_ctx, gpe_context,
4804                                  HEVC_ENC_SURFACE_KERNEL_DEBUG, bti_idx++,
4805                                  0, 0, 0, NULL, NULL);
4806 }
4807
4808 static void
4809 gen9_hevc_8x8_pu_fmode(VADriverContextP ctx,
4810                        struct encode_state *encode_state,
4811                        struct intel_encoder_context *encoder_context)
4812 {
4813     struct encoder_vme_mfc_context *vme_context = NULL;
4814     struct gen9_hevc_encoder_context *priv_ctx = NULL;
4815     struct gen9_hevc_encoder_state *priv_state = NULL;
4816     struct gpe_media_object_walker_parameter param;
4817     struct hevc_enc_kernel_walker_parameter hevc_walker_param;
4818     struct i965_gpe_context *gpe_context = NULL;
4819     int media_state = HEVC_ENC_MEDIA_STATE_8x8_PU_FMODE;
4820
4821     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
4822     priv_ctx = (struct gen9_hevc_encoder_context *)vme_context->private_enc_ctx;
4823     priv_state = (struct gen9_hevc_encoder_state *)vme_context->private_enc_state;
4824
4825     gpe_context = &priv_ctx->mbenc_context.gpe_contexts[HEVC_MBENC_8x8FMODE_IDX];
4826
4827     gen8_gpe_context_init(ctx, gpe_context);
4828     gen9_gpe_reset_binding_table(ctx, gpe_context);
4829     gen9_hevc_8x8_pu_fmode_set_curbe(ctx, encode_state, encoder_context, gpe_context);
4830     gen9_hevc_8x8_pu_fmode_set_surfaces(ctx, encode_state, encoder_context, gpe_context);
4831     gen8_gpe_setup_interface_data(ctx, gpe_context);
4832
4833     memset((void *)&hevc_walker_param, 0, sizeof(hevc_walker_param));
4834     hevc_walker_param.resolution_x = priv_state->width_in_lcu;
4835     hevc_walker_param.resolution_y = priv_state->height_in_lcu;
4836     hevc_walker_param.no_dependency = 1;
4837     gen9_hevc_init_object_walker(&hevc_walker_param, &param);
4838     gen9_hevc_run_object_walker(ctx, encoder_context, gpe_context, &param,
4839                                 media_state);
4840 }
4841
4842 static void
4843 gen9_hevc_8x8_pu_mode_set_curbe(VADriverContextP ctx,
4844                                 struct encode_state *encode_state,
4845                                 struct intel_encoder_context *encoder_context,
4846                                 struct i965_gpe_context *gpe_context)
4847 {
4848     struct encoder_vme_mfc_context *vme_context = NULL;
4849     struct generic_enc_codec_state *generic_state = NULL;
4850     struct gen9_hevc_encoder_state *priv_state = NULL;
4851     gen9_hevc_mbenc_8x8_pu_curbe_data *cmd = NULL;
4852     VAEncSequenceParameterBufferHEVC *seq_param = NULL;
4853     VAEncPictureParameterBufferHEVC *pic_param = NULL;
4854     VAEncSliceParameterBufferHEVC *slice_param = NULL;
4855     int bti_idx = 0;
4856
4857     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
4858     generic_state = (struct generic_enc_codec_state *)vme_context->generic_enc_state;
4859     priv_state = (struct gen9_hevc_encoder_state *)vme_context->private_enc_state;
4860     seq_param = (VAEncSequenceParameterBufferHEVC *)encode_state->seq_param_ext->buffer;
4861     pic_param = (VAEncPictureParameterBufferHEVC *)encode_state->pic_param_ext->buffer;
4862     slice_param = (VAEncSliceParameterBufferHEVC *)encode_state->slice_params_ext[0]->buffer;
4863
4864     cmd = i965_gpe_context_map_curbe(gpe_context);
4865     if (!cmd)
4866         return;
4867
4868     memset((void *)cmd, 0, sizeof(*cmd));
4869
4870     cmd->dw0.frame_width = ALIGN(priv_state->picture_width, 16);
4871     cmd->dw0.frame_height = ALIGN(priv_state->picture_height, 16);
4872     cmd->dw1.slice_type = priv_state->picture_coding_type;
4873     cmd->dw1.pu_type = 2;
4874     cmd->dw1.dc_filter_flag = 1;
4875     cmd->dw1.angle_refine_flag = 1;
4876     if (seq_param->log2_min_luma_coding_block_size_minus3 + 3 +
4877         seq_param->log2_diff_max_min_luma_coding_block_size == 6)
4878         cmd->dw1.lcu_type = 0;
4879     else
4880         cmd->dw1.lcu_type = 1;
4881     cmd->dw1.screen_content_flag = !!pic_param->pic_fields.bits.screen_content_flag;
4882     cmd->dw1.enable_intra_early_exit = priv_state->tu_mode == HEVC_TU_RT_SPEED ?
4883                                        (priv_state->picture_coding_type == HEVC_SLICE_I ? 0 : 1) : 0;
4884     cmd->dw1.enable_debug_dump = 0;
4885     cmd->dw1.brc_enable = generic_state->brc_enabled;
4886     cmd->dw1.lcu_brc_enable = priv_state->lcu_brc_enabled;
4887     cmd->dw1.roi_enable = (priv_state->num_roi > 0);
4888     cmd->dw1.fast_surveillance_flag = priv_state->picture_coding_type == HEVC_SLICE_I ?
4889                                       0 : priv_state->video_surveillance_flag;
4890     if (priv_state->rolling_intra_refresh) {
4891         cmd->dw1.enable_rolling_intra = 1;
4892         cmd->dw1.widi_intra_refresh_en = 1;
4893         cmd->dw1.half_update_mixed_lcu = 0;
4894
4895         cmd->dw5.widi_intra_refresh_mb_num = priv_state->widi_intra_insertion_location;
4896         cmd->dw5.widi_intra_refresh_mb_num = priv_state->widi_intra_insertion_location;
4897         cmd->dw5.widi_intra_refresh_unit_in_mb = priv_state->widi_intra_insertion_size;
4898
4899         cmd->dw1.qp_value = pic_param->pic_init_qp + slice_param->slice_qp_delta;
4900     }
4901
4902     cmd->dw2.luma_lambda = priv_state->fixed_point_lambda_for_luma;
4903     cmd->dw3.chroma_lambda = priv_state->fixed_point_lambda_for_chroma;
4904     cmd->dw4.simplified_flag_for_inter = 0;
4905     cmd->dw4.harr_trans_form_flag = priv_state->picture_coding_type == HEVC_SLICE_I ? 0 : 1;
4906
4907     cmd->dw8.bti_src_y = bti_idx++;
4908     bti_idx++;
4909     cmd->dw9.bti_slice_map = bti_idx++;
4910     cmd->dw10.bti_vme_8x8_mode = bti_idx++;
4911     cmd->dw11.bti_intra_mode = bti_idx++;
4912     cmd->dw12.bti_brc_input = bti_idx++;
4913     cmd->dw13.bti_simplest_intra = bti_idx++;
4914     cmd->dw14.bti_lcu_qp_surface = bti_idx++;
4915     cmd->dw15.bti_brc_data = bti_idx++;
4916     cmd->dw16.bti_debug = bti_idx++;
4917
4918     i965_gpe_context_unmap_curbe(gpe_context);
4919 }
4920
4921 static void
4922 gen9_hevc_8x8_pu_mode_set_surfaces(VADriverContextP ctx,
4923                                    struct encode_state *encode_state,
4924                                    struct intel_encoder_context *encoder_context,
4925                                    struct i965_gpe_context *gpe_context)
4926 {
4927     struct encoder_vme_mfc_context *vme_context = NULL;
4928     struct gen9_hevc_encoder_context *priv_ctx = NULL;
4929     int bti_idx = 0;
4930
4931     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
4932     priv_ctx = (struct gen9_hevc_encoder_context *)vme_context->private_enc_ctx;
4933
4934     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
4935                                  HEVC_ENC_SURFACE_RAW_Y_UV, bti_idx++,
4936                                  1, 1, I965_SURFACEFORMAT_R8_UNORM,
4937                                  NULL, NULL);
4938     bti_idx++;
4939
4940     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
4941                                  HEVC_ENC_SURFACE_SLICE_MAP, bti_idx++,
4942                                  0, 1, I965_SURFACEFORMAT_R8_UNORM,
4943                                  NULL, NULL);
4944
4945     gen9_hevc_set_gpe_1d_surface(ctx, priv_ctx, gpe_context,
4946                                  HEVC_ENC_SURFACE_VME_8x8, bti_idx++,
4947                                  0, 0, 0, NULL, NULL);
4948
4949     gen9_hevc_set_gpe_1d_surface(ctx, priv_ctx, gpe_context,
4950                                  HEVC_ENC_SURFACE_INTRA_MODE, bti_idx++,
4951                                  0, 0, 0, NULL, NULL);
4952
4953     gen9_hevc_set_gpe_1d_surface(ctx, priv_ctx, gpe_context,
4954                                  HEVC_ENC_SURFACE_BRC_INPUT, bti_idx++,
4955                                  0, 0, 0, NULL, NULL);
4956
4957     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
4958                                  HEVC_ENC_SURFACE_SIMPLIFIED_INTRA, bti_idx++,
4959                                  0, 1, I965_SURFACEFORMAT_R8_UNORM,
4960                                  NULL, NULL);
4961
4962     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
4963                                  HEVC_ENC_SURFACE_LCU_QP, bti_idx++,
4964                                  0, 1, I965_SURFACEFORMAT_R8_UNORM,
4965                                  NULL, NULL);
4966
4967     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
4968                                  HEVC_ENC_SURFACE_BRC_DATA, bti_idx++,
4969                                  0, 1, I965_SURFACEFORMAT_R8_UNORM,
4970                                  NULL, NULL);
4971
4972     gen9_hevc_set_gpe_1d_surface(ctx, priv_ctx, gpe_context,
4973                                  HEVC_ENC_SURFACE_KERNEL_DEBUG, bti_idx++,
4974                                  0, 0, 0, NULL, NULL);
4975 }
4976
4977 static void
4978 gen9_hevc_8x8_pu_mode(VADriverContextP ctx,
4979                       struct encode_state *encode_state,
4980                       struct intel_encoder_context *encoder_context)
4981 {
4982     struct encoder_vme_mfc_context *vme_context = NULL;
4983     struct gen9_hevc_encoder_context *priv_ctx = NULL;
4984     struct gen9_hevc_encoder_state *priv_state = NULL;
4985     struct gpe_media_object_walker_parameter param;
4986     struct hevc_enc_kernel_walker_parameter hevc_walker_param;
4987     struct i965_gpe_context *gpe_context = NULL;
4988     int media_state = HEVC_ENC_MEDIA_STATE_8x8_PU;
4989
4990     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
4991     priv_ctx = (struct gen9_hevc_encoder_context *)vme_context->private_enc_ctx;
4992     priv_state = (struct gen9_hevc_encoder_state *)vme_context->private_enc_state;
4993
4994     gpe_context = &priv_ctx->mbenc_context.gpe_contexts[HEVC_MBENC_8x8PU_IDX];
4995
4996     gen8_gpe_context_init(ctx, gpe_context);
4997     gen9_gpe_reset_binding_table(ctx, gpe_context);
4998     gen9_hevc_8x8_pu_mode_set_curbe(ctx, encode_state, encoder_context, gpe_context);
4999     gen9_hevc_8x8_pu_mode_set_surfaces(ctx, encode_state, encoder_context, gpe_context);
5000     gen8_gpe_setup_interface_data(ctx, gpe_context);
5001
5002     memset((void *)&hevc_walker_param, 0, sizeof(hevc_walker_param));
5003     hevc_walker_param.resolution_x = ALIGN(priv_state->picture_width, 16) >> 3;
5004     hevc_walker_param.resolution_y = ALIGN(priv_state->picture_height, 16) >> 3;
5005     hevc_walker_param.no_dependency = 1;
5006     gen9_hevc_init_object_walker(&hevc_walker_param, &param);
5007     gen9_hevc_run_object_walker(ctx, encoder_context, gpe_context, &param,
5008                                 media_state);
5009 }
5010
5011 static void
5012 gen9_hevc_16x16_pu_mode_set_curbe(VADriverContextP ctx,
5013                                   struct encode_state *encode_state,
5014                                   struct intel_encoder_context *encoder_context,
5015                                   struct i965_gpe_context *gpe_context)
5016 {
5017     struct encoder_vme_mfc_context *vme_context = NULL;
5018     struct gen9_hevc_encoder_context *priv_ctx = NULL;
5019     struct generic_enc_codec_state *generic_state = NULL;
5020     struct gen9_hevc_encoder_state *priv_state = NULL;
5021     gen9_hevc_enc_16x16_pu_curbe_data *cmd = NULL;
5022     VAEncSequenceParameterBufferHEVC *seq_param = NULL;
5023     VAEncPictureParameterBufferHEVC *pic_param = NULL;
5024     VAEncSliceParameterBufferHEVC *slice_param = NULL;
5025     double squred_lambda, qp_lambda, lambda_scaling_factor;
5026     unsigned int slice_qp = 0, slice_type = HEVC_SLICE_I;
5027     unsigned int new_point_lambda_for_luma;
5028     unsigned char mode_cost[12];
5029     int bti_idx = 0;
5030
5031     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
5032     priv_ctx = (struct gen9_hevc_encoder_context *)vme_context->private_enc_ctx;
5033     generic_state = (struct generic_enc_codec_state *)vme_context->generic_enc_state;
5034     priv_state = (struct gen9_hevc_encoder_state *)vme_context->private_enc_state;
5035     seq_param = (VAEncSequenceParameterBufferHEVC *)encode_state->seq_param_ext->buffer;
5036     pic_param = (VAEncPictureParameterBufferHEVC *)encode_state->pic_param_ext->buffer;
5037     slice_param = (VAEncSliceParameterBufferHEVC *)encode_state->slice_params_ext[0]->buffer;
5038
5039     slice_qp = pic_param->pic_init_qp + slice_param->slice_qp_delta;
5040     slice_type = priv_state->picture_coding_type;
5041
5042     lambda_scaling_factor = 0.46 + slice_qp - 22;
5043     if (lambda_scaling_factor < 0)
5044         lambda_scaling_factor = 0.46;
5045     else if (lambda_scaling_factor > 15)
5046         lambda_scaling_factor = 15;
5047
5048     squred_lambda = lambda_scaling_factor * pow(2.0, ((double)slice_qp - 12.0) / 6);
5049     priv_state->fixed_point_lambda_for_luma = (unsigned int)(squred_lambda * (1 << 10));
5050
5051     lambda_scaling_factor = 1.0;
5052     qp_lambda = priv_ctx->lambda_md_table[slice_type][slice_qp];
5053     squred_lambda = qp_lambda * qp_lambda;
5054     priv_state->fixed_point_lambda_for_chroma = (unsigned int)(lambda_scaling_factor * squred_lambda * (1 << 10));
5055
5056     qp_lambda = sqrt(0.57 * pow(2.0, ((double)slice_qp - 12.0) / 3));
5057     squred_lambda = qp_lambda * qp_lambda;
5058     new_point_lambda_for_luma = (unsigned int)(squred_lambda * (1 << 10));
5059
5060     gen9_hevc_mbenc_set_costs(priv_ctx, slice_type, HEVC_ENC_INTRA_TRANS_HAAR, slice_qp,
5061                               mode_cost, NULL, NULL, NULL);
5062
5063     cmd = i965_gpe_context_map_curbe(gpe_context);
5064     if (!cmd)
5065         return;
5066
5067     memset((void *)cmd, 0, sizeof(*cmd));
5068
5069     cmd->dw0.frame_width = ALIGN(priv_state->picture_width, 16);
5070     cmd->dw0.frame_height = ALIGN(priv_state->picture_height, 16);
5071     cmd->dw1.log2_min_cu_size = seq_param->log2_min_luma_coding_block_size_minus3 + 3;
5072     cmd->dw1.log2_max_cu_size = cmd->dw1.log2_min_cu_size + seq_param->log2_diff_max_min_luma_coding_block_size;
5073     cmd->dw1.log2_min_tu_size = seq_param->log2_min_transform_block_size_minus2 + 2;
5074     cmd->dw1.slice_qp = slice_qp;
5075     cmd->dw2.fixed_point_lambda_pred_mode = priv_state->fixed_point_lambda_for_chroma;
5076     cmd->dw3.lambda_scaling_factor = 1;
5077     cmd->dw3.slice_type = slice_type;
5078     cmd->dw3.enable_intra_early_exit = priv_state->tu_mode == HEVC_TU_RT_SPEED ?
5079                                        (priv_state->picture_coding_type == HEVC_SLICE_I ? 0 : 1) : 0;
5080     cmd->dw3.brc_enable = !!generic_state->brc_enabled;
5081     cmd->dw3.lcu_brc_enable = !!priv_state->lcu_brc_enabled;
5082     cmd->dw3.roi_enable = (priv_state->num_roi > 0);
5083     cmd->dw3.fast_surveillance_flag = priv_state->picture_coding_type == HEVC_SLICE_I ?
5084                                       0 : priv_state->video_surveillance_flag;
5085     cmd->dw3.enable_rolling_intra = priv_state->rolling_intra_refresh;
5086     cmd->dw3.widi_intra_refresh_en = priv_state->rolling_intra_refresh;
5087     cmd->dw3.half_update_mixed_lcu = 0;
5088     cmd->dw4.penalty_for_intra_8x8_non_dc_pred_mode = 0;
5089     cmd->dw4.intra_compute_type = 1;
5090     cmd->dw4.avc_intra_8x8_mask = 0;
5091     cmd->dw4.intra_sad_adjust = 2;
5092     cmd->dw5.fixed_point_lambda_cu_mode_for_cost_calculation = new_point_lambda_for_luma;
5093     cmd->dw6.screen_content_flag = !!pic_param->pic_fields.bits.screen_content_flag;
5094     cmd->dw7.mode_cost_intra_non_pred = mode_cost[0];
5095     cmd->dw7.mode_cost_intra_16x16 = mode_cost[1];
5096     cmd->dw7.mode_cost_intra_8x8 = mode_cost[2];
5097     cmd->dw7.mode_cost_intra_4x4 = mode_cost[3];
5098     cmd->dw8.fixed_point_lambda_cu_mode_for_luma = priv_state->fixed_point_lambda_for_luma;
5099
5100     if (priv_state->rolling_intra_refresh) {
5101         cmd->dw9.widi_intra_refresh_mb_num = priv_state->widi_intra_insertion_location;
5102         cmd->dw9.widi_intra_refresh_mb_num = priv_state->widi_intra_insertion_location;
5103         cmd->dw9.widi_intra_refresh_unit_in_mb = priv_state->widi_intra_insertion_size;
5104     }
5105
5106     cmd->dw10.simplified_flag_for_inter = 0;
5107     cmd->dw10.haar_transform_mode = priv_state->picture_coding_type == HEVC_SLICE_I ? 0 : 1;
5108
5109     cmd->dw16.bti_src_y = bti_idx++;
5110     bti_idx++;
5111     cmd->dw17.bti_sad_16x16_pu = bti_idx++;
5112     cmd->dw18.bti_pak_object = bti_idx++;
5113     cmd->dw19.bti_sad_32x32_pu_mode = bti_idx++;
5114     cmd->dw20.bti_vme_mode_8x8 = bti_idx++;
5115     cmd->dw21.bti_slice_map = bti_idx++;
5116     cmd->dw22.bti_vme_src = bti_idx++;
5117     cmd->dw23.bti_brc_input = bti_idx++;
5118     cmd->dw24.bti_simplest_intra = bti_idx++;
5119     cmd->dw25.bti_lcu_qp_surface = bti_idx++;
5120     cmd->dw26.bti_brc_data = bti_idx++;
5121     cmd->dw27.bti_debug = bti_idx++;
5122
5123     i965_gpe_context_unmap_curbe(gpe_context);
5124 }
5125
5126 static void
5127 gen9_hevc_16x16_pu_mode_set_surfaces(VADriverContextP ctx,
5128                                      struct encode_state *encode_state,
5129                                      struct intel_encoder_context *encoder_context,
5130                                      struct i965_gpe_context *gpe_context)
5131 {
5132     struct encoder_vme_mfc_context *vme_context = NULL;
5133     struct gen9_hevc_encoder_context *priv_ctx = NULL;
5134     struct gen9_hevc_encoder_state *priv_state = NULL;
5135     int bti_idx = 0;
5136     int size = 0;
5137
5138     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
5139     priv_ctx = (struct gen9_hevc_encoder_context *)vme_context->private_enc_ctx;
5140     priv_state = (struct gen9_hevc_encoder_state *)vme_context->private_enc_state;
5141
5142     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
5143                                  HEVC_ENC_SURFACE_RAW_Y_UV, bti_idx++,
5144                                  1, 1, I965_SURFACEFORMAT_R8_UNORM,
5145                                  NULL, NULL);
5146     bti_idx++;
5147
5148     gen9_hevc_set_gpe_1d_surface(ctx, priv_ctx, gpe_context,
5149                                  HEVC_ENC_SURFACE_16x16PU_SAD, bti_idx++,
5150                                  0, 0, 0, NULL, NULL);
5151
5152     size = priv_state->width_in_cu * priv_state->height_in_cu *
5153            priv_state->cu_record_size;
5154     gen9_hevc_set_gpe_1d_surface(ctx, priv_ctx, gpe_context,
5155                                  HEVC_ENC_SURFACE_CU_RECORD, bti_idx++,
5156                                  0, size, priv_state->mb_data_offset, NULL, NULL);
5157
5158     gen9_hevc_set_gpe_1d_surface(ctx, priv_ctx, gpe_context,
5159                                  HEVC_ENC_SURFACE_32x32_PU_OUTPUT, bti_idx++,
5160                                  0, 0, 0, NULL, NULL);
5161
5162     gen9_hevc_set_gpe_1d_surface(ctx, priv_ctx, gpe_context,
5163                                  HEVC_ENC_SURFACE_VME_8x8, bti_idx++,
5164                                  0, 0, 0, NULL, NULL);
5165
5166     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
5167                                  HEVC_ENC_SURFACE_SLICE_MAP, bti_idx++,
5168                                  0, 1, I965_SURFACEFORMAT_R8_UNORM,
5169                                  NULL, NULL);
5170
5171     gen9_hevc_set_gpe_adv_surface(ctx, priv_ctx, gpe_context,
5172                                   HEVC_ENC_SURFACE_RAW_VME, bti_idx++,
5173                                   NULL);
5174
5175     gen9_hevc_set_gpe_1d_surface(ctx, priv_ctx, gpe_context,
5176                                  HEVC_ENC_SURFACE_BRC_INPUT, bti_idx++,
5177                                  0, 0, 0, NULL, NULL);
5178
5179     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
5180                                  HEVC_ENC_SURFACE_SIMPLIFIED_INTRA, bti_idx++,
5181                                  0, 1, I965_SURFACEFORMAT_R8_UNORM,
5182                                  NULL, NULL);
5183
5184     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
5185                                  HEVC_ENC_SURFACE_LCU_QP, bti_idx++,
5186                                  0, 1, I965_SURFACEFORMAT_R8_UNORM,
5187                                  NULL, NULL);
5188
5189     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
5190                                  HEVC_ENC_SURFACE_BRC_DATA, bti_idx++,
5191                                  0, 1, I965_SURFACEFORMAT_R8_UNORM,
5192                                  NULL, NULL);
5193
5194     gen9_hevc_set_gpe_1d_surface(ctx, priv_ctx, gpe_context,
5195                                  HEVC_ENC_SURFACE_KERNEL_DEBUG, bti_idx++,
5196                                  0, 0, 0, NULL, NULL);
5197 }
5198
5199 static void
5200 gen9_hevc_16x16_pu_mode(VADriverContextP ctx,
5201                         struct encode_state *encode_state,
5202                         struct intel_encoder_context *encoder_context)
5203 {
5204     struct encoder_vme_mfc_context *vme_context = NULL;
5205     struct gen9_hevc_encoder_context *priv_ctx = NULL;
5206     struct gen9_hevc_encoder_state *priv_state = NULL;
5207     struct gpe_media_object_walker_parameter param;
5208     struct hevc_enc_kernel_walker_parameter hevc_walker_param;
5209     struct i965_gpe_context *gpe_context = NULL;
5210     int media_state = HEVC_ENC_MEDIA_STATE_16x16_PU_MODE_DECISION;
5211
5212     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
5213     priv_ctx = (struct gen9_hevc_encoder_context *)vme_context->private_enc_ctx;
5214     priv_state = (struct gen9_hevc_encoder_state *)vme_context->private_enc_state;
5215
5216     gpe_context = &priv_ctx->mbenc_context.gpe_contexts[HEVC_MBENC_16x16MD_IDX];
5217
5218     gen8_gpe_context_init(ctx, gpe_context);
5219     gen9_gpe_reset_binding_table(ctx, gpe_context);
5220     gen9_hevc_16x16_pu_mode_set_curbe(ctx, encode_state, encoder_context, gpe_context);
5221     gen9_hevc_16x16_pu_mode_set_surfaces(ctx, encode_state, encoder_context, gpe_context);
5222     gen8_gpe_setup_interface_data(ctx, gpe_context);
5223
5224     memset((void *)&hevc_walker_param, 0, sizeof(hevc_walker_param));
5225     hevc_walker_param.resolution_x = ALIGN(priv_state->picture_width, 32) >> 5;
5226     hevc_walker_param.resolution_y = ALIGN(priv_state->picture_height, 32) >> 5;
5227     hevc_walker_param.no_dependency = 1;
5228     gen9_hevc_init_object_walker(&hevc_walker_param, &param);
5229     gen9_hevc_run_object_walker(ctx, encoder_context, gpe_context, &param,
5230                                 media_state);
5231 }
5232
5233 static void
5234 gen9_hevc_16x16_sad_pu_comp_set_curbe(VADriverContextP ctx,
5235                                       struct encode_state *encode_state,
5236                                       struct intel_encoder_context *encoder_context,
5237                                       struct i965_gpe_context *gpe_context)
5238 {
5239     struct encoder_vme_mfc_context *vme_context = NULL;
5240     struct gen9_hevc_encoder_state *priv_state = NULL;
5241     gen9_hevc_mbenc_16x16_sad_curbe_data *cmd = NULL;
5242     VAEncSequenceParameterBufferHEVC *seq_param = NULL;
5243     int bti_idx = 0;
5244
5245     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
5246     priv_state = (struct gen9_hevc_encoder_state *)vme_context->private_enc_state;
5247     seq_param = (VAEncSequenceParameterBufferHEVC *)encode_state->seq_param_ext->buffer;
5248
5249     cmd = i965_gpe_context_map_curbe(gpe_context);
5250     if (!cmd)
5251         return;
5252
5253     memset((void *)cmd, 0, sizeof(*cmd));
5254
5255     cmd->dw0.frame_width = ALIGN(priv_state->picture_width, 16);
5256     cmd->dw0.frame_height = ALIGN(priv_state->picture_height, 16);
5257     cmd->dw1.log2_min_cu_size = seq_param->log2_min_luma_coding_block_size_minus3 + 3;
5258     cmd->dw1.log2_max_cu_size = cmd->dw1.log2_min_cu_size + seq_param->log2_diff_max_min_luma_coding_block_size;
5259     cmd->dw1.log2_min_tu_size = seq_param->log2_min_transform_block_size_minus2 + 2;
5260     cmd->dw1.enable_intra_early_exit = priv_state->tu_mode == HEVC_TU_RT_SPEED ?
5261                                        (priv_state->picture_coding_type == HEVC_SLICE_I ? 0 : 1) : 0;
5262     cmd->dw2.sim_flag_for_inter = 0;
5263     cmd->dw2.slice_type = priv_state->picture_coding_type;
5264     cmd->dw2.fast_surveillance_flag = priv_state->picture_coding_type == HEVC_SLICE_I ?
5265                                       0 : priv_state->video_surveillance_flag;
5266
5267     cmd->dw8.bti_src_y = bti_idx++;
5268     bti_idx++;
5269     cmd->dw9.bti_sad_16x16_pu_output = bti_idx++;
5270     cmd->dw10.bti_32x32_pu_mode_decision = bti_idx++;
5271     cmd->dw11.bti_slice_map = bti_idx++;
5272     cmd->dw12.bti_simplest_intra = bti_idx++;
5273     cmd->dw13.bti_debug = bti_idx++;
5274
5275     i965_gpe_context_unmap_curbe(gpe_context);
5276 }
5277
5278 static void
5279 gen9_hevc_16x16_sad_pu_comp_set_surfaces(VADriverContextP ctx,
5280                                          struct encode_state *encode_state,
5281                                          struct intel_encoder_context *encoder_context,
5282                                          struct i965_gpe_context *gpe_context)
5283 {
5284     struct encoder_vme_mfc_context *vme_context = NULL;
5285     struct gen9_hevc_encoder_context *priv_ctx = NULL;
5286     int bti_idx = 0;
5287
5288     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
5289     priv_ctx = (struct gen9_hevc_encoder_context *)vme_context->private_enc_ctx;
5290
5291     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
5292                                  HEVC_ENC_SURFACE_RAW_Y_UV, bti_idx++,
5293                                  1, 1, I965_SURFACEFORMAT_R8_UNORM,
5294                                  NULL, NULL);
5295     bti_idx++;
5296
5297     gen9_hevc_set_gpe_1d_surface(ctx, priv_ctx, gpe_context,
5298                                  HEVC_ENC_SURFACE_16x16PU_SAD, bti_idx++,
5299                                  0, 0, 0, NULL, NULL);
5300
5301     gen9_hevc_set_gpe_1d_surface(ctx, priv_ctx, gpe_context,
5302                                  HEVC_ENC_SURFACE_32x32_PU_OUTPUT, bti_idx++,
5303                                  0, 0, 0, NULL, NULL);
5304
5305     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
5306                                  HEVC_ENC_SURFACE_SLICE_MAP, bti_idx++,
5307                                  0, 1, I965_SURFACEFORMAT_R8_UNORM,
5308                                  NULL, NULL);
5309
5310     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
5311                                  HEVC_ENC_SURFACE_SIMPLIFIED_INTRA, bti_idx++,
5312                                  0, 1, I965_SURFACEFORMAT_R8_UNORM,
5313                                  NULL, NULL);
5314
5315     gen9_hevc_set_gpe_1d_surface(ctx, priv_ctx, gpe_context,
5316                                  HEVC_ENC_SURFACE_KERNEL_DEBUG, bti_idx++,
5317                                  0, 0, 0, NULL, NULL);
5318 }
5319
5320 static void
5321 gen9_hevc_16x16_sad_pu_computation(VADriverContextP ctx,
5322                                    struct encode_state *encode_state,
5323                                    struct intel_encoder_context *encoder_context)
5324 {
5325     struct encoder_vme_mfc_context *vme_context = NULL;
5326     struct gen9_hevc_encoder_context *priv_ctx = NULL;
5327     struct gen9_hevc_encoder_state *priv_state = NULL;
5328     struct gpe_media_object_walker_parameter param;
5329     struct hevc_enc_kernel_walker_parameter hevc_walker_param;
5330     struct i965_gpe_context *gpe_context = NULL;
5331     int media_state = HEVC_ENC_MEDIA_STATE_16x16_PU_SAD;
5332
5333     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
5334     priv_ctx = (struct gen9_hevc_encoder_context *)vme_context->private_enc_ctx;
5335     priv_state = (struct gen9_hevc_encoder_state *)vme_context->private_enc_state;
5336
5337     gpe_context = &priv_ctx->mbenc_context.gpe_contexts[HEVC_MBENC_16x16SAD_IDX];
5338
5339     gen8_gpe_context_init(ctx, gpe_context);
5340     gen9_gpe_reset_binding_table(ctx, gpe_context);
5341     gen9_hevc_16x16_sad_pu_comp_set_curbe(ctx, encode_state, encoder_context, gpe_context);
5342     gen9_hevc_16x16_sad_pu_comp_set_surfaces(ctx, encode_state, encoder_context, gpe_context);
5343     gen8_gpe_setup_interface_data(ctx, gpe_context);
5344
5345     memset((void *)&hevc_walker_param, 0, sizeof(hevc_walker_param));
5346     hevc_walker_param.resolution_x = ALIGN(priv_state->picture_width, 16) >> 4;
5347     hevc_walker_param.resolution_y = ALIGN(priv_state->picture_height, 16) >> 4;
5348     hevc_walker_param.no_dependency = 1;
5349     gen9_hevc_init_object_walker(&hevc_walker_param, &param);
5350     gen9_hevc_run_object_walker(ctx, encoder_context, gpe_context, &param,
5351                                 media_state);
5352 }
5353
5354 static void
5355 gen9_hevc_32x32_b_intra_set_curbe(VADriverContextP ctx,
5356                                   struct encode_state *encode_state,
5357                                   struct intel_encoder_context *encoder_context,
5358                                   struct i965_gpe_context *gpe_context)
5359 {
5360     struct encoder_vme_mfc_context *vme_context = NULL;
5361     struct generic_enc_codec_state *generic_state = NULL;
5362     struct gen9_hevc_encoder_state *priv_state = NULL;
5363     gen9_hevc_mbenc_b_32x32_pu_intra_curbe_data *cmd = NULL;
5364     VAEncSequenceParameterBufferHEVC *seq_param = NULL;
5365     int bti_idx = 0;
5366
5367     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
5368     generic_state = (struct generic_enc_codec_state *)vme_context->generic_enc_state;
5369     priv_state = (struct gen9_hevc_encoder_state *)vme_context->private_enc_state;
5370     seq_param = (VAEncSequenceParameterBufferHEVC *)encode_state->seq_param_ext->buffer;
5371
5372     cmd = i965_gpe_context_map_curbe(gpe_context);
5373     if (!cmd)
5374         return;
5375
5376     memset((void *)cmd, 0, sizeof(*cmd));
5377
5378     cmd->dw0.frame_width = ALIGN(priv_state->picture_width, 16);
5379     cmd->dw0.frame_height = ALIGN(priv_state->picture_height, 16);
5380     cmd->dw1.enable_debug_dump = 0;
5381     cmd->dw1.enable_intra_early_exit = priv_state->tu_mode == HEVC_TU_RT_SPEED ? 1 : 0;
5382     cmd->dw1.flags = 0;
5383     cmd->dw1.log2_min_tu_size = seq_param->log2_min_transform_block_size_minus2 + 2;
5384     cmd->dw1.slice_type = priv_state->picture_coding_type;
5385     cmd->dw1.hme_enable = generic_state->hme_enabled;
5386     cmd->dw1.fast_surveillance_flag = priv_state->picture_coding_type == HEVC_SLICE_I ?
5387                                       0 : priv_state->video_surveillance_flag;
5388
5389     cmd->dw2.qp_multiplier = 100;
5390     cmd->dw2.qp_value = 0;
5391
5392     cmd->dw8.bti_per_32x32_pu_intra_checck = bti_idx++;
5393     cmd->dw9.bti_src_y = bti_idx++;
5394     bti_idx++;
5395     cmd->dw10.bti_src_y2x = bti_idx++;
5396     cmd->dw11.bti_slice_map = bti_idx++;
5397     cmd->dw12.bti_vme_y2x = bti_idx++;
5398     cmd->dw13.bti_simplest_intra = bti_idx++;
5399     cmd->dw14.bti_hme_mv_pred = bti_idx++;
5400     cmd->dw15.bti_hme_dist = bti_idx++;
5401     cmd->dw16.bti_lcu_skip = bti_idx++;
5402     cmd->dw17.bti_debug = bti_idx++;
5403
5404     i965_gpe_context_unmap_curbe(gpe_context);
5405 }
5406
5407 static void
5408 gen9_hevc_32x32_b_intra_set_surfaces(VADriverContextP ctx,
5409                                      struct encode_state *encode_state,
5410                                      struct intel_encoder_context *encoder_context,
5411                                      struct i965_gpe_context *gpe_context)
5412 {
5413     struct encoder_vme_mfc_context *vme_context = NULL;
5414     struct gen9_hevc_encoder_context *priv_ctx = NULL;
5415     int bti_idx = 0;
5416
5417     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
5418     priv_ctx = (struct gen9_hevc_encoder_context *)vme_context->private_enc_ctx;
5419
5420     gen9_hevc_set_gpe_1d_surface(ctx, priv_ctx, gpe_context,
5421                                  HEVC_ENC_SURFACE_32x32_PU_OUTPUT, bti_idx++,
5422                                  0, 0, 0, NULL, NULL);
5423
5424     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
5425                                  HEVC_ENC_SURFACE_RAW_Y_UV, bti_idx++,
5426                                  1, 1, I965_SURFACEFORMAT_R8_UNORM,
5427                                  NULL, NULL);
5428     bti_idx++;
5429
5430     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
5431                                  HEVC_ENC_SURFACE_Y_2X, bti_idx++,
5432                                  0, 1, I965_SURFACEFORMAT_R8_UNORM,
5433                                  NULL, NULL);
5434
5435     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
5436                                  HEVC_ENC_SURFACE_SLICE_MAP, bti_idx++,
5437                                  0, 1, I965_SURFACEFORMAT_R8_UNORM,
5438                                  NULL, NULL);
5439
5440     gen9_hevc_set_gpe_adv_surface(ctx, priv_ctx, gpe_context,
5441                                   HEVC_ENC_SURFACE_Y_2X_VME, bti_idx++,
5442                                   NULL);
5443
5444     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
5445                                  HEVC_ENC_SURFACE_SIMPLIFIED_INTRA, bti_idx++,
5446                                  0, 1, I965_SURFACEFORMAT_R8_UNORM,
5447                                  NULL, NULL);
5448
5449     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
5450                                  HEVC_ENC_SURFACE_HME_MVP, bti_idx++,
5451                                  0, 1, I965_SURFACEFORMAT_R8_UNORM,
5452                                  NULL, NULL);
5453
5454     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
5455                                  HEVC_ENC_SURFACE_HME_DIST, bti_idx++,
5456                                  0, 1, I965_SURFACEFORMAT_R8_UNORM,
5457                                  NULL, NULL);
5458
5459     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
5460                                  HEVC_ENC_SURFACE_LCU_QP, bti_idx++,
5461                                  0, 1, I965_SURFACEFORMAT_R8_UNORM,
5462                                  NULL, NULL);
5463
5464     gen9_hevc_set_gpe_1d_surface(ctx, priv_ctx, gpe_context,
5465                                  HEVC_ENC_SURFACE_KERNEL_DEBUG, bti_idx,
5466                                  0, 0, 0, NULL, NULL);
5467 }
5468
5469 static void
5470 gen9_hevc_32x32_b_intra(VADriverContextP ctx,
5471                         struct encode_state *encode_state,
5472                         struct intel_encoder_context *encoder_context)
5473 {
5474     struct encoder_vme_mfc_context *vme_context = NULL;
5475     struct gen9_hevc_encoder_context *priv_ctx = NULL;
5476     struct gen9_hevc_encoder_state *priv_state = NULL;
5477     struct gpe_media_object_walker_parameter param;
5478     struct hevc_enc_kernel_walker_parameter hevc_walker_param;
5479     struct i965_gpe_context *gpe_context = NULL;
5480     int media_state = HEVC_ENC_MEDIA_STATE_32x32_B_INTRA_CHECK;
5481
5482     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
5483     priv_ctx = (struct gen9_hevc_encoder_context *)vme_context->private_enc_ctx;
5484     priv_state = (struct gen9_hevc_encoder_state *)vme_context->private_enc_state;
5485
5486     gpe_context = &priv_ctx->mbenc_context.gpe_contexts[HEVC_MBENC_32x32INTRACHECK_IDX];
5487
5488     gen8_gpe_context_init(ctx, gpe_context);
5489     gen9_gpe_reset_binding_table(ctx, gpe_context);
5490     gen9_hevc_32x32_b_intra_set_curbe(ctx, encode_state, encoder_context, gpe_context);
5491     gen9_hevc_32x32_b_intra_set_surfaces(ctx, encode_state, encoder_context, gpe_context);
5492     gen8_gpe_setup_interface_data(ctx, gpe_context);
5493
5494     memset((void *)&hevc_walker_param, 0, sizeof(hevc_walker_param));
5495     hevc_walker_param.resolution_x = ALIGN(priv_state->picture_width, 32) >> 5;
5496     hevc_walker_param.resolution_y = ALIGN(priv_state->picture_height, 32) >> 5;
5497     hevc_walker_param.no_dependency = 1;
5498     gen9_hevc_init_object_walker(&hevc_walker_param, &param);
5499     gen9_hevc_run_object_walker(ctx, encoder_context, gpe_context, &param,
5500                                 media_state);
5501 }
5502
5503 static void
5504 gen9_hevc_32x32_pu_mode_set_curbe(VADriverContextP ctx,
5505                                   struct encode_state *encode_state,
5506                                   struct intel_encoder_context *encoder_context,
5507                                   struct i965_gpe_context *gpe_context)
5508 {
5509     struct encoder_vme_mfc_context *vme_context = NULL;
5510     struct gen9_hevc_encoder_context *priv_ctx = NULL;
5511     struct generic_enc_codec_state *generic_state = NULL;
5512     struct gen9_hevc_encoder_state *priv_state = NULL;
5513     VAEncPictureParameterBufferHEVC *pic_param = NULL;
5514     VAEncSliceParameterBufferHEVC *slice_param = NULL;
5515     gen9_hevc_mbenc_32x32_pu_mode_curbe_data *cmd = NULL;
5516     double lambda_scaling_factor = 1.0, qp_lambda = 0.0, squared_qp_lambda = 0.0;
5517     unsigned int slice_qp = 0, fixed_point_lambda = 0;
5518     int bti_idx = 0;
5519
5520     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
5521     priv_ctx = (struct gen9_hevc_encoder_context *)vme_context->private_enc_ctx;
5522     generic_state = (struct generic_enc_codec_state *)vme_context->generic_enc_state;
5523     priv_state = (struct gen9_hevc_encoder_state *)vme_context->private_enc_state;
5524     pic_param = (VAEncPictureParameterBufferHEVC *)encode_state->pic_param_ext->buffer;
5525     slice_param = (VAEncSliceParameterBufferHEVC *)encode_state->slice_params_ext[0]->buffer;
5526
5527     slice_qp = pic_param->pic_init_qp + slice_param->slice_qp_delta;
5528     gen9_hevc_set_lambda_tables(priv_ctx, HEVC_SLICE_I, HEVC_ENC_INTRA_TRANS_HAAR);
5529     lambda_scaling_factor = 1.0;
5530     qp_lambda = priv_ctx->lambda_md_table[HEVC_SLICE_I][slice_qp];
5531     squared_qp_lambda = qp_lambda * qp_lambda;
5532     fixed_point_lambda = (unsigned int)(lambda_scaling_factor * squared_qp_lambda * (1 << 10));
5533
5534     cmd = i965_gpe_context_map_curbe(gpe_context);
5535     if (!cmd)
5536         return;
5537
5538     memset((void *)cmd, 0, sizeof(*cmd));
5539
5540     cmd->dw0.frame_width = ALIGN(priv_state->picture_width, 16);
5541     cmd->dw0.frame_height = ALIGN(priv_state->picture_height, 16);
5542     cmd->dw1.enable_debug_dump = 0;
5543     cmd->dw1.lcu_type = priv_state->lcu_size == 64 ? 0 : 1;
5544     cmd->dw1.pu_type = 0;
5545     cmd->dw1.brc_enable = !!generic_state->brc_enabled;
5546     cmd->dw1.lcu_brc_enable = priv_state->lcu_brc_enabled;
5547     cmd->dw1.slice_type = priv_state->picture_coding_type;
5548     cmd->dw1.fast_surveillance_flag = priv_state->picture_coding_type == HEVC_SLICE_I ?
5549                                       0 : priv_state->video_surveillance_flag;
5550     cmd->dw1.roi_enable = (priv_state->num_roi > 0);
5551
5552     cmd->dw2.lambda = fixed_point_lambda;
5553     cmd->dw3.mode_cost_32x32 = 0;
5554     cmd->dw4.early_exit = (unsigned int) - 1;
5555
5556     cmd->dw8.bti_32x32_pu_output = bti_idx++;
5557     cmd->dw9.bti_src_y = bti_idx++;
5558     bti_idx++;
5559     cmd->dw10.bti_src_y2x = bti_idx++;
5560     cmd->dw11.bti_slice_map = bti_idx++;
5561     cmd->dw12.bti_src_y2x_vme = bti_idx++;
5562     cmd->dw13.bti_brc_input = bti_idx++;
5563     cmd->dw14.bti_lcu_qp_surface = bti_idx++;
5564     cmd->dw15.bti_brc_data = bti_idx++;
5565     cmd->dw16.bti_kernel_debug = bti_idx++;
5566
5567     i965_gpe_context_unmap_curbe(gpe_context);
5568 }
5569
5570 static void
5571 gen9_hevc_32x32_pu_mode_set_surfaces(VADriverContextP ctx,
5572                                      struct encode_state *encode_state,
5573                                      struct intel_encoder_context *encoder_context,
5574                                      struct i965_gpe_context *gpe_context)
5575 {
5576     struct encoder_vme_mfc_context *vme_context = NULL;
5577     struct gen9_hevc_encoder_context *priv_ctx = NULL;
5578     int bti_idx = 0;
5579
5580     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
5581     priv_ctx = (struct gen9_hevc_encoder_context *)vme_context->private_enc_ctx;
5582
5583     gen9_hevc_set_gpe_1d_surface(ctx, priv_ctx, gpe_context,
5584                                  HEVC_ENC_SURFACE_32x32_PU_OUTPUT, bti_idx++,
5585                                  0, 0, 0, NULL, NULL);
5586
5587     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
5588                                  HEVC_ENC_SURFACE_RAW_Y, bti_idx++,
5589                                  1, 1, I965_SURFACEFORMAT_R8_UNORM,
5590                                  NULL, NULL);
5591     bti_idx++;
5592
5593     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
5594                                  HEVC_ENC_SURFACE_Y_2X, bti_idx++,
5595                                  0, 1, I965_SURFACEFORMAT_R8_UNORM,
5596                                  NULL, NULL);
5597
5598     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
5599                                  HEVC_ENC_SURFACE_SLICE_MAP, bti_idx++,
5600                                  0, 1, I965_SURFACEFORMAT_R8_UNORM,
5601                                  NULL, NULL);
5602
5603     gen9_hevc_set_gpe_adv_surface(ctx, priv_ctx, gpe_context,
5604                                   HEVC_ENC_SURFACE_Y_2X_VME, bti_idx++,
5605                                   NULL);
5606
5607     gen9_hevc_set_gpe_1d_surface(ctx, priv_ctx, gpe_context,
5608                                  HEVC_ENC_SURFACE_BRC_INPUT, bti_idx++,
5609                                  0, 0, 0, NULL, NULL);
5610
5611     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
5612                                  HEVC_ENC_SURFACE_LCU_QP, bti_idx++,
5613                                  0, 1, I965_SURFACEFORMAT_R8_UNORM,
5614                                  NULL, NULL);
5615
5616     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
5617                                  HEVC_ENC_SURFACE_BRC_DATA, bti_idx++,
5618                                  0, 1, I965_SURFACEFORMAT_R8_UNORM,
5619                                  NULL, NULL);
5620
5621     gen9_hevc_set_gpe_1d_surface(ctx, priv_ctx, gpe_context,
5622                                  HEVC_ENC_SURFACE_KERNEL_DEBUG, bti_idx,
5623                                  0, 0, 0, NULL, NULL);
5624 }
5625
5626 static void
5627 gen9_hevc_32x32_pu_mode(VADriverContextP ctx,
5628                         struct encode_state *encode_state,
5629                         struct intel_encoder_context *encoder_context)
5630 {
5631     struct encoder_vme_mfc_context *vme_context = NULL;
5632     struct gen9_hevc_encoder_context *priv_ctx = NULL;
5633     struct gen9_hevc_encoder_state *priv_state = NULL;
5634     struct gpe_media_object_walker_parameter param;
5635     struct hevc_enc_kernel_walker_parameter hevc_walker_param;
5636     struct i965_gpe_context *gpe_context = NULL;
5637     int media_state = HEVC_ENC_MEDIA_STATE_32x32_PU_MODE_DECISION;
5638
5639     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
5640     priv_ctx = (struct gen9_hevc_encoder_context *)vme_context->private_enc_ctx;
5641     priv_state = (struct gen9_hevc_encoder_state *)vme_context->private_enc_state;
5642
5643     gpe_context = &priv_ctx->mbenc_context.gpe_contexts[HEVC_MBENC_32x32MD_IDX];
5644
5645     gen8_gpe_context_init(ctx, gpe_context);
5646     gen9_gpe_reset_binding_table(ctx, gpe_context);
5647     gen9_hevc_32x32_pu_mode_set_curbe(ctx, encode_state, encoder_context, gpe_context);
5648     gen9_hevc_32x32_pu_mode_set_surfaces(ctx, encode_state, encoder_context, gpe_context);
5649     gen8_gpe_setup_interface_data(ctx, gpe_context);
5650
5651     memset((void *)&hevc_walker_param, 0, sizeof(hevc_walker_param));
5652     hevc_walker_param.resolution_x = ALIGN(priv_state->picture_width, 32) >> 5;
5653     hevc_walker_param.resolution_y = ALIGN(priv_state->picture_height, 32) >> 5;
5654     hevc_walker_param.no_dependency = 1;
5655     gen9_hevc_init_object_walker(&hevc_walker_param, &param);
5656     gen9_hevc_run_object_walker(ctx, encoder_context, gpe_context, &param,
5657                                 media_state);
5658 }
5659
5660 static void
5661 gen9_hevc_2x_scaling_set_curbe(VADriverContextP ctx,
5662                                struct encode_state *encode_state,
5663                                struct intel_encoder_context *encoder_context,
5664                                struct i965_gpe_context *gpe_context)
5665 {
5666     struct encoder_vme_mfc_context *vme_context = NULL;
5667     struct gen9_hevc_encoder_state *priv_state = NULL;
5668     gen9_hevc_mbenc_downscaling2x_curbe_data *cmd = NULL;
5669     int bti_idx = 0;
5670
5671     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
5672     priv_state = (struct gen9_hevc_encoder_state *)vme_context->private_enc_state;
5673
5674     cmd = i965_gpe_context_map_curbe(gpe_context);
5675     if (!cmd)
5676         return;
5677
5678     memset((void *)cmd, 0, sizeof(gen9_hevc_mbenc_downscaling2x_curbe_data));
5679
5680     cmd->dw0.pic_width = ALIGN(priv_state->picture_width, 16);
5681     cmd->dw0.pic_height = ALIGN(priv_state->picture_height, 16);
5682
5683     cmd->dw8.bti_src_y = bti_idx++;
5684     cmd->dw9.bit_dst_y = bti_idx;
5685
5686     i965_gpe_context_unmap_curbe(gpe_context);
5687 }
5688
5689 static void
5690 gen9_hevc_2x_scaling_set_surfaces(VADriverContextP ctx,
5691                                   struct encode_state *encode_state,
5692                                   struct intel_encoder_context *encoder_context,
5693                                   struct i965_gpe_context *gpe_context)
5694 {
5695     struct encoder_vme_mfc_context *vme_context = NULL;
5696     struct gen9_hevc_encoder_context *priv_ctx = NULL;
5697     int bti_idx = 0;
5698
5699     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
5700     priv_ctx = (struct gen9_hevc_encoder_context *)vme_context->private_enc_ctx;
5701
5702     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
5703                                  HEVC_ENC_SURFACE_RAW_Y, bti_idx++,
5704                                  0, 1, I965_SURFACEFORMAT_R16_UNORM,
5705                                  NULL, NULL);
5706
5707     gen9_hevc_set_gpe_2d_surface(ctx, priv_ctx, gpe_context,
5708                                  HEVC_ENC_SURFACE_Y_2X, bti_idx,
5709                                  0, 1, I965_SURFACEFORMAT_R16_UNORM,
5710                                  NULL, NULL);
5711 }
5712
5713 static void
5714 gen9_hevc_2x_scaling(VADriverContextP ctx,
5715                      struct encode_state *encode_state,
5716                      struct intel_encoder_context *encoder_context)
5717 {
5718     struct encoder_vme_mfc_context *vme_context = NULL;
5719     struct gen9_hevc_encoder_context *priv_ctx = NULL;
5720     struct gen9_hevc_encoder_state *priv_state = NULL;
5721     struct gpe_media_object_walker_parameter param;
5722     struct hevc_enc_kernel_walker_parameter hevc_walker_param;
5723     struct i965_gpe_context *gpe_context = NULL;
5724     int media_state = HEVC_ENC_MEDIA_STATE_2X_SCALING;
5725
5726     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
5727     priv_ctx = (struct gen9_hevc_encoder_context *)vme_context->private_enc_ctx;
5728     priv_state = (struct gen9_hevc_encoder_state *)vme_context->private_enc_state;
5729
5730     gpe_context = &priv_ctx->mbenc_context.gpe_contexts[HEVC_MBENC_2xSCALING_IDX];
5731
5732     gen8_gpe_context_init(ctx, gpe_context);
5733     gen9_gpe_reset_binding_table(ctx, gpe_context);
5734     gen9_hevc_2x_scaling_set_curbe(ctx, encode_state, encoder_context, gpe_context);
5735     gen9_hevc_2x_scaling_set_surfaces(ctx, encode_state, encoder_context, gpe_context);
5736     gen8_gpe_setup_interface_data(ctx, gpe_context);
5737
5738     memset((void *)&hevc_walker_param, 0, sizeof(hevc_walker_param));
5739     hevc_walker_param.resolution_x = ALIGN(priv_state->picture_width, 32) >> 5;
5740     hevc_walker_param.resolution_y = ALIGN(priv_state->picture_height, 32) >> 5;
5741     hevc_walker_param.no_dependency = 1;
5742     gen9_hevc_init_object_walker(&hevc_walker_param, &param);
5743
5744     gen9_hevc_run_object_walker(ctx, encoder_context, gpe_context, &param,
5745                                 media_state);
5746 }
5747
5748 static void
5749 gen9_hevc_mbenc(VADriverContextP ctx,
5750                 struct encode_state *encode_state,
5751                 struct intel_encoder_context *encoder_context)
5752 {
5753     struct encoder_vme_mfc_context *vme_context = NULL;
5754     struct gen9_hevc_encoder_state *priv_state = NULL;
5755
5756     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
5757     priv_state = (struct gen9_hevc_encoder_state *)vme_context->private_enc_state;
5758
5759     if (!priv_state->bit_depth_luma_minus8)
5760         gen9_hevc_2x_scaling(ctx, encode_state, encoder_context);
5761
5762     if (priv_state->picture_coding_type == HEVC_SLICE_I)
5763         gen9_hevc_32x32_pu_mode(ctx, encode_state, encoder_context);
5764     else
5765         gen9_hevc_32x32_b_intra(ctx, encode_state, encoder_context);
5766
5767     gen9_hevc_16x16_sad_pu_computation(ctx, encode_state, encoder_context);
5768     gen9_hevc_16x16_pu_mode(ctx, encode_state, encoder_context);
5769     gen9_hevc_8x8_pu_mode(ctx, encode_state, encoder_context);
5770     gen9_hevc_8x8_pu_fmode(ctx, encode_state, encoder_context);
5771
5772     if (priv_state->picture_coding_type != HEVC_SLICE_I) {
5773         gen9_hevc_8x8_b_mbenc(ctx, encode_state, encoder_context);
5774         gen9_hevc_8x8_b_pak(ctx, encode_state, encoder_context);
5775     }
5776 }
5777
5778 static VAStatus
5779 gen9_hevc_vme_gpe_init(VADriverContextP ctx,
5780                        struct encode_state *encode_state,
5781                        struct intel_encoder_context *encoder_context)
5782 {
5783     struct encoder_vme_mfc_context *vme_context = NULL;
5784     struct gen9_hevc_encoder_context *priv_ctx = NULL;
5785
5786     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
5787     priv_ctx = (struct gen9_hevc_encoder_context *)vme_context->private_enc_ctx;
5788
5789     i965_zero_gpe_resource(&priv_ctx->res_mb_code_surface);
5790     i965_zero_gpe_resource(&priv_ctx->res_slice_map_buffer);
5791
5792     return VA_STATUS_SUCCESS;
5793 }
5794
5795 static VAStatus
5796 gen9_hevc_vme_gpe_run(VADriverContextP ctx,
5797                       struct encode_state *encode_state,
5798                       struct intel_encoder_context *encoder_context)
5799 {
5800     struct encoder_vme_mfc_context *vme_context = NULL;
5801     struct generic_enc_codec_state *generic_state = NULL;
5802     struct gen9_hevc_encoder_state *priv_state = NULL;
5803
5804     vme_context = encoder_context->vme_context;
5805     generic_state = vme_context->generic_enc_state;
5806     priv_state = vme_context->private_enc_state;
5807
5808     if (generic_state->brc_enabled &&
5809         (generic_state->brc_need_reset || !generic_state->brc_inited)) {
5810         gen9_hevc_brc_init_reset(ctx, encode_state, encoder_context,
5811                                  generic_state->brc_inited ? 1 : 0);
5812         generic_state->brc_need_reset = 0;
5813         generic_state->brc_inited = 1;
5814     }
5815
5816     if (generic_state->hme_supported || generic_state->brc_enabled) {
5817         gen9_hevc_hme_scaling(ctx, encode_state, encoder_context);
5818
5819         if (generic_state->brc_enabled)
5820             gen9_hevc_brc_intra_dist(ctx, encode_state, encoder_context);
5821
5822         if (generic_state->hme_enabled)
5823             gen9_hevc_hme_encode_me(ctx, encode_state, encoder_context);
5824
5825         if (generic_state->brc_enabled)
5826             gen9_hevc_brc_update(ctx, encode_state, encoder_context);
5827     }
5828
5829     if (priv_state->num_roi && !generic_state->brc_enabled)
5830         gen9_hevc_brc_update_lcu_based(ctx, encode_state, encoder_context);
5831
5832     if (priv_state->bit_depth_luma_minus8)
5833         gen9_hevc_ref_frame_depth_conversion(ctx, encode_state, encoder_context);
5834
5835     gen9_hevc_mbenc(ctx, encode_state, encoder_context);
5836
5837     return VA_STATUS_SUCCESS;
5838 }
5839
5840 static VAStatus
5841 gen9_hevc_vme_pipeline(VADriverContextP ctx,
5842                        VAProfile profile,
5843                        struct encode_state *encode_state,
5844                        struct intel_encoder_context *encoder_context)
5845 {
5846     VAStatus va_status = VA_STATUS_SUCCESS;
5847
5848     va_status = gen9_hevc_enc_init_parameters(ctx, encode_state, encoder_context);
5849     if (va_status != VA_STATUS_SUCCESS)
5850         goto EXIT;
5851
5852     va_status = gen9_hevc_vme_gpe_init(ctx, encode_state, encoder_context);
5853     if (va_status != VA_STATUS_SUCCESS)
5854         goto EXIT;
5855
5856     va_status = gen9_hevc_vme_gpe_run(ctx, encode_state, encoder_context);
5857
5858 EXIT:
5859     return va_status;
5860 }
5861
5862 static void
5863 gen9_hevc_vme_scaling_context_init(VADriverContextP ctx,
5864                                    struct intel_encoder_context *encoder_context)
5865 {
5866     struct encoder_vme_mfc_context *vme_context = NULL;
5867     struct gen9_hevc_encoder_context *priv_ctx = NULL;
5868     struct gen9_hevc_encoder_state *priv_state = NULL;
5869     struct i965_gpe_context *gpe_context = NULL;
5870     struct i965_kernel kernel_info;
5871     struct gen9_hevc_scaling_context *scaling_ctx = NULL;
5872     GEN9_ENC_OPERATION kernel_idx;
5873     int curbe_size = 0;
5874     int i = 0;
5875
5876     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
5877     priv_ctx = (struct gen9_hevc_encoder_context *)vme_context->private_enc_ctx;
5878     priv_state = (struct gen9_hevc_encoder_state *)vme_context->private_enc_state;
5879
5880     scaling_ctx = &priv_ctx->scaling_context;
5881
5882     for (i = 0; i < NUM_HEVC_ENC_SCALING; i++) {
5883         if (i == HEVC_ENC_SCALING_4X ||
5884             i == HEVC_ENC_SCALING_16X) {
5885             curbe_size = sizeof(gen9_hevc_scaling4x_curbe_data);
5886             kernel_idx = GEN9_ENC_SCALING4X;
5887         } else if (i == HEVC_ENC_SCALING_32X) {
5888             curbe_size = sizeof(gen9_hevc_scaling2x_curbe_data);
5889             kernel_idx = GEN9_ENC_SCALING2X;
5890         }
5891
5892         gpe_context = &scaling_ctx->gpe_contexts[i];
5893
5894         gen9_hevc_vme_init_gpe_context(ctx, gpe_context,
5895                                        curbe_size,
5896                                        curbe_size);
5897         gen9_hevc_vme_init_scoreboard(gpe_context,
5898                                       0xFF,
5899                                       priv_state->use_hw_scoreboard,
5900                                       priv_state->use_hw_non_stalling_scoreborad);
5901
5902         memset(&kernel_info, 0, sizeof(kernel_info));
5903         gen9_hevc_get_kernel_header_and_size((void *)hevc_enc_kernel_ptr,
5904                                              hevc_enc_kernel_size,
5905                                              kernel_idx,
5906                                              0,
5907                                              &kernel_info);
5908         gen8_gpe_load_kernels(ctx,
5909                               gpe_context,
5910                               &kernel_info,
5911                               1);
5912     }
5913 }
5914
5915 static void
5916 gen9_hevc_vme_me_context_init(VADriverContextP ctx,
5917                               struct intel_encoder_context *encoder_context)
5918 {
5919     struct encoder_vme_mfc_context *vme_context = NULL;
5920     struct gen9_hevc_encoder_context *priv_ctx = NULL;
5921     struct gen9_hevc_encoder_state *priv_state = NULL;
5922     struct i965_gpe_context *gpe_context = NULL;
5923     struct i965_kernel kernel_info;
5924     struct gen9_hevc_me_context *me_ctx = NULL;
5925     int i = 0, j = 0;
5926
5927     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
5928     priv_ctx = (struct gen9_hevc_encoder_context *)vme_context->private_enc_ctx;
5929     priv_state = (struct gen9_hevc_encoder_state *)vme_context->private_enc_state;
5930
5931     me_ctx = &priv_ctx->me_context;
5932
5933     for (i = 0; i < NUM_HEVC_ENC_ME; i++) {
5934         for (j = 0; j < NUM_HEVC_ENC_ME_TYPES; j++) {
5935             gpe_context = &me_ctx->gpe_context[j][i];
5936
5937             gen9_hevc_vme_init_gpe_context(ctx, gpe_context,
5938                                            sizeof(gen9_hevc_me_curbe_data),
5939                                            0);
5940             gen9_hevc_vme_init_scoreboard(gpe_context,
5941                                           0xFF,
5942                                           priv_state->use_hw_scoreboard,
5943                                           priv_state->use_hw_non_stalling_scoreborad);
5944
5945             memset(&kernel_info, 0, sizeof(kernel_info));
5946             gen9_hevc_get_kernel_header_and_size((void *)hevc_enc_kernel_ptr,
5947                                                  hevc_enc_kernel_size,
5948                                                  GEN9_ENC_ME,
5949                                                  i,
5950                                                  &kernel_info);
5951             gen8_gpe_load_kernels(ctx,
5952                                   gpe_context,
5953                                   &kernel_info,
5954                                   1);
5955         }
5956     }
5957 }
5958
5959 static void
5960 gen9_hevc_vme_mbenc_context_init(VADriverContextP ctx,
5961                                  struct intel_encoder_context *encoder_context)
5962 {
5963     struct encoder_vme_mfc_context *vme_context = NULL;
5964     struct gen9_hevc_encoder_context *priv_ctx = NULL;
5965     struct gen9_hevc_encoder_state *priv_state = NULL;
5966     struct i965_gpe_context *gpe_context = NULL;
5967     struct i965_kernel kernel_info;
5968     struct gen9_hevc_mbenc_context *mbenc_ctx = NULL;
5969     int i = 0;
5970
5971     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
5972     priv_ctx = (struct gen9_hevc_encoder_context *)vme_context->private_enc_ctx;
5973     priv_state = (struct gen9_hevc_encoder_state *)vme_context->private_enc_state;
5974
5975     mbenc_ctx = &priv_ctx->mbenc_context;
5976
5977     mbenc_ctx->kernel_num = GEN8_HEVC_ENC_MBENC_TOTAL_NUM;
5978
5979     for (i = 0; i < mbenc_ctx->kernel_num; i++) {
5980         gpe_context = &mbenc_ctx->gpe_contexts[i];
5981
5982         gen9_hevc_vme_init_gpe_context(ctx, gpe_context,
5983                                        hevc_mbenc_curbe_size[i],
5984                                        0);
5985         gen9_hevc_vme_init_scoreboard(gpe_context,
5986                                       0xFF,
5987                                       priv_state->use_hw_scoreboard,
5988                                       priv_state->use_hw_non_stalling_scoreborad);
5989
5990         memset(&kernel_info, 0, sizeof(kernel_info));
5991         gen9_hevc_get_kernel_header_and_size((void *)hevc_enc_kernel_ptr,
5992                                              hevc_enc_kernel_size,
5993                                              GEN9_ENC_MBENC,
5994                                              i,
5995                                              &kernel_info);
5996         gen8_gpe_load_kernels(ctx,
5997                               gpe_context,
5998                               &kernel_info,
5999                               1);
6000     }
6001 }
6002
6003 static void
6004 gen9_hevc_vme_brc_context_init(VADriverContextP ctx,
6005                                struct intel_encoder_context *encoder_context)
6006 {
6007     struct encoder_vme_mfc_context *vme_context = NULL;
6008     struct gen9_hevc_encoder_context *priv_ctx = NULL;
6009     struct gen9_hevc_encoder_state *priv_state = NULL;
6010     struct i965_gpe_context *gpe_context = NULL;
6011     struct i965_kernel kernel_info;
6012     struct gen9_hevc_brc_context *brc_ctx = NULL;
6013     int i = 0;
6014
6015     vme_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
6016     priv_ctx = (struct gen9_hevc_encoder_context *)vme_context->private_enc_ctx;
6017     priv_state = (struct gen9_hevc_encoder_state *)vme_context->private_enc_state;
6018
6019     brc_ctx = &priv_ctx->brc_context;
6020
6021     for (i = 0; i < GEN9_HEVC_ENC_BRC_NUM; i++) {
6022         gpe_context = &brc_ctx->gpe_contexts[i];
6023
6024         gen9_hevc_vme_init_gpe_context(ctx, gpe_context,
6025                                        hevc_brc_curbe_size[i],
6026                                        0);
6027         gen9_hevc_vme_init_scoreboard(gpe_context,
6028                                       0xFF,
6029                                       priv_state->use_hw_scoreboard,
6030                                       priv_state->use_hw_non_stalling_scoreborad);
6031
6032         memset(&kernel_info, 0, sizeof(kernel_info));
6033         gen9_hevc_get_kernel_header_and_size((void *)hevc_enc_kernel_ptr,
6034                                              hevc_enc_kernel_size,
6035                                              GEN9_ENC_BRC,
6036                                              i,
6037                                              &kernel_info);
6038         gen8_gpe_load_kernels(ctx,
6039                               gpe_context,
6040                               &kernel_info,
6041                               1);
6042     }
6043 }
6044
6045 static void
6046 gen9_hevc_vme_scaling_context_destroy(struct gen9_hevc_scaling_context *scaling_context)
6047 {
6048     int i;
6049
6050     for (i = 0; i < NUM_HEVC_ENC_SCALING; i++)
6051         gen8_gpe_context_destroy(&scaling_context->gpe_contexts[i]);
6052 }
6053
6054 static void
6055 gen9_hevc_vme_me_context_destroy(struct gen9_hevc_me_context *me_context)
6056 {
6057     int i, j;
6058
6059     for (i = 0; i < NUM_HEVC_ENC_ME; i++)
6060         for (j = 0; j < NUM_HEVC_ENC_ME_TYPES; j++)
6061             gen8_gpe_context_destroy(&me_context->gpe_context[j][i]);
6062 }
6063
6064 static void
6065 gen9_hevc_vme_mbenc_context_destroy(struct gen9_hevc_mbenc_context *mbenc_context)
6066 {
6067     int i;
6068
6069     for (i = 0; i < mbenc_context->kernel_num; i++)
6070         gen8_gpe_context_destroy(&mbenc_context->gpe_contexts[i]);
6071 }
6072
6073 static void
6074 gen9_hevc_vme_brc_context_destroy(struct gen9_hevc_brc_context *brc_context)
6075 {
6076     int i;
6077
6078     for (i = 0; i < GEN9_HEVC_ENC_BRC_NUM; i++)
6079         gen8_gpe_context_destroy(&brc_context->gpe_contexts[i]);
6080 }
6081
6082 static void
6083 gen9_hevc_vme_kernels_context_init(VADriverContextP ctx,
6084                                    struct intel_encoder_context *encoder_context)
6085 {
6086     gen9_hevc_vme_scaling_context_init(ctx, encoder_context);
6087     gen9_hevc_vme_me_context_init(ctx, encoder_context);
6088     gen9_hevc_vme_mbenc_context_init(ctx, encoder_context);
6089     gen9_hevc_vme_brc_context_init(ctx, encoder_context);
6090 }
6091
6092 static void
6093 gen9_hevc_vme_kernels_context_destroy(struct encoder_vme_mfc_context *vme_context)
6094 {
6095     struct gen9_hevc_encoder_context *priv_ctx = NULL;
6096
6097     priv_ctx = (struct gen9_hevc_encoder_context *)vme_context->private_enc_ctx;
6098
6099     gen9_hevc_vme_scaling_context_destroy(&priv_ctx->scaling_context);
6100     gen9_hevc_vme_me_context_destroy(&priv_ctx->me_context);
6101     gen9_hevc_vme_mbenc_context_destroy(&priv_ctx->mbenc_context);
6102     gen9_hevc_vme_brc_context_destroy(&priv_ctx->brc_context);
6103 }
6104
6105 static void
6106 gen9_hevc_vme_context_destroy(void *context)
6107 {
6108     struct encoder_vme_mfc_context *vme_context = (struct encoder_vme_mfc_context *)context;
6109
6110     if (!vme_context)
6111         return;
6112
6113     gen9_hevc_enc_free_resources(vme_context);
6114
6115     gen9_hevc_vme_kernels_context_destroy(vme_context);
6116
6117     if (vme_context->private_enc_ctx) free(vme_context->private_enc_ctx);
6118     if (vme_context->generic_enc_state) free(vme_context->generic_enc_state);
6119     if (vme_context->private_enc_state) free(vme_context->private_enc_state);
6120
6121     free(vme_context);
6122 }
6123
6124 #define PAK_IMPLEMENTATION_START
6125
6126 static void
6127 gen9_hevc_pak_pipe_mode_select(VADriverContextP ctx,
6128                                struct encode_state *encode_state,
6129                                struct intel_encoder_context *encoder_context)
6130 {
6131     struct intel_batchbuffer *batch = encoder_context->base.batch;
6132
6133     BEGIN_BCS_BATCH(batch, 6);
6134
6135     OUT_BCS_BATCH(batch, HCP_PIPE_MODE_SELECT | (6 - 2));
6136     OUT_BCS_BATCH(batch,
6137                   (0 << 5) |
6138                   (0 << 3) |
6139                   HCP_CODEC_SELECT_ENCODE);
6140     OUT_BCS_BATCH(batch, 0);
6141     OUT_BCS_BATCH(batch, 0);
6142     OUT_BCS_BATCH(batch, 0);
6143     OUT_BCS_BATCH(batch, 0);
6144
6145     ADVANCE_BCS_BATCH(batch);
6146 }
6147
6148 static void
6149 gen9_hevc_pak_add_surface_state(VADriverContextP ctx,
6150                                 struct encode_state *encode_state,
6151                                 struct intel_encoder_context *encoder_context,
6152                                 struct object_surface *obj_surface,
6153                                 enum GEN9_HEVC_ENC_SURFACE_TYPE type)
6154 {
6155     struct intel_batchbuffer *batch = encoder_context->base.batch;
6156
6157     BEGIN_BCS_BATCH(batch, 3);
6158
6159     OUT_BCS_BATCH(batch, HCP_SURFACE_STATE | (3 - 2));
6160     OUT_BCS_BATCH(batch,
6161                   (type << 28) |
6162                   (obj_surface->width - 1));
6163     OUT_BCS_BATCH(batch,
6164                   (((obj_surface->fourcc == VA_FOURCC_P010) ?
6165                     SURFACE_FORMAT_P010 :
6166                     SURFACE_FORMAT_PLANAR_420_8) << 28) |
6167                   (obj_surface->y_cb_offset));
6168
6169     ADVANCE_BCS_BATCH(batch);
6170 }
6171
6172 #define OUT_BUFFER(buf_bo, is_target, ma)                               \
6173     do {                                                                \
6174         if (buf_bo) {                                                   \
6175             OUT_RELOC64(batch,                                          \
6176                           buf_bo,                                       \
6177                           I915_GEM_DOMAIN_RENDER,                       \
6178                           is_target ? I915_GEM_DOMAIN_RENDER : 0,       \
6179                           0);                                           \
6180         } else {                                                        \
6181             OUT_BCS_BATCH(batch, 0);                                    \
6182             OUT_BCS_BATCH(batch, 0);                                    \
6183         }                                                               \
6184         if (ma)                                                         \
6185             OUT_BCS_BATCH(batch, priv_ctx->mocs);                       \
6186     } while (0);
6187
6188 #define OUT_BUFFER_MA_TARGET(buf_bo)       OUT_BUFFER(buf_bo, 1, 1)
6189 #define OUT_BUFFER_MA_REFERENCE(buf_bo)    OUT_BUFFER(buf_bo, 0, 1)
6190 #define OUT_BUFFER_NMA_REFERENCE(buf_bo)   OUT_BUFFER(buf_bo, 0, 0)
6191
6192 static void
6193 gen9_hevc_pak_add_pipe_buf_addr_state(VADriverContextP ctx,
6194                                       struct encode_state *encode_state,
6195                                       struct intel_encoder_context *encoder_context)
6196 {
6197     struct i965_driver_data *i965 = i965_driver_data(ctx);
6198     struct intel_batchbuffer *batch = encoder_context->base.batch;
6199     struct encoder_vme_mfc_context *pak_context = NULL;
6200     struct gen9_hevc_encoder_context *priv_ctx = NULL;
6201     dri_bo *bo = NULL;
6202     int i = 0;
6203
6204     pak_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
6205     priv_ctx = (struct gen9_hevc_encoder_context *)pak_context->private_enc_ctx;
6206
6207     if (IS_KBL(i965->intel.device_info) || IS_GLK(i965->intel.device_info)) {
6208         BEGIN_BCS_BATCH(batch, 104);
6209
6210         OUT_BCS_BATCH(batch, HCP_PIPE_BUF_ADDR_STATE | (104 - 2));
6211     } else {
6212         BEGIN_BCS_BATCH(batch, 95);
6213
6214         OUT_BCS_BATCH(batch, HCP_PIPE_BUF_ADDR_STATE | (95 - 2));
6215     }
6216
6217     OUT_BUFFER_MA_TARGET(priv_ctx->reconstructed_object.obj_surface->bo);
6218     OUT_BUFFER_MA_TARGET(priv_ctx->deblocking_filter_line_buffer.bo);
6219     OUT_BUFFER_MA_TARGET(priv_ctx->deblocking_filter_tile_line_buffer.bo);
6220     OUT_BUFFER_MA_TARGET(priv_ctx->deblocking_filter_tile_column_buffer.bo);
6221     OUT_BUFFER_MA_TARGET(priv_ctx->metadata_line_buffer.bo);
6222     OUT_BUFFER_MA_TARGET(priv_ctx->metadata_tile_line_buffer.bo);
6223     OUT_BUFFER_MA_TARGET(priv_ctx->metadata_tile_column_buffer.bo);
6224     OUT_BUFFER_MA_TARGET(priv_ctx->sao_line_buffer.bo);
6225     OUT_BUFFER_MA_TARGET(priv_ctx->sao_tile_line_buffer.bo);
6226     OUT_BUFFER_MA_TARGET(priv_ctx->sao_tile_column_buffer.bo);
6227     OUT_BUFFER_MA_TARGET(priv_ctx->
6228                          mv_temporal_buffer[GEN9_MAX_MV_TEMPORAL_BUFFERS - 1].bo);
6229     OUT_BUFFER_MA_TARGET(NULL);
6230
6231     for (i = 0; i < GEN9_MAX_REF_SURFACES; i++) {
6232         if (priv_ctx->reference_surfaces[i].obj_surface &&
6233             priv_ctx->reference_surfaces[i].obj_surface->bo) {
6234             bo = priv_ctx->reference_surfaces[i].obj_surface->bo;
6235
6236             OUT_BUFFER_NMA_REFERENCE(bo);
6237         } else
6238             OUT_BUFFER_NMA_REFERENCE(NULL);
6239     }
6240     OUT_BCS_BATCH(batch, priv_ctx->mocs);
6241
6242     OUT_BUFFER_MA_TARGET(priv_ctx->
6243                          uncompressed_picture_source.obj_surface->bo);
6244     OUT_BUFFER_MA_TARGET(NULL);
6245     OUT_BUFFER_MA_TARGET(NULL);
6246     OUT_BUFFER_MA_TARGET(NULL);
6247
6248     for (i = 0; i < GEN9_MAX_MV_TEMPORAL_BUFFERS - 1; i++) {
6249         bo = priv_ctx->mv_temporal_buffer[i].bo;
6250
6251         if (bo) {
6252             OUT_BUFFER_NMA_REFERENCE(bo);
6253         } else
6254             OUT_BUFFER_NMA_REFERENCE(NULL);
6255     }
6256     OUT_BCS_BATCH(batch, priv_ctx->mocs);
6257
6258     OUT_BUFFER_MA_TARGET(NULL);
6259     OUT_BUFFER_MA_TARGET(NULL);
6260     OUT_BUFFER_MA_TARGET(NULL);
6261     OUT_BUFFER_MA_TARGET(NULL);
6262
6263     if (IS_KBL(i965->intel.device_info) || IS_GLK(i965->intel.device_info)) {
6264         for (i = 0; i < 9; i++)
6265             OUT_BCS_BATCH(batch, 0);
6266     }
6267
6268     ADVANCE_BCS_BATCH(batch);
6269 }
6270
6271 static void
6272 gen9_hevc_pak_add_ind_obj_base_addr_state(VADriverContextP ctx,
6273                                           struct encode_state *encode_state,
6274                                           struct intel_encoder_context *encoder_context)
6275 {
6276     struct intel_batchbuffer *batch = encoder_context->base.batch;
6277     struct encoder_vme_mfc_context *pak_context = NULL;
6278     struct gen9_hevc_encoder_context *priv_ctx = NULL;
6279     struct gen9_hevc_encoder_state *priv_state = NULL;
6280
6281     pak_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
6282     priv_ctx = (struct gen9_hevc_encoder_context *)pak_context->private_enc_ctx;
6283     priv_state = (struct gen9_hevc_encoder_state *)pak_context->private_enc_state;
6284
6285     BEGIN_BCS_BATCH(batch, 14);
6286
6287     OUT_BCS_BATCH(batch, HCP_IND_OBJ_BASE_ADDR_STATE | (14 - 2));
6288     OUT_BUFFER_MA_REFERENCE(NULL);
6289     OUT_BUFFER_NMA_REFERENCE(NULL);
6290
6291     OUT_RELOC64(batch,
6292                 priv_ctx->res_mb_code_surface.bo,
6293                 I915_GEM_DOMAIN_INSTRUCTION, 0,
6294                 priv_state->mb_data_offset);
6295     OUT_BCS_BATCH(batch, priv_ctx->mocs);
6296
6297     OUT_RELOC64(batch,
6298                 priv_ctx->indirect_pak_bse_object.bo,
6299                 I915_GEM_DOMAIN_RENDER, I915_GEM_DOMAIN_RENDER,
6300                 priv_ctx->indirect_pak_bse_object.offset);
6301     OUT_BCS_BATCH(batch, priv_ctx->mocs);
6302
6303     OUT_RELOC64(batch,
6304                 priv_ctx->indirect_pak_bse_object.bo,
6305                 I915_GEM_DOMAIN_RENDER, 0,
6306                 priv_ctx->indirect_pak_bse_object.end_offset);
6307
6308     ADVANCE_BCS_BATCH(batch);
6309 }
6310
6311 static void
6312 gen9_hevc_pak_set_qm(VADriverContextP ctx,
6313                      int size_id,
6314                      int color_component,
6315                      int pred_type,
6316                      int dc,
6317                      unsigned int *qm,
6318                      int qm_length,
6319                      struct intel_encoder_context *encoder_context)
6320 {
6321     struct intel_batchbuffer *batch = encoder_context->base.batch;
6322     unsigned int qm_buffer[16];
6323
6324     memset(qm_buffer, 0, sizeof(qm_buffer));
6325     memcpy(qm_buffer, qm, qm_length * 4);
6326
6327     BEGIN_BCS_BATCH(batch, 18);
6328
6329     OUT_BCS_BATCH(batch, HCP_QM_STATE | (18 - 2));
6330     OUT_BCS_BATCH(batch,
6331                   dc << 5 |
6332                   color_component << 3 |
6333                   size_id << 1 |
6334                   pred_type);
6335     intel_batchbuffer_data(batch, qm_buffer, 16 * 4);
6336
6337     ADVANCE_BCS_BATCH(batch);
6338 }
6339
6340 static unsigned int qm_default[16] = {
6341     0x10101010, 0x10101010, 0x10101010, 0x10101010,
6342     0x10101010, 0x10101010, 0x10101010, 0x10101010,
6343     0x10101010, 0x10101010, 0x10101010, 0x10101010,
6344     0x10101010, 0x10101010, 0x10101010, 0x10101010
6345 };
6346
6347 static void
6348 gen9_hevc_pak_add_qm_state(VADriverContextP ctx,
6349                            struct encode_state *encode_state,
6350                            struct intel_encoder_context *encoder_context)
6351 {
6352     int i;
6353
6354     for (i = 0; i < 6; i++)
6355         gen9_hevc_pak_set_qm(ctx,
6356                              0, i % 3, i / 3, 0,
6357                              qm_default, 4,
6358                              encoder_context);
6359
6360     for (i = 0; i < 6; i++)
6361         gen9_hevc_pak_set_qm(ctx,
6362                              1, i % 3, i / 3, 0,
6363                              qm_default, 16,
6364                              encoder_context);
6365
6366     for (i = 0; i < 6; i++)
6367         gen9_hevc_pak_set_qm(ctx,
6368                              2, i % 3, i / 3, 16,
6369                              qm_default, 16,
6370                              encoder_context);
6371
6372     for (i = 0; i < 2; i++)
6373         gen9_hevc_pak_set_qm(ctx,
6374                              3, 0, i % 2, 16,
6375                              qm_default, 16,
6376                              encoder_context);
6377 }
6378
6379 static void
6380 gen9_hevc_pak_set_fqm(VADriverContextP ctx,
6381                       int size_id,
6382                       int color_component,
6383                       int pred_type,
6384                       int dc,
6385                       unsigned int *fqm,
6386                       int fqm_length,
6387                       struct intel_encoder_context *encoder_context)
6388 {
6389     struct intel_batchbuffer *batch = encoder_context->base.batch;
6390     unsigned int fqm_buffer[32];
6391
6392     memset(fqm_buffer, 0, sizeof(fqm_buffer));
6393     memcpy(fqm_buffer, fqm, fqm_length * 4);
6394
6395     BEGIN_BCS_BATCH(batch, 34);
6396
6397     OUT_BCS_BATCH(batch, HCP_FQM_STATE | (34 - 2));
6398     OUT_BCS_BATCH(batch,
6399                   dc << 16 |
6400                   color_component << 3 |
6401                   size_id << 1 |
6402                   pred_type);
6403     intel_batchbuffer_data(batch, fqm_buffer, 32 * 4);
6404
6405     ADVANCE_BCS_BATCH(batch);
6406 }
6407
6408 static unsigned int fm_default[32] = {
6409     0x10001000, 0x10001000, 0x10001000, 0x10001000,
6410     0x10001000, 0x10001000, 0x10001000, 0x10001000,
6411     0x10001000, 0x10001000, 0x10001000, 0x10001000,
6412     0x10001000, 0x10001000, 0x10001000, 0x10001000,
6413     0x10001000, 0x10001000, 0x10001000, 0x10001000,
6414     0x10001000, 0x10001000, 0x10001000, 0x10001000,
6415     0x10001000, 0x10001000, 0x10001000, 0x10001000,
6416     0x10001000, 0x10001000, 0x10001000, 0x10001000
6417 };
6418
6419 static void
6420 gen9_hevc_pak_add_fm_state(VADriverContextP ctx,
6421                            struct encode_state *encode_state,
6422                            struct intel_encoder_context *encoder_context)
6423 {
6424     gen9_hevc_pak_set_fqm(ctx,
6425                           0, 0, 0, 0,
6426                           fm_default, 8,
6427                           encoder_context);
6428     gen9_hevc_pak_set_fqm(ctx,
6429                           0, 0, 1, 0,
6430                           fm_default, 8,
6431                           encoder_context);
6432     gen9_hevc_pak_set_fqm(ctx,
6433                           1, 0, 0, 0,
6434                           fm_default, 32,
6435                           encoder_context);
6436     gen9_hevc_pak_set_fqm(ctx,
6437                           1, 0, 1, 0,
6438                           fm_default, 32,
6439                           encoder_context);
6440     gen9_hevc_pak_set_fqm(ctx,
6441                           2, 0, 0, 0x1000,
6442                           fm_default, 0,
6443                           encoder_context);
6444     gen9_hevc_pak_set_fqm(ctx,
6445                           2, 0, 1, 0x1000,
6446                           fm_default, 0,
6447                           encoder_context);
6448     gen9_hevc_pak_set_fqm(ctx,
6449                           3, 0, 0, 0x1000,
6450                           fm_default, 0,
6451                           encoder_context);
6452     gen9_hevc_pak_set_fqm(ctx,
6453                           3, 0, 1, 0x1000,
6454                           fm_default, 0,
6455                           encoder_context);
6456 }
6457
6458 static void
6459 gen9_hevc_set_reflist(VADriverContextP ctx,
6460                       struct gen9_hevc_encoder_context *priv_ctx,
6461                       VAEncPictureParameterBufferHEVC *pic_param,
6462                       VAEncSliceParameterBufferHEVC *slice_param,
6463                       int list_idx,
6464                       struct intel_batchbuffer *batch)
6465 {
6466     struct i965_driver_data *i965 = i965_driver_data(ctx);
6467     int num_ref_minus1 = (list_idx ?
6468                           slice_param->num_ref_idx_l1_active_minus1 :
6469                           slice_param->num_ref_idx_l0_active_minus1);
6470     VAPictureHEVC *ref_list = (list_idx ?
6471                                slice_param->ref_pic_list1 :
6472                                slice_param->ref_pic_list0);
6473     VAPictureHEVC *curr_pic = &pic_param->decoded_curr_pic;
6474     struct object_surface *obj_surface = NULL;
6475     int frame_idx;
6476     int i = 0, j = 0;
6477
6478     BEGIN_BCS_BATCH(batch, 18);
6479
6480     OUT_BCS_BATCH(batch, HCP_REF_IDX_STATE | (18 - 2));
6481     OUT_BCS_BATCH(batch,
6482                   num_ref_minus1 << 1 |
6483                   list_idx);
6484
6485     for (i = 0; i < 16; i++) {
6486         frame_idx = -1;
6487         obj_surface = SURFACE(ref_list[i].picture_id);
6488         if (i < MIN((num_ref_minus1 + 1), GEN9_MAX_REF_SURFACES) && obj_surface) {
6489             for (j = 0; j < GEN9_MAX_REF_SURFACES; j++) {
6490                 if (obj_surface == priv_ctx->reference_surfaces[j].obj_surface) {
6491                     frame_idx = j;
6492                     break;
6493                 }
6494             }
6495         }
6496
6497         if (i < MIN((num_ref_minus1 + 1), GEN9_MAX_REF_SURFACES) &&
6498             frame_idx >= 0) {
6499             OUT_BCS_BATCH(batch,
6500                           1 << 15 |
6501                           0 << 14 |
6502                           !!(ref_list[i].flags & VA_PICTURE_HEVC_LONG_TERM_REFERENCE) << 13 |
6503                           0 << 12 |
6504                           0 << 11 |
6505                           frame_idx << 8 |
6506                           (CLAMP(-128, 127, curr_pic->pic_order_cnt -
6507                                  ref_list[i].pic_order_cnt) & 0xff));
6508         } else {
6509             OUT_BCS_BATCH(batch, 0);
6510         }
6511     }
6512
6513     ADVANCE_BCS_BATCH(batch);
6514 }
6515
6516 static void
6517 gen9_hevc_pak_add_slice_state(VADriverContextP ctx,
6518                               struct encode_state *encode_state,
6519                               struct intel_encoder_context *encoder_context,
6520                               int slice_idx,
6521                               struct intel_batchbuffer *batch)
6522 {
6523     struct i965_driver_data *i965 = i965_driver_data(ctx);
6524     struct encoder_vme_mfc_context *pak_context = NULL;
6525     struct gen9_hevc_encoder_state *priv_state = NULL;
6526     VAEncPictureParameterBufferHEVC *pic_param = NULL;
6527     VAEncSliceParameterBufferHEVC *slice_param = NULL;
6528     int slice_hor_pos, slice_ver_pos, next_slice_hor_pos, next_slice_ver_pos;
6529     int slice_type = 0, slice_end = 0, last_slice = 0;
6530     int collocated_ref_idx = 0;
6531
6532     pak_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
6533     priv_state = (struct gen9_hevc_encoder_state *)pak_context->private_enc_state;
6534     pic_param = (VAEncPictureParameterBufferHEVC *)encode_state->pic_param_ext->buffer;
6535     slice_param = (VAEncSliceParameterBufferHEVC *)encode_state->slice_params_ext[slice_idx]->buffer;
6536
6537     slice_type = slice_param->slice_type;
6538     slice_end = slice_param->slice_segment_address + slice_param->num_ctu_in_slice;
6539     slice_hor_pos = slice_param->slice_segment_address % priv_state->width_in_lcu;
6540     slice_ver_pos = slice_param->slice_segment_address / priv_state->width_in_lcu;
6541     next_slice_hor_pos = slice_end % priv_state->width_in_lcu;
6542     next_slice_ver_pos = slice_end / priv_state->width_in_lcu;
6543
6544     if (slice_end >= priv_state->width_in_lcu * priv_state->height_in_lcu ||
6545         slice_idx == encode_state->num_slice_params_ext - 1)
6546         last_slice = 1;
6547
6548     if (priv_state->picture_coding_type != HEVC_SLICE_I &&
6549         slice_param->slice_fields.bits.slice_temporal_mvp_enabled_flag &&
6550         slice_param->slice_fields.bits.collocated_from_l0_flag)
6551         collocated_ref_idx = pic_param->collocated_ref_pic_index;
6552
6553     if (IS_KBL(i965->intel.device_info) || IS_GLK(i965->intel.device_info)) {
6554         BEGIN_BCS_BATCH(batch, 11);
6555
6556         OUT_BCS_BATCH(batch, HCP_SLICE_STATE | (11 - 2));
6557     } else {
6558         BEGIN_BCS_BATCH(batch, 9);
6559
6560         OUT_BCS_BATCH(batch, HCP_SLICE_STATE | (9 - 2));
6561     }
6562
6563     OUT_BCS_BATCH(batch,
6564                   slice_ver_pos << 16 |
6565                   slice_hor_pos);
6566     OUT_BCS_BATCH(batch,
6567                   next_slice_ver_pos << 16 |
6568                   next_slice_hor_pos);
6569     OUT_BCS_BATCH(batch,
6570                   (slice_param->slice_cr_qp_offset & 0x1f) << 17 |
6571                   (slice_param->slice_cb_qp_offset & 0x1f) << 12 |
6572                   (pic_param->pic_init_qp + slice_param->slice_qp_delta) << 6 |
6573                   slice_param->slice_fields.bits.slice_temporal_mvp_enabled_flag << 5 |
6574                   slice_param->slice_fields.bits.dependent_slice_segment_flag << 4 |
6575                   last_slice << 2 |
6576                   slice_type);
6577     OUT_BCS_BATCH(batch,
6578                   collocated_ref_idx << 26 |
6579                   (slice_param->max_num_merge_cand - 1)  << 23 |
6580                   slice_param->slice_fields.bits.cabac_init_flag << 22 |
6581                   slice_param->luma_log2_weight_denom << 19 |
6582                   (slice_param->luma_log2_weight_denom + slice_param->delta_chroma_log2_weight_denom) << 16 |
6583                   slice_param->slice_fields.bits.collocated_from_l0_flag << 15 |
6584                   priv_state->low_delay << 14 |
6585                   slice_param->slice_fields.bits.mvd_l1_zero_flag << 13 |
6586                   slice_param->slice_fields.bits.slice_sao_luma_flag << 12 |
6587                   slice_param->slice_fields.bits.slice_sao_chroma_flag << 11 |
6588                   slice_param->slice_fields.bits.slice_loop_filter_across_slices_enabled_flag << 10 |
6589                   (slice_param->slice_beta_offset_div2 & 0xf) << 5 |
6590                   (slice_param->slice_tc_offset_div2 & 0xf) << 1 |
6591                   slice_param->slice_fields.bits.slice_deblocking_filter_disabled_flag);
6592     OUT_BCS_BATCH(batch, 0);
6593
6594     if (!pic_param->pic_fields.bits.reference_pic_flag &&
6595         priv_state->picture_coding_type != HEVC_SLICE_I)
6596         OUT_BCS_BATCH(batch, 0 << 26 |
6597                       8 << 20);
6598     else
6599         OUT_BCS_BATCH(batch, 5 << 26 |
6600                       11 << 20);
6601
6602     OUT_BCS_BATCH(batch,
6603                   1 << 10 |
6604                   1 << 9  |
6605                   1 << 2  |
6606                   1 << 1  |
6607                   0);
6608     OUT_BCS_BATCH(batch, 0);
6609
6610     if (IS_KBL(i965->intel.device_info) || IS_GLK(i965->intel.device_info)) {
6611         OUT_BCS_BATCH(batch, 0);
6612         OUT_BCS_BATCH(batch, 0);
6613     }
6614
6615     ADVANCE_BCS_BATCH(batch);
6616 }
6617
6618 static int
6619 gen9_hevc_find_skipemulcnt(unsigned char *buf, unsigned int bits_length)
6620 {
6621     int skip_cnt = 0, i = 0;
6622
6623     if ((bits_length >> 3) < 6)
6624         return 0;
6625
6626     for (i = 0; i < 3; i++)
6627         if (buf[i] != 0)
6628             break;
6629
6630     if (i > 1) {
6631         if (buf[i] == 1)
6632             skip_cnt = i + 3;
6633     }
6634
6635     return skip_cnt;
6636 }
6637
6638 static void
6639 gen9_hevc_pak_insert_object(unsigned int *data_buffer,
6640                             unsigned int data_size,
6641                             unsigned char emulation_flag,
6642                             int is_last_header,
6643                             int is_end_of_slice,
6644                             int skip_emul_byte_cnt,
6645                             struct intel_batchbuffer *batch)
6646 {
6647     int length_in_dws = ALIGN(data_size, 32) >> 5;
6648     int data_bits_in_last_dw = data_size & 0x1f;
6649     int skip_cnt = skip_emul_byte_cnt;
6650
6651     if (data_bits_in_last_dw == 0)
6652         data_bits_in_last_dw = 32;
6653
6654     if (emulation_flag) {
6655         if (!skip_cnt)
6656             skip_cnt = gen9_hevc_find_skipemulcnt((unsigned char *)data_buffer,
6657                                                   data_size);
6658     }
6659
6660     BEGIN_BCS_BATCH(batch, length_in_dws + 2);
6661
6662     OUT_BCS_BATCH(batch, HCP_INSERT_PAK_OBJECT | (length_in_dws + 2 - 2));
6663     OUT_BCS_BATCH(batch,
6664                   (0 << 31) |
6665                   (0 << 16) |
6666                   (0 << 15) |
6667                   (data_bits_in_last_dw << 8) |
6668                   (skip_cnt << 4) |
6669                   ((!!emulation_flag) << 3) |
6670                   ((!!is_last_header) << 2) |
6671                   ((!!is_end_of_slice) << 1) |
6672                   (0 << 0));
6673     intel_batchbuffer_data(batch, data_buffer, length_in_dws * 4);
6674
6675     ADVANCE_BCS_BATCH(batch);
6676 }
6677
6678 static void
6679 gen9_hevc_pak_add_refs(VADriverContextP ctx,
6680                        struct encode_state *encode_state,
6681                        struct intel_encoder_context *encoder_context,
6682                        int slice_idx,
6683                        struct intel_batchbuffer *batch)
6684 {
6685     struct encoder_vme_mfc_context *pak_context = NULL;
6686     struct gen9_hevc_encoder_context *priv_ctx = NULL;
6687     VAEncPictureParameterBufferHEVC *pic_param = NULL;
6688     VAEncSliceParameterBufferHEVC *slice_param = NULL;
6689
6690     pak_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
6691     priv_ctx = (struct gen9_hevc_encoder_context *)pak_context->private_enc_ctx;
6692     pic_param = (VAEncPictureParameterBufferHEVC *)encode_state->pic_param_ext->buffer;
6693     slice_param = (VAEncSliceParameterBufferHEVC *)encode_state->slice_params_ext[slice_idx]->buffer;
6694
6695     if (slice_param->slice_type == HEVC_SLICE_I)
6696         return;
6697
6698     gen9_hevc_set_reflist(ctx, priv_ctx, pic_param, slice_param, 0, batch);
6699
6700     if (slice_param->slice_type == HEVC_SLICE_P)
6701         return;
6702
6703     gen9_hevc_set_reflist(ctx, priv_ctx, pic_param, slice_param, 1, batch);
6704 }
6705
6706 static void
6707 gen9_hevc_pak_insert_packed_data(VADriverContextP ctx,
6708                                  struct encode_state *encode_state,
6709                                  struct intel_encoder_context *encoder_context,
6710                                  int slice_idx,
6711                                  struct intel_batchbuffer *batch)
6712 {
6713     VAEncPackedHeaderParameterBuffer *param = NULL;
6714     unsigned int *header_data = NULL;
6715     unsigned int length_in_bits = 0;
6716     int packed_type = 0;
6717     int idx = 0, idx_offset = 0;
6718     int i = 0;
6719
6720     for (i = 0; i < 4; i++) {
6721         idx_offset = 0;
6722         switch (i) {
6723         case 0:
6724             packed_type = VAEncPackedHeaderHEVC_VPS;
6725             break;
6726         case 1:
6727             packed_type = VAEncPackedHeaderHEVC_VPS;
6728             idx_offset = 1;
6729             break;
6730         case 2:
6731             packed_type = VAEncPackedHeaderHEVC_PPS;
6732             break;
6733         case 3:
6734             packed_type = VAEncPackedHeaderHEVC_SEI;
6735             break;
6736         default:
6737             break;
6738         }
6739
6740         idx = va_enc_packed_type_to_idx(packed_type) + idx_offset;
6741         if (encode_state->packed_header_data[idx]) {
6742             param = (VAEncPackedHeaderParameterBuffer *)encode_state->packed_header_param[idx]->buffer;
6743             header_data = (unsigned int *)encode_state->packed_header_data[idx]->buffer;
6744             length_in_bits = param->bit_length;
6745
6746             gen9_hevc_pak_insert_object(header_data, length_in_bits,
6747                                         !param->has_emulation_bytes, 0, 0, 0,
6748                                         batch);
6749         }
6750     }
6751 }
6752
6753 static void
6754 gen9_hevc_pak_insert_slice_header(VADriverContextP ctx,
6755                                   struct encode_state *encode_state,
6756                                   struct intel_encoder_context *encoder_context,
6757                                   int slice_idx,
6758                                   struct intel_batchbuffer *batch)
6759 {
6760     VAEncPackedHeaderParameterBuffer *param = NULL;
6761     unsigned int *header_data = NULL;
6762     unsigned int length_in_bits = 0;
6763     int count = 0, start_index = -1;
6764     int i = 0;
6765
6766     count = encode_state->slice_rawdata_count[slice_idx];
6767     start_index = encode_state->slice_rawdata_index[slice_idx] &
6768                   SLICE_PACKED_DATA_INDEX_MASK;
6769
6770     for (i = 0; i < count; i++) {
6771         param = (VAEncPackedHeaderParameterBuffer *)
6772                 (encode_state->packed_header_params_ext[start_index + i]->buffer);
6773
6774         if (param->type == VAEncPackedHeaderSlice)
6775             continue;
6776
6777         header_data = (unsigned int *)encode_state->packed_header_data_ext[start_index]->buffer;
6778         length_in_bits = param->bit_length;
6779         gen9_hevc_pak_insert_object(header_data, length_in_bits,
6780                                     !param->has_emulation_bytes, 0, 0, 0,
6781                                     batch);
6782     }
6783
6784     start_index = -1;
6785     if (encode_state->slice_header_index[slice_idx] & SLICE_PACKED_DATA_INDEX_TYPE)
6786         start_index = encode_state->slice_header_index[slice_idx] &
6787                       SLICE_PACKED_DATA_INDEX_MASK;
6788     if (start_index == -1) {
6789         VAEncSequenceParameterBufferHEVC *seq_param = (VAEncSequenceParameterBufferHEVC *)encode_state->seq_param_ext->buffer;
6790         VAEncPictureParameterBufferHEVC *pic_param = (VAEncPictureParameterBufferHEVC *)encode_state->pic_param_ext->buffer;
6791         VAEncSliceParameterBufferHEVC *slice_param = (VAEncSliceParameterBufferHEVC *)encode_state->slice_params_ext[slice_idx]->buffer;
6792         unsigned char *slice_header = NULL;
6793         int slice_header_bits = 0;
6794
6795         slice_header_bits = build_hevc_slice_header(seq_param,
6796                                                     pic_param,
6797                                                     slice_param,
6798                                                     &slice_header,
6799                                                     0);
6800
6801         gen9_hevc_pak_insert_object((unsigned int *)slice_header, slice_header_bits,
6802                                     1, 1, 0, 5,
6803                                     batch);
6804
6805         free(slice_header);
6806     } else {
6807         param = (VAEncPackedHeaderParameterBuffer *)
6808                 (encode_state->packed_header_params_ext[start_index]->buffer);
6809         header_data = (unsigned int *)encode_state->packed_header_data_ext[start_index]->buffer;
6810         length_in_bits = param->bit_length;
6811
6812         gen9_hevc_pak_insert_object(header_data, length_in_bits,
6813                                     !param->has_emulation_bytes, 1, 0, 0,
6814                                     batch);
6815     }
6816 }
6817
6818 static VAStatus
6819 gen9_hevc_pak_slice_level(VADriverContextP ctx,
6820                           struct encode_state *encode_state,
6821                           struct intel_encoder_context *encoder_context,
6822                           int slice_idx)
6823 {
6824     struct intel_batchbuffer *batch = encoder_context->base.batch;
6825     struct encoder_vme_mfc_context *pak_context = NULL;
6826     struct gen9_hevc_encoder_context *priv_ctx = NULL;
6827     struct generic_enc_codec_state *generic_state = NULL;
6828     struct gen9_hevc_encoder_state *priv_state = NULL;
6829     VAEncSliceParameterBufferHEVC *slice_param = NULL;
6830     struct intel_batchbuffer *slice_batch = NULL;
6831     struct gpe_mi_batch_buffer_start_parameter second_level_batch;
6832     VAStatus va_status = VA_STATUS_SUCCESS;
6833     int slice_offset = 0;
6834
6835     pak_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
6836     priv_ctx = (struct gen9_hevc_encoder_context *)pak_context->private_enc_ctx;
6837     generic_state = (struct generic_enc_codec_state *)pak_context->generic_enc_state;
6838     priv_state = (struct gen9_hevc_encoder_state *)pak_context->private_enc_state;
6839     slice_param = (VAEncSliceParameterBufferHEVC *)encode_state->slice_params_ext[slice_idx]->buffer;
6840
6841     slice_batch = priv_ctx->res_pak_slice_batch_buffer;
6842
6843     if (generic_state->curr_pak_pass == 0) {
6844         slice_offset = intel_batchbuffer_used_size(slice_batch);
6845         priv_state->slice_batch_offset[slice_idx] = slice_offset;
6846
6847         if (slice_idx < encode_state->num_slice_params_ext - 1)
6848             priv_state->slice_start_lcu[slice_idx + 1] =
6849                 priv_state->slice_start_lcu[slice_idx] +
6850                 slice_param->num_ctu_in_slice;
6851
6852         gen9_hevc_pak_add_refs(ctx, encode_state, encoder_context,
6853                                slice_idx, slice_batch);
6854         gen9_hevc_pak_add_slice_state(ctx, encode_state, encoder_context,
6855                                       slice_idx, slice_batch);
6856
6857         if (slice_idx == 0)
6858             gen9_hevc_pak_insert_packed_data(ctx, encode_state, encoder_context,
6859                                              slice_idx, slice_batch);
6860
6861         gen9_hevc_pak_insert_slice_header(ctx, encode_state, encoder_context,
6862                                           slice_idx, slice_batch);
6863
6864         BEGIN_BCS_BATCH(slice_batch, 2);
6865         OUT_BCS_BATCH(slice_batch, 0);
6866         OUT_BCS_BATCH(slice_batch, MI_BATCH_BUFFER_END);
6867         ADVANCE_BCS_BATCH(slice_batch);
6868     } else
6869         slice_offset = priv_state->slice_batch_offset[slice_idx];
6870
6871     memset(&second_level_batch, 0, sizeof(second_level_batch));
6872     second_level_batch.offset = slice_offset;
6873     second_level_batch.is_second_level = 1;
6874     second_level_batch.bo = slice_batch->buffer;
6875     gen8_gpe_mi_batch_buffer_start(ctx, batch, &second_level_batch);
6876
6877     memset(&second_level_batch, 0, sizeof(second_level_batch));
6878     second_level_batch.offset = priv_state->slice_start_lcu[slice_idx] *
6879                                 priv_state->pak_obj_size;
6880     second_level_batch.is_second_level = 1;
6881     second_level_batch.bo = priv_ctx->res_mb_code_surface.bo;
6882     gen8_gpe_mi_batch_buffer_start(ctx, batch, &second_level_batch);
6883
6884     return va_status;
6885 }
6886
6887 static VAStatus
6888 gen9_hevc_pak_pipeline_prepare(VADriverContextP ctx,
6889                                struct encode_state *encode_state,
6890                                struct intel_encoder_context *encoder_context)
6891 {
6892     struct i965_driver_data *i965 = i965_driver_data(ctx);
6893     struct encoder_vme_mfc_context *pak_context = NULL;
6894     struct gen9_hevc_encoder_context *priv_ctx = NULL;
6895     struct gen9_hevc_encoder_state *priv_state = NULL;
6896     struct gen9_hevc_surface_priv *surface_priv;
6897     VAEncPictureParameterBufferHEVC *pic_param = NULL;
6898     struct i965_coded_buffer_segment *coded_buffer_segment = NULL;
6899     struct object_surface *obj_surface = NULL;
6900     struct object_buffer *obj_buffer = NULL;
6901     dri_bo *bo = NULL;
6902     int i = 0;
6903
6904     pak_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
6905     priv_ctx = (struct gen9_hevc_encoder_context *)pak_context->private_enc_ctx;
6906     priv_state = (struct gen9_hevc_encoder_state *)pak_context->private_enc_state;
6907     pic_param = (VAEncPictureParameterBufferHEVC *)encode_state->pic_param_ext->buffer;
6908
6909     if (priv_ctx->uncompressed_picture_source.obj_surface &&
6910         priv_ctx->uncompressed_picture_source.obj_surface->bo)
6911         dri_bo_unreference(priv_ctx->uncompressed_picture_source.obj_surface->bo);
6912     priv_ctx->uncompressed_picture_source.obj_surface = encode_state->input_yuv_object;
6913     priv_ctx->uncompressed_picture_source.surface_id = encoder_context->input_yuv_surface;
6914     dri_bo_reference(priv_ctx->uncompressed_picture_source.obj_surface->bo);
6915
6916     if (priv_ctx->reconstructed_object.obj_surface &&
6917         priv_ctx->reconstructed_object.obj_surface->bo)
6918         dri_bo_unreference(priv_ctx->reconstructed_object.obj_surface->bo);
6919     priv_ctx->reconstructed_object.obj_surface = encode_state->reconstructed_object;
6920     priv_ctx->reconstructed_object.surface_id = pic_param->decoded_curr_pic.picture_id;
6921     dri_bo_reference(priv_ctx->reconstructed_object.obj_surface->bo);
6922
6923     surface_priv = (struct gen9_hevc_surface_priv *)encode_state->reconstructed_object->private_data;
6924     if (surface_priv) {
6925         if (priv_ctx->mv_temporal_buffer[GEN9_MAX_MV_TEMPORAL_BUFFERS - 1].bo)
6926             dri_bo_unreference(priv_ctx->mv_temporal_buffer[GEN9_MAX_MV_TEMPORAL_BUFFERS - 1].bo);
6927         priv_ctx->mv_temporal_buffer[GEN9_MAX_MV_TEMPORAL_BUFFERS - 1].bo =
6928             surface_priv->motion_vector_temporal_bo;
6929         dri_bo_reference(surface_priv->motion_vector_temporal_bo);
6930     }
6931
6932     if (priv_state->picture_coding_type != HEVC_SLICE_I) {
6933         for (i = 0; i < GEN9_MAX_REF_SURFACES; i++) {
6934             obj_surface = encode_state->reference_objects[i];
6935             if (obj_surface && obj_surface->bo) {
6936                 if (priv_ctx->reference_surfaces[i].obj_surface &&
6937                     priv_ctx->reference_surfaces[i].obj_surface->bo)
6938                     dri_bo_unreference(priv_ctx->reference_surfaces[i].obj_surface->bo);
6939                 priv_ctx->reference_surfaces[i].obj_surface = obj_surface;
6940                 priv_ctx->reference_surfaces[i].surface_id = pic_param->reference_frames[i].picture_id;
6941                 dri_bo_reference(obj_surface->bo);
6942
6943                 surface_priv = (struct gen9_hevc_surface_priv *) obj_surface->private_data;
6944                 if (surface_priv) {
6945                     if (priv_ctx->mv_temporal_buffer[i].bo)
6946                         dri_bo_unreference(priv_ctx->mv_temporal_buffer[i].bo);
6947                     priv_ctx->mv_temporal_buffer[i].bo = surface_priv->motion_vector_temporal_bo;
6948                     dri_bo_reference(surface_priv->motion_vector_temporal_bo);
6949                 }
6950             } else {
6951                 break;
6952             }
6953         }
6954     }
6955
6956     obj_buffer = encode_state->coded_buf_object;
6957     bo = obj_buffer->buffer_store->bo;
6958
6959     if (priv_ctx->indirect_pak_bse_object.bo)
6960         dri_bo_unreference(priv_ctx->indirect_pak_bse_object.bo);
6961     priv_ctx->indirect_pak_bse_object.offset = I965_CODEDBUFFER_HEADER_SIZE;
6962     priv_ctx->indirect_pak_bse_object.end_offset = ALIGN((obj_buffer->size_element - 0x1000), 0x1000);
6963     priv_ctx->indirect_pak_bse_object.bo = bo;
6964     dri_bo_reference(priv_ctx->indirect_pak_bse_object.bo);
6965
6966     dri_bo_map(bo, 1);
6967
6968     if (!bo->virtual)
6969         return VA_STATUS_ERROR_INVALID_VALUE;
6970
6971     coded_buffer_segment = (struct i965_coded_buffer_segment *)bo->virtual;
6972     coded_buffer_segment->mapped = 0;
6973     coded_buffer_segment->codec = encoder_context->codec;
6974     coded_buffer_segment->status_support = 1;
6975
6976     dri_bo_unmap(bo);
6977
6978     if (priv_ctx->res_pak_slice_batch_buffer)
6979         intel_batchbuffer_free(priv_ctx->res_pak_slice_batch_buffer);
6980
6981     priv_ctx->res_pak_slice_batch_buffer =
6982         intel_batchbuffer_new(&i965->intel, I915_EXEC_BSD,
6983                               GEN9_HEVC_ENC_PAK_SLICE_STATE_SIZE *
6984                               encode_state->num_slice_params_ext);
6985     if (!priv_ctx->res_pak_slice_batch_buffer)
6986         return VA_STATUS_ERROR_ALLOCATION_FAILED;
6987
6988     for (i = 0; i < NUM_SLICES; i++) {
6989         priv_state->slice_batch_offset[i] = 0;
6990         priv_state->slice_start_lcu[i] = 0;
6991     }
6992
6993     return VA_STATUS_SUCCESS;
6994 }
6995
6996 static VAStatus
6997 gen9_hevc_pak_picture_level(VADriverContextP ctx,
6998                             struct encode_state *encode_state,
6999                             struct intel_encoder_context *encoder_context)
7000 {
7001     struct intel_batchbuffer *batch = encoder_context->base.batch;
7002     struct encoder_vme_mfc_context *pak_context = NULL;
7003     struct gen9_hevc_encoder_context *priv_ctx = NULL;
7004     struct generic_enc_codec_state *generic_state = NULL;
7005     struct gen9_hevc_encoder_state *priv_state = NULL;
7006     struct gpe_mi_batch_buffer_start_parameter second_level_batch;
7007     struct hevc_encode_status_buffer *status_buffer = NULL;
7008     VAStatus va_status = VA_STATUS_SUCCESS;
7009     int i = 0;
7010
7011     pak_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
7012     priv_ctx = (struct gen9_hevc_encoder_context *)pak_context->private_enc_ctx;
7013     generic_state = (struct generic_enc_codec_state *)pak_context->generic_enc_state;
7014     priv_state = (struct gen9_hevc_encoder_state *)pak_context->private_enc_state;
7015     status_buffer = &priv_state->status_buffer;
7016
7017     if (generic_state->brc_enabled &&
7018         generic_state->curr_pak_pass) {
7019         gen9_hevc_conditional_end(ctx,
7020                                   batch, status_buffer->bo,
7021                                   status_buffer->status_image_mask_offset,
7022                                   0);
7023
7024         gen9_hevc_load_reg_mem(ctx,
7025                                batch, status_buffer->bo,
7026                                status_buffer->status_image_ctrl_offset,
7027                                status_buffer->mmio_image_ctrl_offset);
7028
7029         gen9_hevc_store_reg_mem(ctx, batch, priv_ctx->res_brc_pak_statistic_buffer.bo,
7030                                 offsetof(GEN9_HEVC_PAK_STATES, HEVC_ENC_IMAGE_STATUS_CONTROL_FOR_LAST_PASS),
7031                                 status_buffer->mmio_image_ctrl_offset);
7032
7033         gen9_hevc_store_reg_mem(ctx, batch, status_buffer->bo,
7034                                 status_buffer->status_image_ctrl_last_pass_offset,
7035                                 status_buffer->mmio_image_ctrl_offset);
7036     }
7037
7038     gen9_hevc_pak_pipe_mode_select(ctx, encode_state, encoder_context);
7039     gen9_hevc_pak_add_surface_state(ctx, encode_state, encoder_context,
7040                                     encode_state->input_yuv_object,
7041                                     GEN9_HEVC_ENC_SURFACE_SOURCE);
7042     gen9_hevc_pak_add_surface_state(ctx, encode_state, encoder_context,
7043                                     encode_state->reconstructed_object,
7044                                     GEN9_HEVC_ENC_SURFACE_RECON);
7045     gen9_hevc_pak_add_pipe_buf_addr_state(ctx, encode_state, encoder_context);
7046     gen9_hevc_pak_add_ind_obj_base_addr_state(ctx, encode_state, encoder_context);
7047     gen9_hevc_pak_add_qm_state(ctx, encode_state, encoder_context);
7048     gen9_hevc_pak_add_fm_state(ctx, encode_state, encoder_context);
7049
7050     if (generic_state->brc_enabled) {
7051         memset(&second_level_batch, 0, sizeof(second_level_batch));
7052
7053         second_level_batch.offset = generic_state->curr_pak_pass * priv_state->pic_state_size;
7054         second_level_batch.bo = priv_ctx->res_brc_pic_states_write_buffer.bo;
7055         second_level_batch.is_second_level = 1;
7056         gen8_gpe_mi_batch_buffer_start(ctx, batch, &second_level_batch);
7057     } else
7058         gen9_hevc_add_pic_state(ctx, encode_state, encoder_context, NULL, 0, 0);
7059
7060     for (i = 0; i < encode_state->num_slice_params_ext; i++) {
7061         va_status = gen9_hevc_pak_slice_level(ctx, encode_state, encoder_context, i);
7062         if (va_status != VA_STATUS_SUCCESS)
7063             goto EXIT;
7064     }
7065
7066 EXIT:
7067     return va_status;
7068 }
7069
7070 static void
7071 gen9_hevc_pak_read_status(VADriverContextP ctx,
7072                           struct intel_encoder_context *encoder_context)
7073 {
7074     struct intel_batchbuffer *batch = encoder_context->base.batch;
7075     struct encoder_vme_mfc_context *pak_context = NULL;
7076     struct gen9_hevc_encoder_context *priv_ctx = NULL;
7077     struct generic_enc_codec_state *generic_state = NULL;
7078     struct gen9_hevc_encoder_state *priv_state = NULL;
7079     struct gpe_mi_flush_dw_parameter mi_flush_dw_param;
7080     struct hevc_encode_status_buffer *status_buffer = NULL;
7081
7082     pak_context = (struct encoder_vme_mfc_context *)encoder_context->vme_context;
7083     priv_ctx = (struct gen9_hevc_encoder_context *)pak_context->private_enc_ctx;
7084     generic_state = (struct generic_enc_codec_state *)pak_context->generic_enc_state;
7085     priv_state = (struct gen9_hevc_encoder_state *)pak_context->private_enc_state;
7086
7087     status_buffer = &priv_state->status_buffer;
7088
7089     memset(&mi_flush_dw_param, 0, sizeof(mi_flush_dw_param));
7090     gen8_gpe_mi_flush_dw(ctx, batch, &mi_flush_dw_param);
7091
7092     gen9_hevc_store_reg_mem(ctx, batch, status_buffer->bo,
7093                             status_buffer->status_bs_byte_count_offset,
7094                             status_buffer->mmio_bs_frame_offset);
7095
7096     gen9_hevc_store_reg_mem(ctx, batch, status_buffer->bo,
7097                             status_buffer->status_image_mask_offset,
7098                             status_buffer->mmio_image_mask_offset);
7099
7100     gen9_hevc_store_reg_mem(ctx, batch, status_buffer->bo,
7101                             status_buffer->status_image_ctrl_offset,
7102                             status_buffer->mmio_image_ctrl_offset);
7103
7104     if (generic_state->brc_enabled) {
7105         gen9_hevc_store_reg_mem(ctx, batch, priv_ctx->res_brc_pak_statistic_buffer.bo,
7106                                 offsetof(GEN9_HEVC_PAK_STATES, HEVC_ENC_BYTECOUNT_FRAME),
7107                                 status_buffer->mmio_bs_frame_offset);
7108         gen9_hevc_store_reg_mem(ctx, batch, priv_ctx->res_brc_pak_statistic_buffer.bo,
7109                                 offsetof(GEN9_HEVC_PAK_STATES, HEVC_ENC_BYTECOUNT_FRAME_NOHEADER),
7110                                 status_buffer->mmio_bs_frame_no_header_offset);
7111         gen9_hevc_store_reg_mem(ctx, batch, priv_ctx->res_brc_pak_statistic_buffer.bo,
7112                                 offsetof(GEN9_HEVC_PAK_STATES, HEVC_ENC_IMAGE_STATUS_CONTROL),
7113                                 status_buffer->mmio_image_ctrl_offset);
7114     }
7115
7116     gen8_gpe_mi_flush_dw(ctx, batch, &mi_flush_dw_param);
7117 }
7118
7119 static VAStatus
7120 gen9_hevc_pak_pipeline(VADriverContextP ctx,
7121                        VAProfile profile,
7122                        struct encode_state *encode_state,
7123                        struct intel_encoder_context *encoder_context)
7124 {
7125     struct i965_driver_data *i965 = i965_driver_data(ctx);
7126     struct intel_batchbuffer *batch = encoder_context->base.batch;
7127     struct encoder_vme_mfc_context *pak_context = encoder_context->vme_context;
7128     struct gen9_hevc_encoder_context *priv_ctx = NULL;
7129     struct generic_enc_codec_state *generic_state = NULL;
7130     struct gen9_hevc_encoder_state *priv_state = NULL;
7131     VAStatus va_status = VA_STATUS_SUCCESS;
7132
7133     if (!pak_context || !pak_context->generic_enc_state || !batch) {
7134         va_status = VA_STATUS_ERROR_INVALID_CONTEXT;
7135         goto EXIT;
7136     }
7137
7138     va_status = gen9_hevc_pak_pipeline_prepare(ctx, encode_state, encoder_context);
7139     if (va_status != VA_STATUS_SUCCESS)
7140         goto EXIT;
7141
7142     if (i965->intel.has_bsd2)
7143         intel_batchbuffer_start_atomic_bcs_override(batch, 0x1000, BSD_RING0);
7144     else
7145         intel_batchbuffer_start_atomic_bcs(batch, 0x1000);
7146
7147     intel_batchbuffer_emit_mi_flush(batch);
7148
7149     generic_state = (struct generic_enc_codec_state *)pak_context->generic_enc_state;
7150     priv_state = (struct gen9_hevc_encoder_state *)pak_context->private_enc_state;
7151     priv_ctx = (struct gen9_hevc_encoder_context *)pak_context->private_enc_ctx;
7152
7153     for (generic_state->curr_pak_pass = 0;
7154          generic_state->curr_pak_pass < generic_state->num_pak_passes;
7155          generic_state->curr_pak_pass++) {
7156         va_status = gen9_hevc_pak_picture_level(ctx, encode_state, encoder_context);
7157         if (va_status != VA_STATUS_SUCCESS)
7158             goto EXIT;
7159
7160         gen9_hevc_pak_read_status(ctx, encoder_context);
7161     }
7162
7163     if (priv_ctx->res_pak_slice_batch_buffer) {
7164         intel_batchbuffer_free(priv_ctx->res_pak_slice_batch_buffer);
7165
7166         priv_ctx->res_pak_slice_batch_buffer = NULL;
7167     }
7168
7169     intel_batchbuffer_end_atomic(batch);
7170
7171     intel_batchbuffer_flush(batch);
7172
7173     priv_state->frame_number++;
7174
7175 EXIT:
7176     return va_status;
7177 }
7178
7179 static void
7180 gen9_hevc_pak_context_destroy(void *context)
7181 {
7182     struct encoder_vme_mfc_context *pak_context = (struct encoder_vme_mfc_context *)context;
7183     struct gen9_hevc_encoder_context *priv_ctx = NULL;
7184     int i = 0;
7185
7186     priv_ctx = (struct gen9_hevc_encoder_context *)pak_context->private_enc_ctx;
7187
7188     if (priv_ctx->res_pak_slice_batch_buffer) {
7189         intel_batchbuffer_free(priv_ctx->res_pak_slice_batch_buffer);
7190         priv_ctx->res_pak_slice_batch_buffer = NULL;
7191     }
7192
7193     dri_bo_unreference(priv_ctx->indirect_pak_bse_object.bo);
7194     priv_ctx->indirect_pak_bse_object.bo = NULL;
7195
7196     if (priv_ctx->uncompressed_picture_source.obj_surface &&
7197         priv_ctx->uncompressed_picture_source.obj_surface->bo)
7198         i965_destroy_surface_storage(priv_ctx->uncompressed_picture_source.obj_surface);
7199
7200     if (priv_ctx->reconstructed_object.obj_surface &&
7201         priv_ctx->reconstructed_object.obj_surface->bo)
7202         i965_destroy_surface_storage(priv_ctx->reconstructed_object.obj_surface);
7203
7204     for (i = 0; i < GEN9_MAX_REF_SURFACES; i++)
7205         if (priv_ctx->reference_surfaces[i].obj_surface &&
7206             priv_ctx->reference_surfaces[i].obj_surface->bo)
7207             i965_destroy_surface_storage(priv_ctx->reference_surfaces[i].obj_surface);
7208 }
7209
7210 #define STATUS_IMPLEMENTATION_START
7211
7212 static void
7213 gen9_hevc_status_buffer_init(struct hevc_encode_status_buffer *status_buffer)
7214 {
7215     uint32_t base_offset = offsetof(struct i965_coded_buffer_segment, codec_private_data);
7216
7217     status_buffer->mmio_bs_frame_offset = MMIO_HCP_ENC_BITSTREAM_BYTECOUNT_FRAME_OFFSET;
7218     status_buffer->mmio_bs_frame_no_header_offset = MMIO_HCP_ENC_BITSTREAM_BYTECOUNT_FRAME_NO_HEADER_OFFSET;
7219     status_buffer->mmio_image_mask_offset = MMIO_HCP_ENC_IMAGE_STATUS_MASK_OFFSET;
7220     status_buffer->mmio_image_ctrl_offset = MMIO_HCP_ENC_IMAGE_STATUS_CTRL_OFFSET;
7221
7222     status_buffer->status_image_mask_offset = base_offset +
7223                                               offsetof(struct hevc_encode_status, image_status_mask);
7224     status_buffer->status_image_ctrl_offset = base_offset +
7225                                               offsetof(struct hevc_encode_status, image_status_ctrl);
7226     status_buffer->status_image_ctrl_last_pass_offset = base_offset +
7227                                                         offsetof(struct hevc_encode_status, image_status_ctrl_last_pass);
7228     status_buffer->status_bs_byte_count_offset = base_offset +
7229                                                  offsetof(struct hevc_encode_status, bs_byte_count);
7230     status_buffer->status_media_state_offset = base_offset +
7231                                                offsetof(struct hevc_encode_status, media_state);
7232     status_buffer->status_pass_num_offset = base_offset +
7233                                             offsetof(struct hevc_encode_status, pass_num);
7234 }
7235
7236 static VAStatus
7237 gen9_hevc_get_coded_status(VADriverContextP ctx,
7238                            struct intel_encoder_context *encoder_context,
7239                            struct i965_coded_buffer_segment *coded_buf_seg)
7240 {
7241     struct hevc_encode_status *encode_status;
7242     struct hevc_enc_image_status_ctrl *image_status_ctrl, *image_status_ctrl_last_pass;
7243
7244     if (!encoder_context || !coded_buf_seg)
7245         return VA_STATUS_ERROR_INVALID_BUFFER;
7246
7247     encode_status = (struct hevc_encode_status *)coded_buf_seg->codec_private_data;
7248
7249     coded_buf_seg->base.size = encode_status->bs_byte_count;
7250
7251     image_status_ctrl = (struct hevc_enc_image_status_ctrl *)&encode_status->image_status_ctrl;
7252     image_status_ctrl_last_pass = (struct hevc_enc_image_status_ctrl *)&encode_status->image_status_ctrl_last_pass;
7253
7254     if (image_status_ctrl->total_pass && image_status_ctrl->cumulative_frame_delta_qp == 0)
7255         image_status_ctrl->cumulative_frame_delta_qp = image_status_ctrl_last_pass->cumulative_frame_delta_qp;
7256
7257     image_status_ctrl_last_pass->cumulative_frame_delta_qp = 0;
7258
7259     return VA_STATUS_SUCCESS;
7260 }
7261
7262 // External initial APIs
7263
7264 Bool
7265 gen9_hevc_vme_context_init(VADriverContextP ctx,
7266                            struct intel_encoder_context *encoder_context)
7267 {
7268     struct i965_driver_data *i965 = i965_driver_data(ctx);
7269     struct encoder_vme_mfc_context *vme_context = NULL;
7270     struct gen9_hevc_encoder_context *priv_ctx = NULL;
7271     struct generic_enc_codec_state *generic_state = NULL;
7272     struct gen9_hevc_encoder_state *priv_state = NULL;
7273
7274     hevc_enc_kernel_ptr = (void *)gen9_hevc_encoder_kernels;
7275     hevc_enc_kernel_size = sizeof(gen9_hevc_encoder_kernels);
7276
7277     vme_context = calloc(1, sizeof(*vme_context));
7278     priv_ctx = calloc(1, sizeof(*priv_ctx));
7279     generic_state = calloc(1, sizeof(*generic_state));
7280     priv_state = calloc(1, sizeof(*priv_state));
7281
7282     if (!vme_context || !generic_state ||
7283         !priv_ctx || !priv_state) {
7284         if (vme_context) free(vme_context);
7285         if (generic_state) free(generic_state);
7286         if (priv_ctx) free(priv_ctx);
7287         if (priv_state) free(priv_state);
7288
7289         return false;
7290     }
7291
7292     encoder_context->vme_context = (void *)vme_context;
7293     vme_context->private_enc_ctx = (void *)priv_ctx;
7294     vme_context->generic_enc_state = (void *)generic_state;
7295     vme_context->private_enc_state = (void *)priv_state;
7296
7297     priv_ctx->ctx = ctx;
7298     priv_ctx->mocs = i965->intel.mocs_state;
7299
7300     generic_state->num_pak_passes = 1;
7301     generic_state->brc_enabled = 0;
7302
7303     priv_state->tu_mode = HEVC_TU_RT_SPEED;
7304     priv_state->use_hw_scoreboard = 1;
7305     priv_state->use_hw_non_stalling_scoreborad = 1;
7306     priv_state->rolling_intra_refresh = 0;
7307     priv_state->flatness_check_supported = 0;
7308     priv_state->walking_pattern_26 = 0;
7309     priv_state->num_regions_in_slice = 4;
7310     priv_state->frames_per_100s = 30000;
7311     priv_state->user_max_frame_size = 0;
7312     priv_state->brc_method = HEVC_BRC_CQP;
7313     priv_state->lcu_brc_enabled = 0;
7314     priv_state->parallel_brc = 0;
7315     priv_state->pak_obj_size = ((IS_KBL(i965->intel.device_info) || IS_GLK(i965->intel.device_info)) ?
7316                                 GEN95_HEVC_ENC_PAK_OBJ_SIZE :
7317                                 GEN9_HEVC_ENC_PAK_OBJ_SIZE) *
7318                                4;
7319     priv_state->cu_record_size = GEN9_HEVC_ENC_PAK_CU_RECORD_SIZE * 4;
7320     priv_state->pic_state_size = GEN9_HEVC_ENC_BRC_PIC_STATE_SIZE;
7321
7322     gen9_hevc_status_buffer_init(&priv_state->status_buffer);
7323     gen9_hevc_vme_kernels_context_init(ctx, encoder_context);
7324     gen9_hevc_lambda_tables_init(priv_ctx);
7325
7326     encoder_context->vme_pipeline = gen9_hevc_vme_pipeline;
7327     encoder_context->vme_context_destroy = gen9_hevc_vme_context_destroy;
7328     return true;
7329 }
7330
7331 Bool
7332 gen9_hevc_pak_context_init(VADriverContextP ctx,
7333                            struct intel_encoder_context *encoder_context)
7334 {
7335     struct encoder_vme_mfc_context *pak_context = encoder_context->vme_context;
7336
7337     if (!pak_context)
7338         return false;
7339
7340     encoder_context->mfc_context = pak_context;
7341     encoder_context->mfc_context_destroy = gen9_hevc_pak_context_destroy;
7342     encoder_context->mfc_pipeline = gen9_hevc_pak_pipeline;
7343     encoder_context->mfc_brc_prepare = gen9_hevc_brc_prepare;
7344     encoder_context->get_status = gen9_hevc_get_coded_status;
7345     return true;
7346 }