2 * Copyright 2006 VMware, Inc.
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the
7 * "Software"), to deal in the Software without restriction, including
8 * without limitation the rights to use, copy, modify, merge, publish,
9 * distribute, sublicense, and/or sell copies of the Software, and to
10 * permit persons to whom the Software is furnished to do so, subject to
11 * the following conditions:
13 * The above copyright notice and this permission notice (including the
14 * next paragraph) shall be included in all copies or substantial portions
17 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
18 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
19 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
20 * IN NO EVENT SHALL VMWARE AND/OR ITS SUPPLIERS BE LIABLE FOR
21 * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
22 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
23 * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
26 #include "intel_batchbuffer.h"
27 #include "intel_buffer_objects.h"
28 #include "brw_bufmgr.h"
29 #include "intel_buffers.h"
30 #include "intel_fbo.h"
31 #include "brw_context.h"
32 #include "brw_defines.h"
33 #include "brw_state.h"
34 #include "common/gen_decoder.h"
36 #include "util/hash_table.h"
41 #define FILE_DEBUG_FLAG DEBUG_BUFMGR
44 * Target sizes of the batch and state buffers. We create the initial
45 * buffers at these sizes, and flush when they're nearly full. If we
46 * underestimate how close we are to the end, and suddenly need more space
47 * in the middle of a draw, we can grow the buffers, and finish the draw.
48 * At that point, we'll be over our target size, so the next operation
49 * should flush. Each time we flush the batch, we recreate both buffers
50 * at the original target size, so it doesn't grow without bound.
52 #define BATCH_SZ (20 * 1024)
53 #define STATE_SZ (16 * 1024)
56 intel_batchbuffer_reset(struct brw_context *brw);
59 dump_validation_list(struct intel_batchbuffer *batch)
61 fprintf(stderr, "Validation list (length %d):\n", batch->exec_count);
63 for (int i = 0; i < batch->exec_count; i++) {
64 uint64_t flags = batch->validation_list[i].flags;
65 assert(batch->validation_list[i].handle ==
66 batch->exec_bos[i]->gem_handle);
67 fprintf(stderr, "[%2d]: %2d %-14s %p %s%-7s @ 0x%016llx%s (%"PRIu64"B)\n",
69 batch->validation_list[i].handle,
70 batch->exec_bos[i]->name,
72 (flags & EXEC_OBJECT_SUPPORTS_48B_ADDRESS) ? "(48b" : "(32b",
73 (flags & EXEC_OBJECT_WRITE) ? " write)" : ")",
74 batch->validation_list[i].offset,
75 (flags & EXEC_OBJECT_PINNED) ? " (pinned)" : "",
76 batch->exec_bos[i]->size);
81 uint_key_compare(const void *a, const void *b)
87 uint_key_hash(const void *key)
89 return (uintptr_t) key;
93 init_reloc_list(struct brw_reloc_list *rlist, int count)
95 rlist->reloc_count = 0;
96 rlist->reloc_array_size = count;
97 rlist->relocs = malloc(rlist->reloc_array_size *
98 sizeof(struct drm_i915_gem_relocation_entry));
102 intel_batchbuffer_init(struct brw_context *brw)
104 struct intel_screen *screen = brw->screen;
105 struct intel_batchbuffer *batch = &brw->batch;
106 const struct gen_device_info *devinfo = &screen->devinfo;
108 batch->use_shadow_copy = !devinfo->has_llc;
110 init_reloc_list(&batch->batch_relocs, 250);
111 init_reloc_list(&batch->state_relocs, 250);
113 batch->batch.map = NULL;
114 batch->state.map = NULL;
115 batch->exec_count = 0;
116 batch->exec_array_size = 100;
118 malloc(batch->exec_array_size * sizeof(batch->exec_bos[0]));
119 batch->validation_list =
120 malloc(batch->exec_array_size * sizeof(batch->validation_list[0]));
122 if (INTEL_DEBUG & DEBUG_BATCH) {
123 batch->state_batch_sizes =
124 _mesa_hash_table_create(NULL, uint_key_hash, uint_key_compare);
127 batch->use_batch_first =
128 screen->kernel_features & KERNEL_ALLOWS_EXEC_BATCH_FIRST;
130 /* PIPE_CONTROL needs a w/a but only on gen6 */
131 batch->valid_reloc_flags = EXEC_OBJECT_WRITE;
132 if (devinfo->gen == 6)
133 batch->valid_reloc_flags |= EXEC_OBJECT_NEEDS_GTT;
135 intel_batchbuffer_reset(brw);
138 #define READ_ONCE(x) (*(volatile __typeof__(x) *)&(x))
141 add_exec_bo(struct intel_batchbuffer *batch, struct brw_bo *bo)
143 unsigned index = READ_ONCE(bo->index);
145 if (index < batch->exec_count && batch->exec_bos[index] == bo)
148 /* May have been shared between multiple active batches */
149 for (index = 0; index < batch->exec_count; index++) {
150 if (batch->exec_bos[index] == bo)
154 brw_bo_reference(bo);
156 if (batch->exec_count == batch->exec_array_size) {
157 batch->exec_array_size *= 2;
159 realloc(batch->exec_bos,
160 batch->exec_array_size * sizeof(batch->exec_bos[0]));
161 batch->validation_list =
162 realloc(batch->validation_list,
163 batch->exec_array_size * sizeof(batch->validation_list[0]));
166 batch->validation_list[batch->exec_count] =
167 (struct drm_i915_gem_exec_object2) {
168 .handle = bo->gem_handle,
169 .offset = bo->gtt_offset,
173 bo->index = batch->exec_count;
174 batch->exec_bos[batch->exec_count] = bo;
175 batch->aperture_space += bo->size;
177 return batch->exec_count++;
181 recreate_growing_buffer(struct brw_context *brw,
182 struct brw_growing_bo *grow,
183 const char *name, unsigned size)
185 struct intel_screen *screen = brw->screen;
186 struct intel_batchbuffer *batch = &brw->batch;
187 struct brw_bufmgr *bufmgr = screen->bufmgr;
189 grow->bo = brw_bo_alloc(bufmgr, name, size);
190 grow->bo->kflags |= can_do_exec_capture(screen) ? EXEC_OBJECT_CAPTURE : 0;
191 grow->partial_bo = NULL;
192 grow->partial_bo_map = NULL;
193 grow->partial_bytes = 0;
195 if (batch->use_shadow_copy)
196 grow->map = realloc(grow->map, grow->bo->size);
198 grow->map = brw_bo_map(brw, grow->bo, MAP_READ | MAP_WRITE);
202 intel_batchbuffer_reset(struct brw_context *brw)
204 struct intel_batchbuffer *batch = &brw->batch;
206 if (batch->last_bo != NULL) {
207 brw_bo_unreference(batch->last_bo);
208 batch->last_bo = NULL;
210 batch->last_bo = batch->batch.bo;
212 recreate_growing_buffer(brw, &batch->batch, "batchbuffer", BATCH_SZ);
213 batch->map_next = batch->batch.map;
215 recreate_growing_buffer(brw, &batch->state, "statebuffer", STATE_SZ);
217 /* Avoid making 0 a valid state offset - otherwise the decoder will try
218 * and decode data when we use offset 0 as a null pointer.
220 batch->state_used = 1;
222 add_exec_bo(batch, batch->batch.bo);
223 assert(batch->batch.bo->index == 0);
225 batch->needs_sol_reset = false;
226 batch->state_base_address_emitted = false;
228 /* We don't know what ring the new batch will be sent to until we see the
229 * first BEGIN_BATCH or BEGIN_BATCH_BLT. Mark it as unknown.
231 batch->ring = UNKNOWN_RING;
233 if (batch->state_batch_sizes)
234 _mesa_hash_table_clear(batch->state_batch_sizes, NULL);
238 intel_batchbuffer_reset_and_clear_render_cache(struct brw_context *brw)
240 intel_batchbuffer_reset(brw);
241 brw_cache_sets_clear(brw);
245 intel_batchbuffer_save_state(struct brw_context *brw)
247 brw->batch.saved.map_next = brw->batch.map_next;
248 brw->batch.saved.batch_reloc_count = brw->batch.batch_relocs.reloc_count;
249 brw->batch.saved.state_reloc_count = brw->batch.state_relocs.reloc_count;
250 brw->batch.saved.exec_count = brw->batch.exec_count;
254 intel_batchbuffer_reset_to_saved(struct brw_context *brw)
256 for (int i = brw->batch.saved.exec_count;
257 i < brw->batch.exec_count; i++) {
258 brw_bo_unreference(brw->batch.exec_bos[i]);
260 brw->batch.batch_relocs.reloc_count = brw->batch.saved.batch_reloc_count;
261 brw->batch.state_relocs.reloc_count = brw->batch.saved.state_reloc_count;
262 brw->batch.exec_count = brw->batch.saved.exec_count;
264 brw->batch.map_next = brw->batch.saved.map_next;
265 if (USED_BATCH(brw->batch) == 0)
266 brw->batch.ring = UNKNOWN_RING;
270 intel_batchbuffer_free(struct intel_batchbuffer *batch)
272 if (batch->use_shadow_copy) {
273 free(batch->batch.map);
274 free(batch->state.map);
277 for (int i = 0; i < batch->exec_count; i++) {
278 brw_bo_unreference(batch->exec_bos[i]);
280 free(batch->batch_relocs.relocs);
281 free(batch->state_relocs.relocs);
282 free(batch->exec_bos);
283 free(batch->validation_list);
285 brw_bo_unreference(batch->last_bo);
286 brw_bo_unreference(batch->batch.bo);
287 brw_bo_unreference(batch->state.bo);
288 if (batch->state_batch_sizes)
289 _mesa_hash_table_destroy(batch->state_batch_sizes, NULL);
293 * Finish copying the old batch/state buffer's contents to the new one
294 * after we tried to "grow" the buffer in an earlier operation.
297 finish_growing_bos(struct brw_growing_bo *grow)
299 struct brw_bo *old_bo = grow->partial_bo;
303 memcpy(grow->map, grow->partial_bo_map, grow->partial_bytes);
305 grow->partial_bo = NULL;
306 grow->partial_bo_map = NULL;
307 grow->partial_bytes = 0;
309 brw_bo_unreference(old_bo);
313 replace_bo_in_reloc_list(struct brw_reloc_list *rlist,
314 uint32_t old_handle, uint32_t new_handle)
316 for (int i = 0; i < rlist->reloc_count; i++) {
317 if (rlist->relocs[i].target_handle == old_handle)
318 rlist->relocs[i].target_handle = new_handle;
323 * Grow either the batch or state buffer to a new larger size.
325 * We can't actually grow buffers, so we allocate a new one, copy over
326 * the existing contents, and update our lists to refer to the new one.
328 * Note that this is only temporary - each new batch recreates the buffers
329 * at their original target size (BATCH_SZ or STATE_SZ).
332 grow_buffer(struct brw_context *brw,
333 struct brw_growing_bo *grow,
334 unsigned existing_bytes,
337 struct intel_batchbuffer *batch = &brw->batch;
338 struct brw_bufmgr *bufmgr = brw->bufmgr;
339 struct brw_bo *bo = grow->bo;
341 perf_debug("Growing %s - ran out of space\n", bo->name);
343 if (grow->partial_bo) {
344 /* We've already grown once, and now we need to do it again.
345 * Finish our last grow operation so we can start a new one.
346 * This should basically never happen.
348 perf_debug("Had to grow multiple times");
349 finish_growing_bos(grow);
352 struct brw_bo *new_bo = brw_bo_alloc(bufmgr, bo->name, new_size);
354 /* Copy existing data to the new larger buffer */
355 grow->partial_bo_map = grow->map;
357 if (batch->use_shadow_copy) {
358 /* We can't safely use realloc, as it may move the existing buffer,
359 * breaking existing pointers the caller may still be using. Just
360 * malloc a new copy and memcpy it like the normal BO path.
362 * Use bo->size rather than new_size because the bufmgr may have
363 * rounded up the size, and we want the shadow size to match.
365 grow->map = malloc(new_bo->size);
367 grow->map = brw_bo_map(brw, new_bo, MAP_READ | MAP_WRITE);
370 /* Try to put the new BO at the same GTT offset as the old BO (which
371 * we're throwing away, so it doesn't need to be there).
373 * This guarantees that our relocations continue to work: values we've
374 * already written into the buffer, values we're going to write into the
375 * buffer, and the validation/relocation lists all will match.
377 * Also preserve kflags for EXEC_OBJECT_CAPTURE.
379 new_bo->gtt_offset = bo->gtt_offset;
380 new_bo->index = bo->index;
381 new_bo->kflags = bo->kflags;
383 /* Batch/state buffers are per-context, and if we've run out of space,
384 * we must have actually used them before, so...they will be in the list.
386 assert(bo->index < batch->exec_count);
387 assert(batch->exec_bos[bo->index] == bo);
389 /* Update the validation list to use the new BO. */
390 batch->validation_list[bo->index].handle = new_bo->gem_handle;
392 if (!batch->use_batch_first) {
393 /* We're not using I915_EXEC_HANDLE_LUT, which means we need to go
394 * update the relocation list entries to point at the new BO as well.
395 * (With newer kernels, the "handle" is an offset into the validation
396 * list, which remains unchanged, so we can skip this.)
398 replace_bo_in_reloc_list(&batch->batch_relocs,
399 bo->gem_handle, new_bo->gem_handle);
400 replace_bo_in_reloc_list(&batch->state_relocs,
401 bo->gem_handle, new_bo->gem_handle);
404 /* Exchange the two BOs...without breaking pointers to the old BO.
406 * Consider this scenario:
408 * 1. Somebody calls brw_state_batch() to get a region of memory, and
409 * and then creates a brw_address pointing to brw->batch.state.bo.
410 * 2. They then call brw_state_batch() a second time, which happens to
411 * grow and replace the state buffer. They then try to emit a
412 * relocation to their first section of memory.
414 * If we replace the brw->batch.state.bo pointer at step 2, we would
415 * break the address created in step 1. They'd have a pointer to the
416 * old destroyed BO. Emitting a relocation would add this dead BO to
417 * the validation list...causing /both/ statebuffers to be in the list,
418 * and all kinds of disasters.
420 * This is not a contrived case - BLORP vertex data upload hits this.
422 * There are worse scenarios too. Fences for GL sync objects reference
423 * brw->batch.batch.bo. If we replaced the batch pointer when growing,
424 * we'd need to chase down every fence and update it to point to the
425 * new BO. Otherwise, it would refer to a "batch" that never actually
426 * gets submitted, and would fail to trigger.
428 * To work around both of these issues, we transmutate the buffers in
429 * place, making the existing struct brw_bo represent the new buffer,
430 * and "new_bo" represent the old BO. This is highly unusual, but it
431 * seems like a necessary evil.
433 * We also defer the memcpy of the existing batch's contents. Callers
434 * may make multiple brw_state_batch calls, and retain pointers to the
435 * old BO's map. We'll perform the memcpy in finish_growing_bo() when
436 * we finally submit the batch, at which point we've finished uploading
437 * state, and nobody should have any old references anymore.
439 * To do that, we keep a reference to the old BO in grow->partial_bo,
440 * and store the number of bytes to copy in grow->partial_bytes. We
441 * can monkey with the refcounts directly without atomics because these
442 * are per-context BOs and they can only be touched by this thread.
444 assert(new_bo->refcount == 1);
445 new_bo->refcount = bo->refcount;
449 memcpy(&tmp, bo, sizeof(struct brw_bo));
450 memcpy(bo, new_bo, sizeof(struct brw_bo));
451 memcpy(new_bo, &tmp, sizeof(struct brw_bo));
453 grow->partial_bo = new_bo; /* the one reference of the OLD bo */
454 grow->partial_bytes = existing_bytes;
458 intel_batchbuffer_require_space(struct brw_context *brw, GLuint sz,
459 enum brw_gpu_ring ring)
461 const struct gen_device_info *devinfo = &brw->screen->devinfo;
462 struct intel_batchbuffer *batch = &brw->batch;
464 /* If we're switching rings, implicitly flush the batch. */
465 if (unlikely(ring != brw->batch.ring) && brw->batch.ring != UNKNOWN_RING &&
467 intel_batchbuffer_flush(brw);
470 const unsigned batch_used = USED_BATCH(*batch) * 4;
471 if (batch_used + sz >= BATCH_SZ && !batch->no_wrap) {
472 intel_batchbuffer_flush(brw);
473 } else if (batch_used + sz >= batch->batch.bo->size) {
474 const unsigned new_size =
475 MIN2(batch->batch.bo->size + batch->batch.bo->size / 2,
477 grow_buffer(brw, &batch->batch, batch_used, new_size);
478 batch->map_next = (void *) batch->batch.map + batch_used;
479 assert(batch_used + sz < batch->batch.bo->size);
482 /* The intel_batchbuffer_flush() calls above might have changed
483 * brw->batch.ring to UNKNOWN_RING, so we need to set it here at the end.
485 brw->batch.ring = ring;
490 #define BLUE_HEADER CSI "0;44m"
491 #define NORMAL CSI "0m"
495 decode_struct(struct brw_context *brw, struct gen_spec *spec,
496 const char *struct_name, uint32_t *data,
497 uint32_t gtt_offset, uint32_t offset, bool color)
499 struct gen_group *group = gen_spec_find_struct(spec, struct_name);
503 fprintf(stderr, "%s\n", struct_name);
504 gen_print_group(stderr, group, gtt_offset + offset,
505 &data[offset / 4], 0, color);
509 decode_structs(struct brw_context *brw, struct gen_spec *spec,
510 const char *struct_name,
511 uint32_t *data, uint32_t gtt_offset, uint32_t offset,
512 int struct_size, bool color)
514 struct gen_group *group = gen_spec_find_struct(spec, struct_name);
518 int entries = brw_state_batch_size(brw, offset) / struct_size;
519 for (int i = 0; i < entries; i++) {
520 fprintf(stderr, "%s %d\n", struct_name, i);
521 gen_print_group(stderr, group, gtt_offset + offset,
522 &data[(offset + i * struct_size) / 4], 0, color);
527 do_batch_dump(struct brw_context *brw)
529 const struct gen_device_info *devinfo = &brw->screen->devinfo;
530 struct intel_batchbuffer *batch = &brw->batch;
531 struct gen_spec *spec = gen_spec_load(&brw->screen->devinfo);
533 if (batch->ring != RENDER_RING)
536 uint32_t *batch_data = brw_bo_map(brw, batch->batch.bo, MAP_READ);
537 uint32_t *state = brw_bo_map(brw, batch->state.bo, MAP_READ);
538 if (batch_data == NULL || state == NULL) {
539 fprintf(stderr, "WARNING: failed to map batchbuffer/statebuffer\n");
543 uint32_t *end = batch_data + USED_BATCH(*batch);
544 uint32_t batch_gtt_offset = batch->batch.bo->gtt_offset;
545 uint32_t state_gtt_offset = batch->state.bo->gtt_offset;
548 bool color = INTEL_DEBUG & DEBUG_COLOR;
549 const char *header_color = color ? BLUE_HEADER : "";
550 const char *reset_color = color ? NORMAL : "";
552 for (uint32_t *p = batch_data; p < end; p += length) {
553 struct gen_group *inst = gen_spec_find_instruction(spec, p);
554 length = gen_group_get_length(inst, p);
555 assert(inst == NULL || length > 0);
556 length = MAX2(1, length);
558 fprintf(stderr, "unknown instruction %08x\n", p[0]);
562 uint64_t offset = batch_gtt_offset + 4 * (p - batch_data);
564 fprintf(stderr, "%s0x%08"PRIx64": 0x%08x: %-80s%s\n", header_color,
565 offset, p[0], gen_group_get_name(inst), reset_color);
567 gen_print_group(stderr, inst, offset, p, 0, color);
569 switch (gen_group_get_opcode(inst) >> 16) {
570 case _3DSTATE_PIPELINED_POINTERS:
571 /* Note: these Gen4-5 pointers are full relocations rather than
572 * offsets from the start of the statebuffer. So we need to subtract
573 * gtt_offset (the start of the statebuffer) to obtain an offset we
574 * can add to the map and get at the data.
576 decode_struct(brw, spec, "VS_STATE", state, state_gtt_offset,
577 (p[1] & ~0x1fu) - state_gtt_offset, color);
579 decode_struct(brw, spec, "GS_STATE", state, state_gtt_offset,
580 (p[2] & ~0x1fu) - state_gtt_offset, color);
583 decode_struct(brw, spec, "CLIP_STATE", state, state_gtt_offset,
584 (p[3] & ~0x1fu) - state_gtt_offset, color);
586 decode_struct(brw, spec, "SF_STATE", state, state_gtt_offset,
587 (p[4] & ~0x1fu) - state_gtt_offset, color);
588 decode_struct(brw, spec, "WM_STATE", state, state_gtt_offset,
589 (p[5] & ~0x1fu) - state_gtt_offset, color);
590 decode_struct(brw, spec, "COLOR_CALC_STATE", state, state_gtt_offset,
591 (p[6] & ~0x3fu) - state_gtt_offset, color);
593 case _3DSTATE_BINDING_TABLE_POINTERS_VS:
594 case _3DSTATE_BINDING_TABLE_POINTERS_HS:
595 case _3DSTATE_BINDING_TABLE_POINTERS_DS:
596 case _3DSTATE_BINDING_TABLE_POINTERS_GS:
597 case _3DSTATE_BINDING_TABLE_POINTERS_PS: {
598 struct gen_group *group =
599 gen_spec_find_struct(spec, "RENDER_SURFACE_STATE");
603 uint32_t bt_offset = p[1] & ~0x1fu;
604 int bt_entries = brw_state_batch_size(brw, bt_offset) / 4;
605 uint32_t *bt_pointers = &state[bt_offset / 4];
606 for (int i = 0; i < bt_entries; i++) {
607 fprintf(stderr, "SURFACE_STATE - BTI = %d\n", i);
608 gen_print_group(stderr, group, state_gtt_offset + bt_pointers[i],
609 &state[bt_pointers[i] / 4], 0, color);
613 case _3DSTATE_SAMPLER_STATE_POINTERS_VS:
614 case _3DSTATE_SAMPLER_STATE_POINTERS_HS:
615 case _3DSTATE_SAMPLER_STATE_POINTERS_DS:
616 case _3DSTATE_SAMPLER_STATE_POINTERS_GS:
617 case _3DSTATE_SAMPLER_STATE_POINTERS_PS:
618 decode_structs(brw, spec, "SAMPLER_STATE", state,
619 state_gtt_offset, p[1] & ~0x1fu, 4 * 4, color);
621 case _3DSTATE_VIEWPORT_STATE_POINTERS:
622 decode_structs(brw, spec, "CLIP_VIEWPORT", state,
623 state_gtt_offset, p[1] & ~0x3fu, 4 * 4, color);
624 decode_structs(brw, spec, "SF_VIEWPORT", state,
625 state_gtt_offset, p[1] & ~0x3fu, 8 * 4, color);
626 decode_structs(brw, spec, "CC_VIEWPORT", state,
627 state_gtt_offset, p[3] & ~0x3fu, 2 * 4, color);
629 case _3DSTATE_VIEWPORT_STATE_POINTERS_CC:
630 decode_structs(brw, spec, "CC_VIEWPORT", state,
631 state_gtt_offset, p[1] & ~0x3fu, 2 * 4, color);
633 case _3DSTATE_VIEWPORT_STATE_POINTERS_SF_CL:
634 decode_structs(brw, spec, "SF_CLIP_VIEWPORT", state,
635 state_gtt_offset, p[1] & ~0x3fu, 16 * 4, color);
637 case _3DSTATE_SCISSOR_STATE_POINTERS:
638 decode_structs(brw, spec, "SCISSOR_RECT", state,
639 state_gtt_offset, p[1] & ~0x1fu, 2 * 4, color);
641 case _3DSTATE_BLEND_STATE_POINTERS:
642 /* TODO: handle Gen8+ extra dword at the beginning */
643 decode_structs(brw, spec, "BLEND_STATE", state,
644 state_gtt_offset, p[1] & ~0x3fu, 8 * 4, color);
646 case _3DSTATE_CC_STATE_POINTERS:
647 if (devinfo->gen >= 7) {
648 decode_struct(brw, spec, "COLOR_CALC_STATE", state,
649 state_gtt_offset, p[1] & ~0x3fu, color);
650 } else if (devinfo->gen == 6) {
651 decode_structs(brw, spec, "BLEND_STATE", state,
652 state_gtt_offset, p[1] & ~0x3fu, 2 * 4, color);
653 decode_struct(brw, spec, "DEPTH_STENCIL_STATE", state,
654 state_gtt_offset, p[2] & ~0x3fu, color);
655 decode_struct(brw, spec, "COLOR_CALC_STATE", state,
656 state_gtt_offset, p[3] & ~0x3fu, color);
659 case _3DSTATE_DEPTH_STENCIL_STATE_POINTERS:
660 decode_struct(brw, spec, "DEPTH_STENCIL_STATE", state,
661 state_gtt_offset, p[1] & ~0x3fu, color);
663 case MEDIA_INTERFACE_DESCRIPTOR_LOAD: {
664 struct gen_group *group =
665 gen_spec_find_struct(spec, "RENDER_SURFACE_STATE");
669 uint32_t idd_offset = p[3] & ~0x1fu;
670 decode_struct(brw, spec, "INTERFACE_DESCRIPTOR_DATA", state,
671 state_gtt_offset, idd_offset, color);
673 uint32_t ss_offset = state[idd_offset / 4 + 3] & ~0x1fu;
674 decode_structs(brw, spec, "SAMPLER_STATE", state,
675 state_gtt_offset, ss_offset, 4 * 4, color);
677 uint32_t bt_offset = state[idd_offset / 4 + 4] & ~0x1fu;
678 int bt_entries = brw_state_batch_size(brw, bt_offset) / 4;
679 uint32_t *bt_pointers = &state[bt_offset / 4];
680 for (int i = 0; i < bt_entries; i++) {
681 fprintf(stderr, "SURFACE_STATE - BTI = %d\n", i);
682 gen_print_group(stderr, group, state_gtt_offset + bt_pointers[i],
683 &state[bt_pointers[i] / 4], 0, color);
690 brw_bo_unmap(batch->batch.bo);
691 brw_bo_unmap(batch->state.bo);
694 static void do_batch_dump(struct brw_context *brw) { }
698 * Called when starting a new batch buffer.
701 brw_new_batch(struct brw_context *brw)
703 /* Unreference any BOs held by the previous batch, and reset counts. */
704 for (int i = 0; i < brw->batch.exec_count; i++) {
705 brw_bo_unreference(brw->batch.exec_bos[i]);
706 brw->batch.exec_bos[i] = NULL;
708 brw->batch.batch_relocs.reloc_count = 0;
709 brw->batch.state_relocs.reloc_count = 0;
710 brw->batch.exec_count = 0;
711 brw->batch.aperture_space = 0;
713 brw_bo_unreference(brw->batch.state.bo);
715 /* Create a new batchbuffer and reset the associated state: */
716 intel_batchbuffer_reset_and_clear_render_cache(brw);
718 /* If the kernel supports hardware contexts, then most hardware state is
719 * preserved between batches; we only need to re-emit state that is required
720 * to be in every batch. Otherwise we need to re-emit all the state that
721 * would otherwise be stored in the context (which for all intents and
722 * purposes means everything).
724 if (brw->hw_ctx == 0) {
725 brw->ctx.NewDriverState |= BRW_NEW_CONTEXT;
726 brw_upload_invariant_state(brw);
729 brw->ctx.NewDriverState |= BRW_NEW_BATCH;
731 brw->ib.index_size = -1;
733 /* We need to periodically reap the shader time results, because rollover
734 * happens every few seconds. We also want to see results every once in a
735 * while, because many programs won't cleanly destroy our context, so the
736 * end-of-run printout may not happen.
738 if (INTEL_DEBUG & DEBUG_SHADER_TIME)
739 brw_collect_and_report_shader_time(brw);
743 * Called from intel_batchbuffer_flush before emitting MI_BATCHBUFFER_END and
746 * This function can emit state (say, to preserve registers that aren't saved
750 brw_finish_batch(struct brw_context *brw)
752 const struct gen_device_info *devinfo = &brw->screen->devinfo;
754 brw->batch.no_wrap = true;
756 /* Capture the closing pipeline statistics register values necessary to
757 * support query objects (in the non-hardware context world).
759 brw_emit_query_end(brw);
761 if (brw->batch.ring == RENDER_RING) {
762 /* Work around L3 state leaks into contexts set MI_RESTORE_INHIBIT which
763 * assume that the L3 cache is configured according to the hardware
764 * defaults. On Kernel 4.16+, we no longer need to do this.
766 if (devinfo->gen >= 7 &&
767 !(brw->screen->kernel_features & KERNEL_ALLOWS_CONTEXT_ISOLATION))
768 gen7_restore_default_l3_config(brw);
770 if (devinfo->is_haswell) {
771 /* From the Haswell PRM, Volume 2b, Command Reference: Instructions,
772 * 3DSTATE_CC_STATE_POINTERS > "Note":
774 * "SW must program 3DSTATE_CC_STATE_POINTERS command at the end of every
775 * 3D batch buffer followed by a PIPE_CONTROL with RC flush and CS stall."
777 * From the example in the docs, it seems to expect a regular pipe control
778 * flush here as well. We may have done it already, but meh.
780 * See also WaAvoidRCZCounterRollover.
782 brw_emit_mi_flush(brw);
784 OUT_BATCH(_3DSTATE_CC_STATE_POINTERS << 16 | (2 - 2));
785 OUT_BATCH(brw->cc.state_offset | 1);
787 brw_emit_pipe_control_flush(brw, PIPE_CONTROL_RENDER_TARGET_FLUSH |
788 PIPE_CONTROL_CS_STALL);
791 /* Do not restore push constant packets during context restore. */
792 if (devinfo->gen >= 7)
793 gen10_emit_isp_disable(brw);
796 /* Emit MI_BATCH_BUFFER_END to finish our batch. Note that execbuf2
797 * requires our batch size to be QWord aligned, so we pad it out if
798 * necessary by emitting an extra MI_NOOP after the end.
800 intel_batchbuffer_require_space(brw, 8, brw->batch.ring);
801 *brw->batch.map_next++ = MI_BATCH_BUFFER_END;
802 if (USED_BATCH(brw->batch) & 1) {
803 *brw->batch.map_next++ = MI_NOOP;
806 brw->batch.no_wrap = false;
810 throttle(struct brw_context *brw)
812 /* Wait for the swapbuffers before the one we just emitted, so we
813 * don't get too many swaps outstanding for apps that are GPU-heavy
816 * We're using intelDRI2Flush (called from the loader before
817 * swapbuffer) and glFlush (for front buffer rendering) as the
818 * indicator that a frame is done and then throttle when we get
819 * here as we prepare to render the next frame. At this point for
820 * round trips for swap/copy and getting new buffers are done and
821 * we'll spend less time waiting on the GPU.
823 * Unfortunately, we don't have a handle to the batch containing
824 * the swap, and getting our hands on that doesn't seem worth it,
825 * so we just use the first batch we emitted after the last swap.
827 if (brw->need_swap_throttle && brw->throttle_batch[0]) {
828 if (brw->throttle_batch[1]) {
829 if (!brw->disable_throttling) {
830 brw_bo_wait_rendering(brw->throttle_batch[1]);
832 brw_bo_unreference(brw->throttle_batch[1]);
834 brw->throttle_batch[1] = brw->throttle_batch[0];
835 brw->throttle_batch[0] = NULL;
836 brw->need_swap_throttle = false;
837 /* Throttling here is more precise than the throttle ioctl, so skip it */
838 brw->need_flush_throttle = false;
841 if (brw->need_flush_throttle) {
842 __DRIscreen *dri_screen = brw->screen->driScrnPriv;
843 drmCommandNone(dri_screen->fd, DRM_I915_GEM_THROTTLE);
844 brw->need_flush_throttle = false;
850 struct intel_batchbuffer *batch,
857 struct drm_i915_gem_execbuffer2 execbuf = {
858 .buffers_ptr = (uintptr_t) batch->validation_list,
859 .buffer_count = batch->exec_count,
860 .batch_start_offset = 0,
863 .rsvd1 = ctx_id, /* rsvd1 is actually the context ID */
866 unsigned long cmd = DRM_IOCTL_I915_GEM_EXECBUFFER2;
868 if (in_fence != -1) {
869 execbuf.rsvd2 = in_fence;
870 execbuf.flags |= I915_EXEC_FENCE_IN;
873 if (out_fence != NULL) {
874 cmd = DRM_IOCTL_I915_GEM_EXECBUFFER2_WR;
876 execbuf.flags |= I915_EXEC_FENCE_OUT;
879 int ret = drmIoctl(fd, cmd, &execbuf);
883 for (int i = 0; i < batch->exec_count; i++) {
884 struct brw_bo *bo = batch->exec_bos[i];
889 /* Update brw_bo::gtt_offset */
890 if (batch->validation_list[i].offset != bo->gtt_offset) {
891 DBG("BO %d migrated: 0x%" PRIx64 " -> 0x%llx\n",
892 bo->gem_handle, bo->gtt_offset,
893 batch->validation_list[i].offset);
894 bo->gtt_offset = batch->validation_list[i].offset;
898 if (ret == 0 && out_fence != NULL)
899 *out_fence = execbuf.rsvd2 >> 32;
905 submit_batch(struct brw_context *brw, int in_fence_fd, int *out_fence_fd)
907 const struct gen_device_info *devinfo = &brw->screen->devinfo;
908 __DRIscreen *dri_screen = brw->screen->driScrnPriv;
909 struct intel_batchbuffer *batch = &brw->batch;
912 if (batch->use_shadow_copy) {
913 void *bo_map = brw_bo_map(brw, batch->batch.bo, MAP_WRITE);
914 memcpy(bo_map, batch->batch.map, 4 * USED_BATCH(*batch));
916 bo_map = brw_bo_map(brw, batch->state.bo, MAP_WRITE);
917 memcpy(bo_map, batch->state.map, batch->state_used);
920 brw_bo_unmap(batch->batch.bo);
921 brw_bo_unmap(batch->state.bo);
923 if (!brw->screen->no_hw) {
924 /* The requirement for using I915_EXEC_NO_RELOC are:
926 * The addresses written in the objects must match the corresponding
927 * reloc.gtt_offset which in turn must match the corresponding
930 * Any render targets written to in the batch must be flagged with
933 * To avoid stalling, execobject.offset should match the current
934 * address of that object within the active context.
936 int flags = I915_EXEC_NO_RELOC;
938 if (devinfo->gen >= 6 && batch->ring == BLT_RING) {
939 flags |= I915_EXEC_BLT;
941 flags |= I915_EXEC_RENDER;
943 if (batch->needs_sol_reset)
944 flags |= I915_EXEC_GEN7_SOL_RESET;
946 uint32_t hw_ctx = batch->ring == RENDER_RING ? brw->hw_ctx : 0;
948 /* Set statebuffer relocations */
949 const unsigned state_index = batch->state.bo->index;
950 if (state_index < batch->exec_count &&
951 batch->exec_bos[state_index] == batch->state.bo) {
952 struct drm_i915_gem_exec_object2 *entry =
953 &batch->validation_list[state_index];
954 assert(entry->handle == batch->state.bo->gem_handle);
955 entry->relocation_count = batch->state_relocs.reloc_count;
956 entry->relocs_ptr = (uintptr_t) batch->state_relocs.relocs;
959 /* Set batchbuffer relocations */
960 struct drm_i915_gem_exec_object2 *entry = &batch->validation_list[0];
961 assert(entry->handle == batch->batch.bo->gem_handle);
962 entry->relocation_count = batch->batch_relocs.reloc_count;
963 entry->relocs_ptr = (uintptr_t) batch->batch_relocs.relocs;
965 if (batch->use_batch_first) {
966 flags |= I915_EXEC_BATCH_FIRST | I915_EXEC_HANDLE_LUT;
968 /* Move the batch to the end of the validation list */
969 struct drm_i915_gem_exec_object2 tmp;
970 const unsigned index = batch->exec_count - 1;
973 *entry = batch->validation_list[index];
974 batch->validation_list[index] = tmp;
977 ret = execbuffer(dri_screen->fd, batch, hw_ctx,
978 4 * USED_BATCH(*batch),
979 in_fence_fd, out_fence_fd, flags);
984 if (unlikely(INTEL_DEBUG & DEBUG_BATCH))
987 if (brw->ctx.Const.ResetStrategy == GL_LOSE_CONTEXT_ON_RESET_ARB)
988 brw_check_for_reset(brw);
991 fprintf(stderr, "i965: Failed to submit batchbuffer: %s\n",
1000 * The in_fence_fd is ignored if -1. Otherwise this function takes ownership
1003 * The out_fence_fd is ignored if NULL. Otherwise, the caller takes ownership
1004 * of the returned fd.
1007 _intel_batchbuffer_flush_fence(struct brw_context *brw,
1008 int in_fence_fd, int *out_fence_fd,
1009 const char *file, int line)
1013 if (USED_BATCH(brw->batch) == 0)
1016 /* Check that we didn't just wrap our batchbuffer at a bad time. */
1017 assert(!brw->batch.no_wrap);
1019 brw_finish_batch(brw);
1020 brw_upload_finish(&brw->upload);
1022 finish_growing_bos(&brw->batch.batch);
1023 finish_growing_bos(&brw->batch.state);
1025 if (brw->throttle_batch[0] == NULL) {
1026 brw->throttle_batch[0] = brw->batch.batch.bo;
1027 brw_bo_reference(brw->throttle_batch[0]);
1030 if (unlikely(INTEL_DEBUG & (DEBUG_BATCH | DEBUG_SUBMIT))) {
1031 int bytes_for_commands = 4 * USED_BATCH(brw->batch);
1032 int bytes_for_state = brw->batch.state_used;
1033 fprintf(stderr, "%19s:%-3d: Batchbuffer flush with %5db (%0.1f%%) (pkt),"
1034 " %5db (%0.1f%%) (state), %4d BOs (%0.1fMb aperture),"
1035 " %4d batch relocs, %4d state relocs\n", file, line,
1036 bytes_for_commands, 100.0f * bytes_for_commands / BATCH_SZ,
1037 bytes_for_state, 100.0f * bytes_for_state / STATE_SZ,
1038 brw->batch.exec_count,
1039 (float) brw->batch.aperture_space / (1024 * 1024),
1040 brw->batch.batch_relocs.reloc_count,
1041 brw->batch.state_relocs.reloc_count);
1044 ret = submit_batch(brw, in_fence_fd, out_fence_fd);
1046 if (unlikely(INTEL_DEBUG & DEBUG_SYNC)) {
1047 fprintf(stderr, "waiting for idle\n");
1048 brw_bo_wait_rendering(brw->batch.batch.bo);
1051 /* Start a new batch buffer. */
1058 brw_batch_has_aperture_space(struct brw_context *brw, unsigned extra_space)
1060 return brw->batch.aperture_space + extra_space <=
1061 brw->screen->aperture_threshold;
1065 brw_batch_references(struct intel_batchbuffer *batch, struct brw_bo *bo)
1067 unsigned index = READ_ONCE(bo->index);
1068 if (index < batch->exec_count && batch->exec_bos[index] == bo)
1071 for (int i = 0; i < batch->exec_count; i++) {
1072 if (batch->exec_bos[i] == bo)
1078 /* This is the only way buffers get added to the validate list.
1081 emit_reloc(struct intel_batchbuffer *batch,
1082 struct brw_reloc_list *rlist, uint32_t offset,
1083 struct brw_bo *target, int32_t target_offset,
1084 unsigned int reloc_flags)
1086 assert(target != NULL);
1088 if (rlist->reloc_count == rlist->reloc_array_size) {
1089 rlist->reloc_array_size *= 2;
1090 rlist->relocs = realloc(rlist->relocs,
1091 rlist->reloc_array_size *
1092 sizeof(struct drm_i915_gem_relocation_entry));
1095 unsigned int index = add_exec_bo(batch, target);
1096 struct drm_i915_gem_exec_object2 *entry = &batch->validation_list[index];
1098 if (reloc_flags & RELOC_32BIT) {
1099 /* Restrict this buffer to the low 32 bits of the address space.
1101 * Altering the validation list flags restricts it for this batch,
1102 * but we also alter the BO's kflags to restrict it permanently
1103 * (until the BO is destroyed and put back in the cache). Buffers
1104 * may stay bound across batches, and we want keep it constrained.
1106 target->kflags &= ~EXEC_OBJECT_SUPPORTS_48B_ADDRESS;
1107 entry->flags &= ~EXEC_OBJECT_SUPPORTS_48B_ADDRESS;
1109 /* RELOC_32BIT is not an EXEC_OBJECT_* flag, so get rid of it. */
1110 reloc_flags &= ~RELOC_32BIT;
1114 entry->flags |= reloc_flags & batch->valid_reloc_flags;
1116 rlist->relocs[rlist->reloc_count++] =
1117 (struct drm_i915_gem_relocation_entry) {
1119 .delta = target_offset,
1120 .target_handle = batch->use_batch_first ? index : target->gem_handle,
1121 .presumed_offset = entry->offset,
1124 /* Using the old buffer offset, write in what the right data would be, in
1125 * case the buffer doesn't move and we can short-circuit the relocation
1126 * processing in the kernel
1128 return entry->offset + target_offset;
1132 brw_batch_reloc(struct intel_batchbuffer *batch, uint32_t batch_offset,
1133 struct brw_bo *target, uint32_t target_offset,
1134 unsigned int reloc_flags)
1136 assert(batch_offset <= batch->batch.bo->size - sizeof(uint32_t));
1138 return emit_reloc(batch, &batch->batch_relocs, batch_offset,
1139 target, target_offset, reloc_flags);
1143 brw_state_reloc(struct intel_batchbuffer *batch, uint32_t state_offset,
1144 struct brw_bo *target, uint32_t target_offset,
1145 unsigned int reloc_flags)
1147 assert(state_offset <= batch->state.bo->size - sizeof(uint32_t));
1149 return emit_reloc(batch, &batch->state_relocs, state_offset,
1150 target, target_offset, reloc_flags);
1155 brw_state_batch_size(struct brw_context *brw, uint32_t offset)
1157 struct hash_entry *entry =
1158 _mesa_hash_table_search(brw->batch.state_batch_sizes,
1159 (void *) (uintptr_t) offset);
1160 return entry ? (uintptr_t) entry->data : 0;
1164 * Reserve some space in the statebuffer, or flush.
1166 * This is used to estimate when we're near the end of the batch,
1167 * so we can flush early.
1170 brw_require_statebuffer_space(struct brw_context *brw, int size)
1172 if (brw->batch.state_used + size >= STATE_SZ)
1173 intel_batchbuffer_flush(brw);
1177 * Allocates a block of space in the batchbuffer for indirect state.
1180 brw_state_batch(struct brw_context *brw,
1183 uint32_t *out_offset)
1185 struct intel_batchbuffer *batch = &brw->batch;
1187 assert(size < batch->state.bo->size);
1189 uint32_t offset = ALIGN(batch->state_used, alignment);
1191 if (offset + size >= STATE_SZ && !batch->no_wrap) {
1192 intel_batchbuffer_flush(brw);
1193 offset = ALIGN(batch->state_used, alignment);
1194 } else if (offset + size >= batch->state.bo->size) {
1195 const unsigned new_size =
1196 MIN2(batch->state.bo->size + batch->state.bo->size / 2,
1198 grow_buffer(brw, &batch->state, batch->state_used, new_size);
1199 assert(offset + size < batch->state.bo->size);
1202 if (unlikely(INTEL_DEBUG & DEBUG_BATCH)) {
1203 _mesa_hash_table_insert(batch->state_batch_sizes,
1204 (void *) (uintptr_t) offset,
1205 (void *) (uintptr_t) size);
1208 batch->state_used = offset + size;
1210 *out_offset = offset;
1211 return batch->state.map + (offset >> 2);
1215 intel_batchbuffer_data(struct brw_context *brw,
1216 const void *data, GLuint bytes, enum brw_gpu_ring ring)
1218 assert((bytes & 3) == 0);
1219 intel_batchbuffer_require_space(brw, bytes, ring);
1220 memcpy(brw->batch.map_next, data, bytes);
1221 brw->batch.map_next += bytes >> 2;
1225 load_sized_register_mem(struct brw_context *brw,
1231 const struct gen_device_info *devinfo = &brw->screen->devinfo;
1234 /* MI_LOAD_REGISTER_MEM only exists on Gen7+. */
1235 assert(devinfo->gen >= 7);
1237 if (devinfo->gen >= 8) {
1238 BEGIN_BATCH(4 * size);
1239 for (i = 0; i < size; i++) {
1240 OUT_BATCH(GEN7_MI_LOAD_REGISTER_MEM | (4 - 2));
1241 OUT_BATCH(reg + i * 4);
1242 OUT_RELOC64(bo, 0, offset + i * 4);
1246 BEGIN_BATCH(3 * size);
1247 for (i = 0; i < size; i++) {
1248 OUT_BATCH(GEN7_MI_LOAD_REGISTER_MEM | (3 - 2));
1249 OUT_BATCH(reg + i * 4);
1250 OUT_RELOC(bo, 0, offset + i * 4);
1257 brw_load_register_mem(struct brw_context *brw,
1262 load_sized_register_mem(brw, reg, bo, offset, 1);
1266 brw_load_register_mem64(struct brw_context *brw,
1271 load_sized_register_mem(brw, reg, bo, offset, 2);
1275 * Write an arbitrary 32-bit register to a buffer via MI_STORE_REGISTER_MEM.
1278 brw_store_register_mem32(struct brw_context *brw,
1279 struct brw_bo *bo, uint32_t reg, uint32_t offset)
1281 const struct gen_device_info *devinfo = &brw->screen->devinfo;
1283 assert(devinfo->gen >= 6);
1285 if (devinfo->gen >= 8) {
1287 OUT_BATCH(MI_STORE_REGISTER_MEM | (4 - 2));
1289 OUT_RELOC64(bo, RELOC_WRITE, offset);
1293 OUT_BATCH(MI_STORE_REGISTER_MEM | (3 - 2));
1295 OUT_RELOC(bo, RELOC_WRITE | RELOC_NEEDS_GGTT, offset);
1301 * Write an arbitrary 64-bit register to a buffer via MI_STORE_REGISTER_MEM.
1304 brw_store_register_mem64(struct brw_context *brw,
1305 struct brw_bo *bo, uint32_t reg, uint32_t offset)
1307 const struct gen_device_info *devinfo = &brw->screen->devinfo;
1309 assert(devinfo->gen >= 6);
1311 /* MI_STORE_REGISTER_MEM only stores a single 32-bit value, so to
1312 * read a full 64-bit register, we need to do two of them.
1314 if (devinfo->gen >= 8) {
1316 OUT_BATCH(MI_STORE_REGISTER_MEM | (4 - 2));
1318 OUT_RELOC64(bo, RELOC_WRITE, offset);
1319 OUT_BATCH(MI_STORE_REGISTER_MEM | (4 - 2));
1320 OUT_BATCH(reg + sizeof(uint32_t));
1321 OUT_RELOC64(bo, RELOC_WRITE, offset + sizeof(uint32_t));
1325 OUT_BATCH(MI_STORE_REGISTER_MEM | (3 - 2));
1327 OUT_RELOC(bo, RELOC_WRITE | RELOC_NEEDS_GGTT, offset);
1328 OUT_BATCH(MI_STORE_REGISTER_MEM | (3 - 2));
1329 OUT_BATCH(reg + sizeof(uint32_t));
1330 OUT_RELOC(bo, RELOC_WRITE | RELOC_NEEDS_GGTT, offset + sizeof(uint32_t));
1336 * Write a 32-bit register using immediate data.
1339 brw_load_register_imm32(struct brw_context *brw, uint32_t reg, uint32_t imm)
1341 assert(brw->screen->devinfo.gen >= 6);
1344 OUT_BATCH(MI_LOAD_REGISTER_IMM | (3 - 2));
1351 * Write a 64-bit register using immediate data.
1354 brw_load_register_imm64(struct brw_context *brw, uint32_t reg, uint64_t imm)
1356 assert(brw->screen->devinfo.gen >= 6);
1359 OUT_BATCH(MI_LOAD_REGISTER_IMM | (5 - 2));
1361 OUT_BATCH(imm & 0xffffffff);
1363 OUT_BATCH(imm >> 32);
1368 * Copies a 32-bit register.
1371 brw_load_register_reg(struct brw_context *brw, uint32_t src, uint32_t dest)
1373 assert(brw->screen->devinfo.gen >= 8 || brw->screen->devinfo.is_haswell);
1376 OUT_BATCH(MI_LOAD_REGISTER_REG | (3 - 2));
1383 * Copies a 64-bit register.
1386 brw_load_register_reg64(struct brw_context *brw, uint32_t src, uint32_t dest)
1388 assert(brw->screen->devinfo.gen >= 8 || brw->screen->devinfo.is_haswell);
1391 OUT_BATCH(MI_LOAD_REGISTER_REG | (3 - 2));
1394 OUT_BATCH(MI_LOAD_REGISTER_REG | (3 - 2));
1395 OUT_BATCH(src + sizeof(uint32_t));
1396 OUT_BATCH(dest + sizeof(uint32_t));
1401 * Write 32-bits of immediate data to a GPU memory buffer.
1404 brw_store_data_imm32(struct brw_context *brw, struct brw_bo *bo,
1405 uint32_t offset, uint32_t imm)
1407 const struct gen_device_info *devinfo = &brw->screen->devinfo;
1409 assert(devinfo->gen >= 6);
1412 OUT_BATCH(MI_STORE_DATA_IMM | (4 - 2));
1413 if (devinfo->gen >= 8)
1414 OUT_RELOC64(bo, RELOC_WRITE, offset);
1416 OUT_BATCH(0); /* MBZ */
1417 OUT_RELOC(bo, RELOC_WRITE, offset);
1424 * Write 64-bits of immediate data to a GPU memory buffer.
1427 brw_store_data_imm64(struct brw_context *brw, struct brw_bo *bo,
1428 uint32_t offset, uint64_t imm)
1430 const struct gen_device_info *devinfo = &brw->screen->devinfo;
1432 assert(devinfo->gen >= 6);
1435 OUT_BATCH(MI_STORE_DATA_IMM | (5 - 2));
1436 if (devinfo->gen >= 8)
1437 OUT_RELOC64(bo, RELOC_WRITE, offset);
1439 OUT_BATCH(0); /* MBZ */
1440 OUT_RELOC(bo, RELOC_WRITE, offset);
1442 OUT_BATCH(imm & 0xffffffffu);
1443 OUT_BATCH(imm >> 32);