1 /**************************************************************************
3 * Copyright 2006 Tungsten Graphics, Inc., Cedar Park, Texas.
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the
8 * "Software"), to deal in the Software without restriction, including
9 * without limitation the rights to use, copy, modify, merge, publish,
10 * distribute, sub license, and/or sell copies of the Software, and to
11 * permit persons to whom the Software is furnished to do so, subject to
12 * the following conditions:
14 * The above copyright notice and this permission notice (including the
15 * next paragraph) shall be included in all copies or substantial portions
18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
19 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
20 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
21 * IN NO EVENT SHALL TUNGSTEN GRAPHICS AND/OR ITS SUPPLIERS BE LIABLE FOR
22 * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
23 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
24 * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
26 **************************************************************************/
32 #include "intel_batchbuffer.h"
34 #define MAX_BATCH_SIZE 0x400000
37 #define LOCAL_I915_EXEC_BSD_MASK (3<<13)
38 #define LOCAL_I915_EXEC_BSD_DEFAULT (0<<13) /* default ping-pong mode */
39 #define LOCAL_I915_EXEC_BSD_RING0 (1<<13)
40 #define LOCAL_I915_EXEC_BSD_RING1 (2<<13)
43 intel_batchbuffer_reset(struct intel_batchbuffer *batch, int buffer_size)
45 struct intel_driver_data *intel = batch->intel;
46 int batch_size = buffer_size;
49 ring_flag = batch->flag & I915_EXEC_RING_MASK;
51 assert(ring_flag == I915_EXEC_RENDER ||
52 ring_flag == I915_EXEC_BLT ||
53 ring_flag == I915_EXEC_BSD ||
54 ring_flag == I915_EXEC_VEBOX);
56 dri_bo_unreference(batch->buffer);
57 batch->buffer = dri_bo_alloc(intel->bufmgr,
61 assert(batch->buffer);
62 dri_bo_map(batch->buffer, 1);
63 assert(batch->buffer->virtual);
64 batch->map = batch->buffer->virtual;
65 batch->size = batch_size;
66 batch->ptr = batch->map;
71 intel_batchbuffer_space(struct intel_batchbuffer *batch)
73 return (batch->size - BATCH_RESERVED) - (batch->ptr - batch->map);
77 struct intel_batchbuffer *
78 intel_batchbuffer_new(struct intel_driver_data *intel, int flag, int buffer_size)
80 struct intel_batchbuffer *batch = calloc(1, sizeof(*batch));
83 ring_flag = flag & I915_EXEC_RING_MASK;
84 assert(ring_flag == I915_EXEC_RENDER ||
85 ring_flag == I915_EXEC_BSD ||
86 ring_flag == I915_EXEC_BLT ||
87 ring_flag == I915_EXEC_VEBOX);
89 if (!buffer_size || buffer_size < BATCH_SIZE) {
90 buffer_size = BATCH_SIZE;
93 /* the buffer size can't exceed 4M */
94 if (buffer_size > MAX_BATCH_SIZE) {
95 buffer_size = MAX_BATCH_SIZE;
101 batch->run = drm_intel_bo_mrb_exec;
103 if (IS_GEN6(intel->device_info) &&
104 flag == I915_EXEC_RENDER)
105 batch->wa_render_bo = dri_bo_alloc(intel->bufmgr,
110 batch->wa_render_bo = NULL;
112 intel_batchbuffer_reset(batch, buffer_size);
117 void intel_batchbuffer_free(struct intel_batchbuffer *batch)
120 dri_bo_unmap(batch->buffer);
124 dri_bo_unreference(batch->buffer);
125 dri_bo_unreference(batch->wa_render_bo);
130 intel_batchbuffer_flush(struct intel_batchbuffer *batch)
132 unsigned int used = batch->ptr - batch->map;
138 if ((used & 4) == 0) {
139 *(unsigned int*)batch->ptr = 0;
143 *(unsigned int*)batch->ptr = MI_BATCH_BUFFER_END;
145 dri_bo_unmap(batch->buffer);
146 used = batch->ptr - batch->map;
147 batch->run(batch->buffer, used, 0, 0, 0, batch->flag);
148 intel_batchbuffer_reset(batch, batch->size);
152 intel_batchbuffer_emit_dword(struct intel_batchbuffer *batch, unsigned int x)
154 assert(intel_batchbuffer_space(batch) >= 4);
155 *(unsigned int *)batch->ptr = x;
160 intel_batchbuffer_emit_reloc(struct intel_batchbuffer *batch, dri_bo *bo,
161 uint32_t read_domains, uint32_t write_domains,
164 assert(batch->ptr - batch->map < batch->size);
165 dri_bo_emit_reloc(batch->buffer, read_domains, write_domains,
166 delta, batch->ptr - batch->map, bo);
167 intel_batchbuffer_emit_dword(batch, bo->offset + delta);
171 intel_batchbuffer_emit_reloc64(struct intel_batchbuffer *batch, dri_bo *bo,
172 uint32_t read_domains, uint32_t write_domains,
175 assert(batch->ptr - batch->map < batch->size);
176 dri_bo_emit_reloc(batch->buffer, read_domains, write_domains,
177 delta, batch->ptr - batch->map, bo);
179 /* Using the old buffer offset, write in what the right data would be, in
180 * case the buffer doesn't move and we can short-circuit the relocation
181 * processing in the kernel.
183 uint64_t offset = bo->offset64 + delta;
184 intel_batchbuffer_emit_dword(batch, offset);
185 intel_batchbuffer_emit_dword(batch, offset >> 32);
189 intel_batchbuffer_require_space(struct intel_batchbuffer *batch,
192 assert(size < batch->size - 8);
194 if (intel_batchbuffer_space(batch) < size) {
195 intel_batchbuffer_flush(batch);
200 intel_batchbuffer_data(struct intel_batchbuffer *batch,
204 assert((size & 3) == 0);
205 intel_batchbuffer_require_space(batch, size);
208 memcpy(batch->ptr, data, size);
213 intel_batchbuffer_emit_mi_flush(struct intel_batchbuffer *batch)
215 struct intel_driver_data *intel = batch->intel;
218 ring_flag = batch->flag & I915_EXEC_RING_MASK;
220 if (IS_GEN6(intel->device_info) ||
221 IS_GEN7(intel->device_info) ||
222 IS_GEN8(intel->device_info) ||
223 IS_GEN9(intel->device_info) ||
224 IS_GEN10(intel->device_info)) {
225 if (ring_flag == I915_EXEC_RENDER) {
226 if (IS_GEN8(intel->device_info) || IS_GEN9(intel->device_info) ||
227 IS_GEN10(intel->device_info)) {
228 BEGIN_BATCH(batch, 6);
229 OUT_BATCH(batch, CMD_PIPE_CONTROL | (6 - 2));
232 CMD_PIPE_CONTROL_CS_STALL |
233 CMD_PIPE_CONTROL_WC_FLUSH |
234 CMD_PIPE_CONTROL_TC_FLUSH |
235 CMD_PIPE_CONTROL_DC_FLUSH |
236 CMD_PIPE_CONTROL_NOWRITE);
237 OUT_BATCH(batch, 0); /* write address */
239 OUT_BATCH(batch, 0); /* write data */
241 ADVANCE_BATCH(batch);
242 } else if (IS_GEN6(intel->device_info)) {
243 assert(batch->wa_render_bo);
245 BEGIN_BATCH(batch, 4 * 3);
247 OUT_BATCH(batch, CMD_PIPE_CONTROL | (4 - 2));
249 CMD_PIPE_CONTROL_CS_STALL |
250 CMD_PIPE_CONTROL_STALL_AT_SCOREBOARD);
251 OUT_BATCH(batch, 0); /* address */
252 OUT_BATCH(batch, 0); /* write data */
254 OUT_BATCH(batch, CMD_PIPE_CONTROL | (4 - 2));
255 OUT_BATCH(batch, CMD_PIPE_CONTROL_WRITE_QWORD);
258 I915_GEM_DOMAIN_INSTRUCTION,
259 I915_GEM_DOMAIN_INSTRUCTION,
261 OUT_BATCH(batch, 0); /* write data */
263 /* now finally the _real flush */
264 OUT_BATCH(batch, CMD_PIPE_CONTROL | (4 - 2));
266 CMD_PIPE_CONTROL_WC_FLUSH |
267 CMD_PIPE_CONTROL_TC_FLUSH |
268 CMD_PIPE_CONTROL_NOWRITE);
269 OUT_BATCH(batch, 0); /* write address */
270 OUT_BATCH(batch, 0); /* write data */
271 ADVANCE_BATCH(batch);
273 BEGIN_BATCH(batch, 4);
274 OUT_BATCH(batch, CMD_PIPE_CONTROL | (4 - 2));
277 CMD_PIPE_CONTROL_WC_FLUSH |
278 CMD_PIPE_CONTROL_TC_FLUSH |
279 CMD_PIPE_CONTROL_DC_FLUSH |
280 CMD_PIPE_CONTROL_NOWRITE);
281 OUT_BATCH(batch, 0); /* write address */
282 OUT_BATCH(batch, 0); /* write data */
283 ADVANCE_BATCH(batch);
287 if (ring_flag == I915_EXEC_BLT) {
288 BEGIN_BLT_BATCH(batch, 4);
289 OUT_BLT_BATCH(batch, MI_FLUSH_DW);
290 OUT_BLT_BATCH(batch, 0);
291 OUT_BLT_BATCH(batch, 0);
292 OUT_BLT_BATCH(batch, 0);
293 ADVANCE_BLT_BATCH(batch);
294 } else if (ring_flag == I915_EXEC_VEBOX) {
295 BEGIN_VEB_BATCH(batch, 4);
296 OUT_VEB_BATCH(batch, MI_FLUSH_DW);
297 OUT_VEB_BATCH(batch, 0);
298 OUT_VEB_BATCH(batch, 0);
299 OUT_VEB_BATCH(batch, 0);
300 ADVANCE_VEB_BATCH(batch);
302 assert(ring_flag == I915_EXEC_BSD);
303 BEGIN_BCS_BATCH(batch, 4);
304 OUT_BCS_BATCH(batch, MI_FLUSH_DW | MI_FLUSH_DW_VIDEO_PIPELINE_CACHE_INVALIDATE);
305 OUT_BCS_BATCH(batch, 0);
306 OUT_BCS_BATCH(batch, 0);
307 OUT_BCS_BATCH(batch, 0);
308 ADVANCE_BCS_BATCH(batch);
312 if (ring_flag == I915_EXEC_RENDER) {
313 BEGIN_BATCH(batch, 1);
314 OUT_BATCH(batch, MI_FLUSH | MI_FLUSH_STATE_INSTRUCTION_CACHE_INVALIDATE);
315 ADVANCE_BATCH(batch);
317 assert(ring_flag == I915_EXEC_BSD);
318 BEGIN_BCS_BATCH(batch, 1);
319 OUT_BCS_BATCH(batch, MI_FLUSH | MI_FLUSH_STATE_INSTRUCTION_CACHE_INVALIDATE);
320 ADVANCE_BCS_BATCH(batch);
326 intel_batchbuffer_begin_batch(struct intel_batchbuffer *batch, int total)
328 batch->emit_total = total * 4;
329 batch->emit_start = batch->ptr;
333 intel_batchbuffer_advance_batch(struct intel_batchbuffer *batch)
335 assert(batch->emit_total == (batch->ptr - batch->emit_start));
339 intel_batchbuffer_check_batchbuffer_flag(struct intel_batchbuffer *batch, int flag)
343 ring_flag = flag & I915_EXEC_RING_MASK;
345 if (ring_flag != I915_EXEC_RENDER &&
346 ring_flag != I915_EXEC_BLT &&
347 ring_flag != I915_EXEC_BSD &&
348 ring_flag != I915_EXEC_VEBOX)
351 if (batch->flag == flag)
354 intel_batchbuffer_flush(batch);
359 intel_batchbuffer_check_free_space(struct intel_batchbuffer *batch, int size)
361 return intel_batchbuffer_space(batch) >= size;
365 intel_batchbuffer_start_atomic_helper(struct intel_batchbuffer *batch,
369 assert(!batch->atomic);
370 intel_batchbuffer_check_batchbuffer_flag(batch, flag);
371 intel_batchbuffer_require_space(batch, size);
376 intel_batchbuffer_start_atomic(struct intel_batchbuffer *batch, unsigned int size)
378 intel_batchbuffer_start_atomic_helper(batch, I915_EXEC_RENDER, size);
382 intel_batchbuffer_start_atomic_blt(struct intel_batchbuffer *batch, unsigned int size)
384 intel_batchbuffer_start_atomic_helper(batch, I915_EXEC_BLT, size);
388 intel_batchbuffer_start_atomic_bcs(struct intel_batchbuffer *batch, unsigned int size)
390 intel_batchbuffer_start_atomic_helper(batch, I915_EXEC_BSD, size);
394 intel_batchbuffer_start_atomic_veb(struct intel_batchbuffer *batch, unsigned int size)
396 intel_batchbuffer_start_atomic_helper(batch, I915_EXEC_VEBOX, size);
399 void intel_batchbuffer_start_atomic_bcs_override(struct intel_batchbuffer *batch, unsigned int size,
400 bsd_ring_flag override_flag)
404 switch (override_flag) {
406 ring_flag = I915_EXEC_BSD + LOCAL_I915_EXEC_BSD_RING0;
409 ring_flag = I915_EXEC_BSD + LOCAL_I915_EXEC_BSD_RING1;
412 ring_flag = I915_EXEC_BSD + LOCAL_I915_EXEC_BSD_DEFAULT;
415 intel_batchbuffer_start_atomic_helper(batch, ring_flag, size);
420 intel_batchbuffer_end_atomic(struct intel_batchbuffer *batch)
422 assert(batch->atomic);
427 intel_batchbuffer_used_size(struct intel_batchbuffer *batch)
429 return batch->ptr - batch->map;
433 intel_batchbuffer_align(struct intel_batchbuffer *batch, unsigned int alignedment)
435 int used = batch->ptr - batch->map;
438 assert((alignedment & 3) == 0);
439 pad_size = ALIGN(used, alignedment) - used;
440 assert((pad_size & 3) == 0);
441 assert(intel_batchbuffer_space(batch) >= pad_size);
443 while (pad_size >= 4) {
444 intel_batchbuffer_emit_dword(batch, 0);