1 /**************************************************************************
3 * Copyright 2006 Tungsten Graphics, Inc., Cedar Park, Texas.
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the
8 * "Software"), to deal in the Software without restriction, including
9 * without limitation the rights to use, copy, modify, merge, publish,
10 * distribute, sub license, and/or sell copies of the Software, and to
11 * permit persons to whom the Software is furnished to do so, subject to
12 * the following conditions:
14 * The above copyright notice and this permission notice (including the
15 * next paragraph) shall be included in all copies or substantial portions
18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
19 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
20 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
21 * IN NO EVENT SHALL TUNGSTEN GRAPHICS AND/OR ITS SUPPLIERS BE LIABLE FOR
22 * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
23 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
24 * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
26 **************************************************************************/
32 #include "intel_batchbuffer.h"
34 #define MAX_BATCH_SIZE 0x400000
37 #define LOCAL_I915_EXEC_BSD_MASK (3<<13)
38 #define LOCAL_I915_EXEC_BSD_DEFAULT (0<<13) /* default ping-pong mode */
39 #define LOCAL_I915_EXEC_BSD_RING0 (1<<13)
40 #define LOCAL_I915_EXEC_BSD_RING1 (2<<13)
43 intel_batchbuffer_reset(struct intel_batchbuffer *batch, int buffer_size)
45 struct intel_driver_data *intel = batch->intel;
46 int batch_size = buffer_size;
49 ring_flag = batch->flag & I915_EXEC_RING_MASK;
51 assert(ring_flag == I915_EXEC_RENDER ||
52 ring_flag == I915_EXEC_BLT ||
53 ring_flag == I915_EXEC_BSD ||
54 ring_flag == I915_EXEC_VEBOX);
56 dri_bo_unreference(batch->buffer);
57 batch->buffer = dri_bo_alloc(intel->bufmgr,
61 assert(batch->buffer);
62 dri_bo_map(batch->buffer, 1);
63 assert(batch->buffer->virtual);
64 batch->map = batch->buffer->virtual;
65 batch->size = batch_size;
66 batch->ptr = batch->map;
71 intel_batchbuffer_space(struct intel_batchbuffer *batch)
73 return (batch->size - BATCH_RESERVED) - (batch->ptr - batch->map);
77 struct intel_batchbuffer *
78 intel_batchbuffer_new(struct intel_driver_data *intel, int flag, int buffer_size)
80 struct intel_batchbuffer *batch = calloc(1, sizeof(*batch));
83 ring_flag = flag & I915_EXEC_RING_MASK;
84 assert(ring_flag == I915_EXEC_RENDER ||
85 ring_flag == I915_EXEC_BSD ||
86 ring_flag == I915_EXEC_BLT ||
87 ring_flag == I915_EXEC_VEBOX);
89 if (!buffer_size || buffer_size < BATCH_SIZE) {
90 buffer_size = BATCH_SIZE;
93 /* the buffer size can't exceed 4M */
94 if (buffer_size > MAX_BATCH_SIZE) {
95 buffer_size = MAX_BATCH_SIZE;
101 batch->run = drm_intel_bo_mrb_exec;
103 if (IS_GEN6(intel->device_info) &&
104 flag == I915_EXEC_RENDER)
105 batch->wa_render_bo = dri_bo_alloc(intel->bufmgr,
110 batch->wa_render_bo = NULL;
112 intel_batchbuffer_reset(batch, buffer_size);
117 void intel_batchbuffer_free(struct intel_batchbuffer *batch)
120 dri_bo_unmap(batch->buffer);
124 dri_bo_unreference(batch->buffer);
125 dri_bo_unreference(batch->wa_render_bo);
130 intel_batchbuffer_flush(struct intel_batchbuffer *batch)
132 unsigned int used = batch->ptr - batch->map;
138 if ((used & 4) == 0) {
139 *(unsigned int*)batch->ptr = 0;
143 *(unsigned int*)batch->ptr = MI_BATCH_BUFFER_END;
145 dri_bo_unmap(batch->buffer);
146 used = batch->ptr - batch->map;
147 batch->run(batch->buffer, used, 0, 0, 0, batch->flag);
148 intel_batchbuffer_reset(batch, batch->size);
152 intel_batchbuffer_emit_dword(struct intel_batchbuffer *batch, unsigned int x)
154 assert(intel_batchbuffer_space(batch) >= 4);
155 *(unsigned int *)batch->ptr = x;
160 intel_batchbuffer_emit_reloc(struct intel_batchbuffer *batch, dri_bo *bo,
161 uint32_t read_domains, uint32_t write_domains,
164 assert(batch->ptr - batch->map < batch->size);
165 dri_bo_emit_reloc(batch->buffer, read_domains, write_domains,
166 delta, batch->ptr - batch->map, bo);
167 intel_batchbuffer_emit_dword(batch, bo->offset + delta);
171 intel_batchbuffer_emit_reloc64(struct intel_batchbuffer *batch, dri_bo *bo,
172 uint32_t read_domains, uint32_t write_domains,
175 assert(batch->ptr - batch->map < batch->size);
176 dri_bo_emit_reloc(batch->buffer, read_domains, write_domains,
177 delta, batch->ptr - batch->map, bo);
179 /* Using the old buffer offset, write in what the right data would be, in
180 * case the buffer doesn't move and we can short-circuit the relocation
181 * processing in the kernel.
183 uint64_t offset = bo->offset64 + delta;
184 intel_batchbuffer_emit_dword(batch, offset);
185 intel_batchbuffer_emit_dword(batch, offset >> 32);
189 intel_batchbuffer_require_space(struct intel_batchbuffer *batch,
192 assert(size < batch->size - 8);
194 if (intel_batchbuffer_space(batch) < size) {
195 intel_batchbuffer_flush(batch);
200 intel_batchbuffer_data(struct intel_batchbuffer *batch,
204 assert((size & 3) == 0);
205 intel_batchbuffer_require_space(batch, size);
208 memcpy(batch->ptr, data, size);
213 intel_batchbuffer_emit_mi_flush(struct intel_batchbuffer *batch)
215 struct intel_driver_data *intel = batch->intel;
218 ring_flag = batch->flag & I915_EXEC_RING_MASK;
220 if (IS_GEN6(intel->device_info) ||
221 IS_GEN7(intel->device_info) ||
222 IS_GEN8(intel->device_info) ||
223 IS_GEN9(intel->device_info)) {
224 if (ring_flag == I915_EXEC_RENDER) {
225 if (IS_GEN8(intel->device_info) || IS_GEN9(intel->device_info)) {
226 BEGIN_BATCH(batch, 6);
227 OUT_BATCH(batch, CMD_PIPE_CONTROL | (6 - 2));
230 CMD_PIPE_CONTROL_CS_STALL |
231 CMD_PIPE_CONTROL_WC_FLUSH |
232 CMD_PIPE_CONTROL_TC_FLUSH |
233 CMD_PIPE_CONTROL_DC_FLUSH |
234 CMD_PIPE_CONTROL_NOWRITE);
235 OUT_BATCH(batch, 0); /* write address */
237 OUT_BATCH(batch, 0); /* write data */
239 ADVANCE_BATCH(batch);
240 } else if (IS_GEN6(intel->device_info)) {
241 assert(batch->wa_render_bo);
243 BEGIN_BATCH(batch, 4 * 3);
245 OUT_BATCH(batch, CMD_PIPE_CONTROL | (4 - 2));
247 CMD_PIPE_CONTROL_CS_STALL |
248 CMD_PIPE_CONTROL_STALL_AT_SCOREBOARD);
249 OUT_BATCH(batch, 0); /* address */
250 OUT_BATCH(batch, 0); /* write data */
252 OUT_BATCH(batch, CMD_PIPE_CONTROL | (4 - 2));
253 OUT_BATCH(batch, CMD_PIPE_CONTROL_WRITE_QWORD);
256 I915_GEM_DOMAIN_INSTRUCTION,
257 I915_GEM_DOMAIN_INSTRUCTION,
259 OUT_BATCH(batch, 0); /* write data */
261 /* now finally the _real flush */
262 OUT_BATCH(batch, CMD_PIPE_CONTROL | (4 - 2));
264 CMD_PIPE_CONTROL_WC_FLUSH |
265 CMD_PIPE_CONTROL_TC_FLUSH |
266 CMD_PIPE_CONTROL_NOWRITE);
267 OUT_BATCH(batch, 0); /* write address */
268 OUT_BATCH(batch, 0); /* write data */
269 ADVANCE_BATCH(batch);
271 BEGIN_BATCH(batch, 4);
272 OUT_BATCH(batch, CMD_PIPE_CONTROL | (4 - 2));
275 CMD_PIPE_CONTROL_WC_FLUSH |
276 CMD_PIPE_CONTROL_TC_FLUSH |
277 CMD_PIPE_CONTROL_DC_FLUSH |
278 CMD_PIPE_CONTROL_NOWRITE);
279 OUT_BATCH(batch, 0); /* write address */
280 OUT_BATCH(batch, 0); /* write data */
281 ADVANCE_BATCH(batch);
285 if (ring_flag == I915_EXEC_BLT) {
286 BEGIN_BLT_BATCH(batch, 4);
287 OUT_BLT_BATCH(batch, MI_FLUSH_DW);
288 OUT_BLT_BATCH(batch, 0);
289 OUT_BLT_BATCH(batch, 0);
290 OUT_BLT_BATCH(batch, 0);
291 ADVANCE_BLT_BATCH(batch);
292 } else if (ring_flag == I915_EXEC_VEBOX) {
293 BEGIN_VEB_BATCH(batch, 4);
294 OUT_VEB_BATCH(batch, MI_FLUSH_DW);
295 OUT_VEB_BATCH(batch, 0);
296 OUT_VEB_BATCH(batch, 0);
297 OUT_VEB_BATCH(batch, 0);
298 ADVANCE_VEB_BATCH(batch);
300 assert(ring_flag == I915_EXEC_BSD);
301 BEGIN_BCS_BATCH(batch, 4);
302 OUT_BCS_BATCH(batch, MI_FLUSH_DW | MI_FLUSH_DW_VIDEO_PIPELINE_CACHE_INVALIDATE);
303 OUT_BCS_BATCH(batch, 0);
304 OUT_BCS_BATCH(batch, 0);
305 OUT_BCS_BATCH(batch, 0);
306 ADVANCE_BCS_BATCH(batch);
310 if (ring_flag == I915_EXEC_RENDER) {
311 BEGIN_BATCH(batch, 1);
312 OUT_BATCH(batch, MI_FLUSH | MI_FLUSH_STATE_INSTRUCTION_CACHE_INVALIDATE);
313 ADVANCE_BATCH(batch);
315 assert(ring_flag == I915_EXEC_BSD);
316 BEGIN_BCS_BATCH(batch, 1);
317 OUT_BCS_BATCH(batch, MI_FLUSH | MI_FLUSH_STATE_INSTRUCTION_CACHE_INVALIDATE);
318 ADVANCE_BCS_BATCH(batch);
324 intel_batchbuffer_begin_batch(struct intel_batchbuffer *batch, int total)
326 batch->emit_total = total * 4;
327 batch->emit_start = batch->ptr;
331 intel_batchbuffer_advance_batch(struct intel_batchbuffer *batch)
333 assert(batch->emit_total == (batch->ptr - batch->emit_start));
337 intel_batchbuffer_check_batchbuffer_flag(struct intel_batchbuffer *batch, int flag)
341 ring_flag = flag & I915_EXEC_RING_MASK;
343 if (ring_flag != I915_EXEC_RENDER &&
344 ring_flag != I915_EXEC_BLT &&
345 ring_flag != I915_EXEC_BSD &&
346 ring_flag != I915_EXEC_VEBOX)
349 if (batch->flag == flag)
352 intel_batchbuffer_flush(batch);
357 intel_batchbuffer_check_free_space(struct intel_batchbuffer *batch, int size)
359 return intel_batchbuffer_space(batch) >= size;
363 intel_batchbuffer_start_atomic_helper(struct intel_batchbuffer *batch,
367 assert(!batch->atomic);
368 intel_batchbuffer_check_batchbuffer_flag(batch, flag);
369 intel_batchbuffer_require_space(batch, size);
374 intel_batchbuffer_start_atomic(struct intel_batchbuffer *batch, unsigned int size)
376 intel_batchbuffer_start_atomic_helper(batch, I915_EXEC_RENDER, size);
380 intel_batchbuffer_start_atomic_blt(struct intel_batchbuffer *batch, unsigned int size)
382 intel_batchbuffer_start_atomic_helper(batch, I915_EXEC_BLT, size);
386 intel_batchbuffer_start_atomic_bcs(struct intel_batchbuffer *batch, unsigned int size)
388 intel_batchbuffer_start_atomic_helper(batch, I915_EXEC_BSD, size);
392 intel_batchbuffer_start_atomic_veb(struct intel_batchbuffer *batch, unsigned int size)
394 intel_batchbuffer_start_atomic_helper(batch, I915_EXEC_VEBOX, size);
397 void intel_batchbuffer_start_atomic_bcs_override(struct intel_batchbuffer *batch, unsigned int size,
398 bsd_ring_flag override_flag)
402 switch (override_flag) {
404 ring_flag = I915_EXEC_BSD + LOCAL_I915_EXEC_BSD_RING0;
407 ring_flag = I915_EXEC_BSD + LOCAL_I915_EXEC_BSD_RING1;
410 ring_flag = I915_EXEC_BSD + LOCAL_I915_EXEC_BSD_DEFAULT;
413 intel_batchbuffer_start_atomic_helper(batch, ring_flag, size);
418 intel_batchbuffer_end_atomic(struct intel_batchbuffer *batch)
420 assert(batch->atomic);
425 intel_batchbuffer_used_size(struct intel_batchbuffer *batch)
427 return batch->ptr - batch->map;
431 intel_batchbuffer_align(struct intel_batchbuffer *batch, unsigned int alignedment)
433 int used = batch->ptr - batch->map;
436 assert((alignedment & 3) == 0);
437 pad_size = ALIGN(used, alignedment) - used;
438 assert((pad_size & 3) == 0);
439 assert(intel_batchbuffer_space(batch) >= pad_size);
441 while (pad_size >= 4) {
442 intel_batchbuffer_emit_dword(batch, 0);