2 * Copyright © 2014-2015 Broadcom
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
25 * DOC: Render command list generation
27 * In the V3D hardware, render command lists are what load and store
28 * tiles of a framebuffer and optionally call out to binner-generated
29 * command lists to do the 3D drawing for that tile.
31 * In the VC4 driver, render command list generation is performed by the
32 * kernel instead of userspace. We do this because validating a
33 * user-submitted command list is hard to get right and has high CPU overhead,
34 * while the number of valid configurations for render command lists is
35 * actually fairly low.
38 #include "uapi/drm/vc4_drm.h"
40 #include "vc4_packet.h"
42 struct vc4_rcl_setup {
43 struct drm_gem_cma_object *color_read;
44 struct drm_gem_cma_object *color_write;
45 struct drm_gem_cma_object *zs_read;
46 struct drm_gem_cma_object *zs_write;
47 struct drm_gem_cma_object *msaa_color_write;
48 struct drm_gem_cma_object *msaa_zs_write;
50 struct drm_gem_cma_object *rcl;
53 u32 next_write_bo_index;
56 static inline void rcl_u8(struct vc4_rcl_setup *setup, u8 val)
58 *(u8 *)(setup->rcl->vaddr + setup->next_offset) = val;
59 setup->next_offset += 1;
62 static inline void rcl_u16(struct vc4_rcl_setup *setup, u16 val)
64 *(u16 *)(setup->rcl->vaddr + setup->next_offset) = val;
65 setup->next_offset += 2;
68 static inline void rcl_u32(struct vc4_rcl_setup *setup, u32 val)
70 *(u32 *)(setup->rcl->vaddr + setup->next_offset) = val;
71 setup->next_offset += 4;
75 * Emits a no-op STORE_TILE_BUFFER_GENERAL.
77 * If we emit a PACKET_TILE_COORDINATES, it must be followed by a store of
78 * some sort before another load is triggered.
80 static void vc4_store_before_load(struct vc4_rcl_setup *setup)
82 rcl_u8(setup, VC4_PACKET_STORE_TILE_BUFFER_GENERAL);
84 VC4_SET_FIELD(VC4_LOADSTORE_TILE_BUFFER_NONE,
85 VC4_LOADSTORE_TILE_BUFFER_BUFFER) |
86 VC4_STORE_TILE_BUFFER_DISABLE_COLOR_CLEAR |
87 VC4_STORE_TILE_BUFFER_DISABLE_ZS_CLEAR |
88 VC4_STORE_TILE_BUFFER_DISABLE_VG_MASK_CLEAR);
89 rcl_u32(setup, 0); /* no address, since we're in None mode */
93 * Calculates the physical address of the start of a tile in a RCL surface.
95 * Unlike the other load/store packets,
96 * VC4_PACKET_LOAD/STORE_FULL_RES_TILE_BUFFER don't look at the tile
97 * coordinates packet, and instead just store to the address given.
99 static uint32_t vc4_full_res_offset(struct vc4_exec_info *exec,
100 struct drm_gem_cma_object *bo,
101 struct drm_vc4_submit_rcl_surface *surf,
102 uint8_t x, uint8_t y)
104 return bo->paddr + surf->offset + VC4_TILE_BUFFER_SIZE *
105 (DIV_ROUND_UP(exec->args->width, 32) * y + x);
109 * Emits a PACKET_TILE_COORDINATES if one isn't already pending.
111 * The tile coordinates packet triggers a pending load if there is one, are
112 * used for clipping during rendering, and determine where loads/stores happen
113 * relative to their base address.
115 static void vc4_tile_coordinates(struct vc4_rcl_setup *setup,
116 uint32_t x, uint32_t y)
118 rcl_u8(setup, VC4_PACKET_TILE_COORDINATES);
123 static void emit_tile(struct vc4_exec_info *exec,
124 struct vc4_rcl_setup *setup,
125 uint8_t x, uint8_t y, bool first, bool last)
127 struct drm_vc4_submit_cl *args = exec->args;
128 bool has_bin = args->bin_cl_size != 0;
130 /* Note that the load doesn't actually occur until the
131 * tile coords packet is processed, and only one load
132 * may be outstanding at a time.
134 if (setup->color_read) {
135 if (args->color_read.flags &
136 VC4_SUBMIT_RCL_SURFACE_READ_IS_FULL_RES) {
137 rcl_u8(setup, VC4_PACKET_LOAD_FULL_RES_TILE_BUFFER);
139 vc4_full_res_offset(exec, setup->color_read,
140 &args->color_read, x, y) |
141 VC4_LOADSTORE_FULL_RES_DISABLE_ZS);
143 rcl_u8(setup, VC4_PACKET_LOAD_TILE_BUFFER_GENERAL);
144 rcl_u16(setup, args->color_read.bits);
145 rcl_u32(setup, setup->color_read->paddr +
146 args->color_read.offset);
150 if (setup->zs_read) {
151 if (args->zs_read.flags &
152 VC4_SUBMIT_RCL_SURFACE_READ_IS_FULL_RES) {
153 rcl_u8(setup, VC4_PACKET_LOAD_FULL_RES_TILE_BUFFER);
155 vc4_full_res_offset(exec, setup->zs_read,
156 &args->zs_read, x, y) |
157 VC4_LOADSTORE_FULL_RES_DISABLE_COLOR);
159 if (setup->color_read) {
160 /* Exec previous load. */
161 vc4_tile_coordinates(setup, x, y);
162 vc4_store_before_load(setup);
165 rcl_u8(setup, VC4_PACKET_LOAD_TILE_BUFFER_GENERAL);
166 rcl_u16(setup, args->zs_read.bits);
167 rcl_u32(setup, setup->zs_read->paddr +
168 args->zs_read.offset);
172 /* Clipping depends on tile coordinates having been
173 * emitted, so we always need one here.
175 vc4_tile_coordinates(setup, x, y);
177 /* Wait for the binner before jumping to the first
180 if (first && has_bin)
181 rcl_u8(setup, VC4_PACKET_WAIT_ON_SEMAPHORE);
184 rcl_u8(setup, VC4_PACKET_BRANCH_TO_SUB_LIST);
185 rcl_u32(setup, (exec->tile_alloc_offset +
186 (y * exec->bin_tiles_x + x) * 32));
189 if (setup->msaa_color_write) {
190 bool last_tile_write = (!setup->msaa_zs_write &&
192 !setup->color_write);
193 uint32_t bits = VC4_LOADSTORE_FULL_RES_DISABLE_ZS;
195 if (!last_tile_write)
196 bits |= VC4_LOADSTORE_FULL_RES_DISABLE_CLEAR_ALL;
198 bits |= VC4_LOADSTORE_FULL_RES_EOF;
199 rcl_u8(setup, VC4_PACKET_STORE_FULL_RES_TILE_BUFFER);
201 vc4_full_res_offset(exec, setup->msaa_color_write,
202 &args->msaa_color_write, x, y) |
206 if (setup->msaa_zs_write) {
207 bool last_tile_write = (!setup->zs_write &&
208 !setup->color_write);
209 uint32_t bits = VC4_LOADSTORE_FULL_RES_DISABLE_COLOR;
211 if (setup->msaa_color_write)
212 vc4_tile_coordinates(setup, x, y);
213 if (!last_tile_write)
214 bits |= VC4_LOADSTORE_FULL_RES_DISABLE_CLEAR_ALL;
216 bits |= VC4_LOADSTORE_FULL_RES_EOF;
217 rcl_u8(setup, VC4_PACKET_STORE_FULL_RES_TILE_BUFFER);
219 vc4_full_res_offset(exec, setup->msaa_zs_write,
220 &args->msaa_zs_write, x, y) |
224 if (setup->zs_write) {
225 bool last_tile_write = !setup->color_write;
227 if (setup->msaa_color_write || setup->msaa_zs_write)
228 vc4_tile_coordinates(setup, x, y);
230 rcl_u8(setup, VC4_PACKET_STORE_TILE_BUFFER_GENERAL);
231 rcl_u16(setup, args->zs_write.bits |
233 0 : VC4_STORE_TILE_BUFFER_DISABLE_COLOR_CLEAR));
235 (setup->zs_write->paddr + args->zs_write.offset) |
236 ((last && last_tile_write) ?
237 VC4_LOADSTORE_TILE_BUFFER_EOF : 0));
240 if (setup->color_write) {
241 if (setup->msaa_color_write || setup->msaa_zs_write ||
243 vc4_tile_coordinates(setup, x, y);
247 rcl_u8(setup, VC4_PACKET_STORE_MS_TILE_BUFFER_AND_EOF);
249 rcl_u8(setup, VC4_PACKET_STORE_MS_TILE_BUFFER);
253 static int vc4_create_rcl_bo(struct drm_device *dev, struct vc4_exec_info *exec,
254 struct vc4_rcl_setup *setup)
256 struct drm_vc4_submit_cl *args = exec->args;
257 bool has_bin = args->bin_cl_size != 0;
258 uint8_t min_x_tile = args->min_x_tile;
259 uint8_t min_y_tile = args->min_y_tile;
260 uint8_t max_x_tile = args->max_x_tile;
261 uint8_t max_y_tile = args->max_y_tile;
262 uint8_t xtiles = max_x_tile - min_x_tile + 1;
263 uint8_t ytiles = max_y_tile - min_y_tile + 1;
265 uint32_t size, loop_body_size;
267 size = VC4_PACKET_TILE_RENDERING_MODE_CONFIG_SIZE;
268 loop_body_size = VC4_PACKET_TILE_COORDINATES_SIZE;
270 if (args->flags & VC4_SUBMIT_CL_USE_CLEAR_COLOR) {
271 size += VC4_PACKET_CLEAR_COLORS_SIZE +
272 VC4_PACKET_TILE_COORDINATES_SIZE +
273 VC4_PACKET_STORE_TILE_BUFFER_GENERAL_SIZE;
276 if (setup->color_read) {
277 if (args->color_read.flags &
278 VC4_SUBMIT_RCL_SURFACE_READ_IS_FULL_RES) {
279 loop_body_size += VC4_PACKET_LOAD_FULL_RES_TILE_BUFFER_SIZE;
281 loop_body_size += VC4_PACKET_LOAD_TILE_BUFFER_GENERAL_SIZE;
284 if (setup->zs_read) {
285 if (args->zs_read.flags &
286 VC4_SUBMIT_RCL_SURFACE_READ_IS_FULL_RES) {
287 loop_body_size += VC4_PACKET_LOAD_FULL_RES_TILE_BUFFER_SIZE;
289 if (setup->color_read &&
290 !(args->color_read.flags &
291 VC4_SUBMIT_RCL_SURFACE_READ_IS_FULL_RES)) {
292 loop_body_size += VC4_PACKET_TILE_COORDINATES_SIZE;
293 loop_body_size += VC4_PACKET_STORE_TILE_BUFFER_GENERAL_SIZE;
295 loop_body_size += VC4_PACKET_LOAD_TILE_BUFFER_GENERAL_SIZE;
300 size += VC4_PACKET_WAIT_ON_SEMAPHORE_SIZE;
301 loop_body_size += VC4_PACKET_BRANCH_TO_SUB_LIST_SIZE;
304 if (setup->msaa_color_write)
305 loop_body_size += VC4_PACKET_STORE_FULL_RES_TILE_BUFFER_SIZE;
306 if (setup->msaa_zs_write)
307 loop_body_size += VC4_PACKET_STORE_FULL_RES_TILE_BUFFER_SIZE;
310 loop_body_size += VC4_PACKET_STORE_TILE_BUFFER_GENERAL_SIZE;
311 if (setup->color_write)
312 loop_body_size += VC4_PACKET_STORE_MS_TILE_BUFFER_SIZE;
314 /* We need a VC4_PACKET_TILE_COORDINATES in between each store. */
315 loop_body_size += VC4_PACKET_TILE_COORDINATES_SIZE *
316 ((setup->msaa_color_write != NULL) +
317 (setup->msaa_zs_write != NULL) +
318 (setup->color_write != NULL) +
319 (setup->zs_write != NULL) - 1);
321 size += xtiles * ytiles * loop_body_size;
323 setup->rcl = &vc4_bo_create(dev, size, true)->base;
324 if (IS_ERR(setup->rcl))
325 return PTR_ERR(setup->rcl);
326 list_add_tail(&to_vc4_bo(&setup->rcl->base)->unref_head,
329 /* The tile buffer gets cleared when the previous tile is stored. If
330 * the clear values changed between frames, then the tile buffer has
331 * stale clear values in it, so we have to do a store in None mode (no
332 * writes) so that we trigger the tile buffer clear.
334 if (args->flags & VC4_SUBMIT_CL_USE_CLEAR_COLOR) {
335 rcl_u8(setup, VC4_PACKET_CLEAR_COLORS);
336 rcl_u32(setup, args->clear_color[0]);
337 rcl_u32(setup, args->clear_color[1]);
338 rcl_u32(setup, args->clear_z);
339 rcl_u8(setup, args->clear_s);
341 vc4_tile_coordinates(setup, 0, 0);
343 rcl_u8(setup, VC4_PACKET_STORE_TILE_BUFFER_GENERAL);
344 rcl_u16(setup, VC4_LOADSTORE_TILE_BUFFER_NONE);
345 rcl_u32(setup, 0); /* no address, since we're in None mode */
348 rcl_u8(setup, VC4_PACKET_TILE_RENDERING_MODE_CONFIG);
350 (setup->color_write ? (setup->color_write->paddr +
351 args->color_write.offset) :
353 rcl_u16(setup, args->width);
354 rcl_u16(setup, args->height);
355 rcl_u16(setup, args->color_write.bits);
357 for (y = min_y_tile; y <= max_y_tile; y++) {
358 for (x = min_x_tile; x <= max_x_tile; x++) {
359 bool first = (x == min_x_tile && y == min_y_tile);
360 bool last = (x == max_x_tile && y == max_y_tile);
362 emit_tile(exec, setup, x, y, first, last);
366 BUG_ON(setup->next_offset != size);
367 exec->ct1ca = setup->rcl->paddr;
368 exec->ct1ea = setup->rcl->paddr + setup->next_offset;
373 static int vc4_full_res_bounds_check(struct vc4_exec_info *exec,
374 struct drm_gem_cma_object *obj,
375 struct drm_vc4_submit_rcl_surface *surf)
377 struct drm_vc4_submit_cl *args = exec->args;
378 u32 render_tiles_stride = DIV_ROUND_UP(exec->args->width, 32);
380 if (surf->offset > obj->base.size) {
381 DRM_ERROR("surface offset %d > BO size %zd\n",
382 surf->offset, obj->base.size);
386 if ((obj->base.size - surf->offset) / VC4_TILE_BUFFER_SIZE <
387 render_tiles_stride * args->max_y_tile + args->max_x_tile) {
388 DRM_ERROR("MSAA tile %d, %d out of bounds "
389 "(bo size %zd, offset %d).\n",
390 args->max_x_tile, args->max_y_tile,
399 static int vc4_rcl_msaa_surface_setup(struct vc4_exec_info *exec,
400 struct drm_gem_cma_object **obj,
401 struct drm_vc4_submit_rcl_surface *surf)
403 if (surf->flags != 0 || surf->bits != 0) {
404 DRM_ERROR("MSAA surface had nonzero flags/bits\n");
408 if (surf->hindex == ~0)
411 *obj = vc4_use_bo(exec, surf->hindex);
415 exec->rcl_write_bo[exec->rcl_write_bo_count++] = *obj;
417 if (surf->offset & 0xf) {
418 DRM_ERROR("MSAA write must be 16b aligned.\n");
422 return vc4_full_res_bounds_check(exec, *obj, surf);
425 static int vc4_rcl_surface_setup(struct vc4_exec_info *exec,
426 struct drm_gem_cma_object **obj,
427 struct drm_vc4_submit_rcl_surface *surf,
430 uint8_t tiling = VC4_GET_FIELD(surf->bits,
431 VC4_LOADSTORE_TILE_BUFFER_TILING);
432 uint8_t buffer = VC4_GET_FIELD(surf->bits,
433 VC4_LOADSTORE_TILE_BUFFER_BUFFER);
434 uint8_t format = VC4_GET_FIELD(surf->bits,
435 VC4_LOADSTORE_TILE_BUFFER_FORMAT);
439 if (surf->flags & ~VC4_SUBMIT_RCL_SURFACE_READ_IS_FULL_RES) {
440 DRM_ERROR("Extra flags set\n");
444 if (surf->hindex == ~0)
447 *obj = vc4_use_bo(exec, surf->hindex);
452 exec->rcl_write_bo[exec->rcl_write_bo_count++] = *obj;
454 if (surf->flags & VC4_SUBMIT_RCL_SURFACE_READ_IS_FULL_RES) {
455 if (surf == &exec->args->zs_write) {
456 DRM_ERROR("general zs write may not be a full-res.\n");
460 if (surf->bits != 0) {
461 DRM_ERROR("load/store general bits set with "
462 "full res load/store.\n");
466 ret = vc4_full_res_bounds_check(exec, *obj, surf);
473 if (surf->bits & ~(VC4_LOADSTORE_TILE_BUFFER_TILING_MASK |
474 VC4_LOADSTORE_TILE_BUFFER_BUFFER_MASK |
475 VC4_LOADSTORE_TILE_BUFFER_FORMAT_MASK)) {
476 DRM_ERROR("Unknown bits in load/store: 0x%04x\n",
481 if (tiling > VC4_TILING_FORMAT_LT) {
482 DRM_ERROR("Bad tiling format\n");
486 if (buffer == VC4_LOADSTORE_TILE_BUFFER_ZS) {
488 DRM_ERROR("No color format should be set for ZS\n");
492 } else if (buffer == VC4_LOADSTORE_TILE_BUFFER_COLOR) {
494 case VC4_LOADSTORE_TILE_BUFFER_BGR565:
495 case VC4_LOADSTORE_TILE_BUFFER_BGR565_DITHER:
498 case VC4_LOADSTORE_TILE_BUFFER_RGBA8888:
502 DRM_ERROR("Bad tile buffer format\n");
506 DRM_ERROR("Bad load/store buffer %d.\n", buffer);
510 if (surf->offset & 0xf) {
511 DRM_ERROR("load/store buffer must be 16b aligned.\n");
515 if (!vc4_check_tex_size(exec, *obj, surf->offset, tiling,
516 exec->args->width, exec->args->height, cpp)) {
524 vc4_rcl_render_config_surface_setup(struct vc4_exec_info *exec,
525 struct vc4_rcl_setup *setup,
526 struct drm_gem_cma_object **obj,
527 struct drm_vc4_submit_rcl_surface *surf)
529 uint8_t tiling = VC4_GET_FIELD(surf->bits,
530 VC4_RENDER_CONFIG_MEMORY_FORMAT);
531 uint8_t format = VC4_GET_FIELD(surf->bits,
532 VC4_RENDER_CONFIG_FORMAT);
535 if (surf->flags != 0) {
536 DRM_ERROR("No flags supported on render config.\n");
540 if (surf->bits & ~(VC4_RENDER_CONFIG_MEMORY_FORMAT_MASK |
541 VC4_RENDER_CONFIG_FORMAT_MASK |
542 VC4_RENDER_CONFIG_MS_MODE_4X |
543 VC4_RENDER_CONFIG_DECIMATE_MODE_4X)) {
544 DRM_ERROR("Unknown bits in render config: 0x%04x\n",
549 if (surf->hindex == ~0)
552 *obj = vc4_use_bo(exec, surf->hindex);
556 exec->rcl_write_bo[exec->rcl_write_bo_count++] = *obj;
558 if (tiling > VC4_TILING_FORMAT_LT) {
559 DRM_ERROR("Bad tiling format\n");
564 case VC4_RENDER_CONFIG_FORMAT_BGR565_DITHERED:
565 case VC4_RENDER_CONFIG_FORMAT_BGR565:
568 case VC4_RENDER_CONFIG_FORMAT_RGBA8888:
572 DRM_ERROR("Bad tile buffer format\n");
576 if (!vc4_check_tex_size(exec, *obj, surf->offset, tiling,
577 exec->args->width, exec->args->height, cpp)) {
584 int vc4_get_rcl(struct drm_device *dev, struct vc4_exec_info *exec)
586 struct vc4_rcl_setup setup = {0};
587 struct drm_vc4_submit_cl *args = exec->args;
588 bool has_bin = args->bin_cl_size != 0;
591 if (args->min_x_tile > args->max_x_tile ||
592 args->min_y_tile > args->max_y_tile) {
593 DRM_ERROR("Bad render tile set (%d,%d)-(%d,%d)\n",
594 args->min_x_tile, args->min_y_tile,
595 args->max_x_tile, args->max_y_tile);
600 (args->max_x_tile > exec->bin_tiles_x ||
601 args->max_y_tile > exec->bin_tiles_y)) {
602 DRM_ERROR("Render tiles (%d,%d) outside of bin config "
604 args->max_x_tile, args->max_y_tile,
605 exec->bin_tiles_x, exec->bin_tiles_y);
609 ret = vc4_rcl_render_config_surface_setup(exec, &setup,
615 ret = vc4_rcl_surface_setup(exec, &setup.color_read, &args->color_read,
620 ret = vc4_rcl_surface_setup(exec, &setup.zs_read, &args->zs_read,
625 ret = vc4_rcl_surface_setup(exec, &setup.zs_write, &args->zs_write,
630 ret = vc4_rcl_msaa_surface_setup(exec, &setup.msaa_color_write,
631 &args->msaa_color_write);
635 ret = vc4_rcl_msaa_surface_setup(exec, &setup.msaa_zs_write,
636 &args->msaa_zs_write);
640 /* We shouldn't even have the job submitted to us if there's no
641 * surface to write out.
643 if (!setup.color_write && !setup.zs_write &&
644 !setup.msaa_color_write && !setup.msaa_zs_write) {
645 DRM_ERROR("RCL requires color or Z/S write\n");
649 return vc4_create_rcl_bo(dev, exec, &setup);