2 * Copyright © 2010 Intel Corporation
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
21 * DEALINGS IN THE SOFTWARE.
24 #include "compiler/glsl_types.h"
25 #include "loop_analysis.h"
26 #include "ir_hierarchical_visitor.h"
28 #include "main/mtypes.h"
32 class loop_unroll_visitor : public ir_hierarchical_visitor {
34 loop_unroll_visitor(loop_state *state,
35 const struct gl_shader_compiler_options *options)
38 this->progress = false;
39 this->options = options;
42 virtual ir_visitor_status visit_leave(ir_loop *ir);
43 void simple_unroll(ir_loop *ir, int iterations);
44 void complex_unroll(ir_loop *ir, int iterations,
45 bool continue_from_then_branch,
46 bool limiting_term_first,
47 bool lt_continue_from_then_branch);
48 void splice_post_if_instructions(ir_if *ir_if, exec_list *splice_dest);
53 const struct gl_shader_compiler_options *options;
56 } /* anonymous namespace */
58 class loop_unroll_count : public ir_hierarchical_visitor {
61 bool unsupported_variable_indexing;
62 bool array_indexed_by_induction_var_with_exact_iterations;
63 /* If there are nested loops, the node count will be inaccurate. */
66 loop_unroll_count(exec_list *list, loop_variable_state *ls,
67 const struct gl_shader_compiler_options *options)
68 : ls(ls), options(options)
72 unsupported_variable_indexing = false;
73 array_indexed_by_induction_var_with_exact_iterations = false;
78 virtual ir_visitor_status visit_enter(ir_assignment *)
81 return visit_continue;
84 virtual ir_visitor_status visit_enter(ir_expression *)
87 return visit_continue;
90 virtual ir_visitor_status visit_enter(ir_loop *)
93 return visit_continue;
96 virtual ir_visitor_status visit_enter(ir_dereference_array *ir)
98 /* Force unroll in case of dynamic indexing with sampler arrays
99 * when EmitNoIndirectSampler is set.
101 if (options->EmitNoIndirectSampler) {
102 if ((ir->array->type->is_array() &&
103 ir->array->type->contains_sampler()) &&
104 !ir->array_index->constant_expression_value(ralloc_parent(ir))) {
105 unsupported_variable_indexing = true;
106 return visit_continue;
110 /* Check for arrays variably-indexed by a loop induction variable.
111 * Unrolling the loop may convert that access into constant-indexing.
113 * Many drivers don't support particular kinds of variable indexing,
114 * and have to resort to using lower_variable_index_to_cond_assign to
115 * handle it. This results in huge amounts of horrible code, so we'd
116 * like to avoid that if possible. Here, we just note that it will
119 if ((ir->array->type->is_array() || ir->array->type->is_matrix()) &&
120 !ir->array_index->as_constant()) {
121 ir_variable *array = ir->array->variable_referenced();
122 loop_variable *lv = ls->get(ir->array_index->variable_referenced());
123 if (array && lv && lv->is_induction_var()) {
124 /* If an array is indexed by a loop induction variable, and the
125 * array size is exactly the number of loop iterations, this is
126 * probably a simple for-loop trying to access each element in
127 * turn; the application may expect it to be unrolled.
129 if (int(array->type->length) == ls->limiting_terminator->iterations)
130 array_indexed_by_induction_var_with_exact_iterations = true;
132 switch (array->data.mode) {
134 case ir_var_temporary:
135 case ir_var_const_in:
136 case ir_var_function_in:
137 case ir_var_function_out:
138 case ir_var_function_inout:
139 if (options->EmitNoIndirectTemp)
140 unsupported_variable_indexing = true;
143 case ir_var_shader_storage:
144 if (options->EmitNoIndirectUniform)
145 unsupported_variable_indexing = true;
147 case ir_var_shader_in:
148 if (options->EmitNoIndirectInput)
149 unsupported_variable_indexing = true;
151 case ir_var_shader_out:
152 if (options->EmitNoIndirectOutput)
153 unsupported_variable_indexing = true;
158 return visit_continue;
162 loop_variable_state *ls;
163 const struct gl_shader_compiler_options *options;
168 * Unroll a loop which does not contain any jumps. For example, if the input
171 * (loop (...) ...instrs...)
173 * And the iteration count is 3, the output will be:
175 * ...instrs... ...instrs... ...instrs...
178 loop_unroll_visitor::simple_unroll(ir_loop *ir, int iterations)
180 void *const mem_ctx = ralloc_parent(ir);
181 loop_variable_state *const ls = this->state->get(ir);
183 ir_instruction *first_ir =
184 (ir_instruction *) ir->body_instructions.get_head();
187 /* The loop is empty remove it and return */
192 ir_if *limit_if = NULL;
193 bool exit_branch_has_instructions = false;
194 if (ls->limiting_terminator) {
195 limit_if = ls->limiting_terminator->ir;
196 ir_instruction *ir_if_last = (ir_instruction *)
197 limit_if->then_instructions.get_tail();
199 if (is_break(ir_if_last)) {
200 if (ir_if_last != limit_if->then_instructions.get_head())
201 exit_branch_has_instructions = true;
203 splice_post_if_instructions(limit_if, &limit_if->else_instructions);
204 ir_if_last->remove();
206 ir_if_last = (ir_instruction *)
207 limit_if->else_instructions.get_tail();
208 assert(is_break(ir_if_last));
210 if (ir_if_last != limit_if->else_instructions.get_head())
211 exit_branch_has_instructions = true;
213 splice_post_if_instructions(limit_if, &limit_if->then_instructions);
214 ir_if_last->remove();
218 /* Because 'iterations' is the number of times we pass over the *entire*
219 * loop body before hitting the first break, we need to bump the number of
220 * iterations if the limiting terminator is not the first instruction in
221 * the loop, or it the exit branch contains instructions. This ensures we
222 * execute any instructions before the terminator or in its exit branch.
224 if (limit_if != first_ir->as_if() || exit_branch_has_instructions)
227 for (int i = 0; i < iterations; i++) {
230 copy_list.make_empty();
231 clone_ir_list(mem_ctx, ©_list, &ir->body_instructions);
233 ir->insert_before(©_list);
236 /* The loop has been replaced by the unrolled copies. Remove the original
237 * loop from the IR sequence.
241 this->progress = true;
246 * Unroll a loop whose last statement is an ir_if. If \c
247 * continue_from_then_branch is true, the loop is repeated only when the
248 * "then" branch of the if is taken; otherwise it is repeated only when the
249 * "else" branch of the if is taken.
251 * For example, if the input is:
256 * (...then_instrs...)
257 * (...else_instrs...)))
259 * And the iteration count is 3, and \c continue_from_then_branch is true,
260 * then the output will be:
270 * (...then_instrs...)
271 * (...else_instrs...)))
272 * (...else_instrs...)))
276 loop_unroll_visitor::complex_unroll(ir_loop *ir, int iterations,
277 bool second_term_then_continue,
278 bool extra_iteration_required,
279 bool first_term_then_continue)
281 void *const mem_ctx = ralloc_parent(ir);
282 ir_instruction *ir_to_replace = ir;
284 /* Because 'iterations' is the number of times we pass over the *entire*
285 * loop body before hitting the first break, we need to bump the number of
286 * iterations if the limiting terminator is not the first instruction in
287 * the loop, or it the exit branch contains instructions. This ensures we
288 * execute any instructions before the terminator or in its exit branch.
290 if (extra_iteration_required)
293 for (int i = 0; i < iterations; i++) {
296 copy_list.make_empty();
297 clone_ir_list(mem_ctx, ©_list, &ir->body_instructions);
299 ir_if *ir_if = ((ir_instruction *) copy_list.get_tail())->as_if();
300 assert(ir_if != NULL);
302 exec_list *const first_list = first_term_then_continue
303 ? &ir_if->then_instructions : &ir_if->else_instructions;
304 ir_if = ((ir_instruction *) first_list->get_tail())->as_if();
306 ir_to_replace->insert_before(©_list);
307 ir_to_replace->remove();
309 /* placeholder that will be removed in the next iteration */
311 new(mem_ctx) ir_loop_jump(ir_loop_jump::jump_continue);
313 exec_list *const second_term_continue_list = second_term_then_continue
314 ? &ir_if->then_instructions : &ir_if->else_instructions;
316 second_term_continue_list->push_tail(ir_to_replace);
319 ir_to_replace->remove();
321 this->progress = true;
326 * Move all of the instructions which follow \c ir_if to the end of
329 * For example, in the code snippet:
332 * (...then_instructions...
334 * (...else_instructions...))
335 * ...post_if_instructions...
337 * If \c ir_if points to the "if" instruction, and \c splice_dest points to
338 * (...else_instructions...), the code snippet is transformed into:
341 * (...then_instructions...
343 * (...else_instructions...
344 * ...post_if_instructions...))
347 loop_unroll_visitor::splice_post_if_instructions(ir_if *ir_if,
348 exec_list *splice_dest)
350 while (!ir_if->get_next()->is_tail_sentinel()) {
351 ir_instruction *move_ir = (ir_instruction *) ir_if->get_next();
354 splice_dest->push_tail(move_ir);
359 exit_branch_has_instructions(ir_if *term_if, bool lt_then_continue)
361 if (lt_then_continue) {
362 if (term_if->else_instructions.get_head() ==
363 term_if->else_instructions.get_tail())
366 if (term_if->then_instructions.get_head() ==
367 term_if->then_instructions.get_tail())
375 loop_unroll_visitor::visit_leave(ir_loop *ir)
377 loop_variable_state *const ls = this->state->get(ir);
379 /* If we've entered a loop that hasn't been analyzed, something really,
380 * really bad has happened.
384 return visit_continue;
387 if (ls->limiting_terminator != NULL) {
388 /* If the limiting terminator has an iteration count of zero, then we've
389 * proven that the loop cannot run, so delete it.
391 int iterations = ls->limiting_terminator->iterations;
392 if (iterations == 0) {
394 this->progress = true;
395 return visit_continue;
399 /* Remove the conditional break statements associated with all terminators
400 * that are associated with a fixed iteration count, except for the one
401 * associated with the limiting terminator--that one needs to stay, since
402 * it terminates the loop. Exception: if the loop still has a normative
403 * bound, then that terminates the loop, so we don't even need the limiting
406 foreach_in_list_safe(loop_terminator, t, &ls->terminators) {
407 if (t->iterations < 0)
410 exec_list *branch_instructions;
411 if (t != ls->limiting_terminator) {
412 ir_instruction *ir_if_last = (ir_instruction *)
413 t->ir->then_instructions.get_tail();
414 if (is_break(ir_if_last)) {
415 branch_instructions = &t->ir->else_instructions;
417 branch_instructions = &t->ir->then_instructions;
418 assert(is_break((ir_instruction *)
419 t->ir->else_instructions.get_tail()));
423 copy_list.make_empty();
424 clone_ir_list(ir, ©_list, branch_instructions);
426 t->ir->insert_before(©_list);
429 assert(ls->num_loop_jumps > 0);
430 ls->num_loop_jumps--;
432 /* Also remove it from the terminator list */
435 this->progress = true;
439 if (ls->limiting_terminator == NULL) {
440 ir_instruction *last_ir =
441 (ir_instruction *) ir->body_instructions.get_tail();
443 /* If a loop has no induction variable and the last instruction is
444 * a break, unroll the loop with a count of 1. This is the classic
450 * that is used to wrap multi-line macros.
452 * If num_loop_jumps is not zero, last_ir cannot be NULL... there has to
453 * be at least num_loop_jumps instructions in the loop.
455 if (ls->num_loop_jumps == 1 && is_break(last_ir)) {
458 simple_unroll(ir, 1);
461 /* Don't try to unroll loops where the number of iterations is not known
464 return visit_continue;
467 int iterations = ls->limiting_terminator->iterations;
469 const int max_iterations = options->MaxUnrollIterations;
471 /* Don't try to unroll loops that have zillions of iterations either.
473 if (iterations > max_iterations)
474 return visit_continue;
476 /* Don't try to unroll nested loops and loops with a huge body.
478 loop_unroll_count count(&ir->body_instructions, ls, options);
480 bool loop_too_large =
481 count.nested_loop || count.nodes * iterations > max_iterations * 5;
483 if (loop_too_large && !count.unsupported_variable_indexing &&
484 !count.array_indexed_by_induction_var_with_exact_iterations)
485 return visit_continue;
487 /* Note: the limiting terminator contributes 1 to ls->num_loop_jumps.
488 * We'll be removing the limiting terminator before we unroll.
490 assert(ls->num_loop_jumps > 0);
491 unsigned predicted_num_loop_jumps = ls->num_loop_jumps - 1;
493 if (predicted_num_loop_jumps > 1)
494 return visit_continue;
496 if (predicted_num_loop_jumps == 0) {
497 simple_unroll(ir, iterations);
498 return visit_continue;
501 ir_instruction *last_ir = (ir_instruction *) ir->body_instructions.get_tail();
502 assert(last_ir != NULL);
504 if (is_break(last_ir)) {
505 /* If the only loop-jump is a break at the end of the loop, the loop
506 * will execute exactly once. Remove the break and use the simple
507 * unroller with an iteration count of 1.
511 simple_unroll(ir, 1);
512 return visit_continue;
515 /* Complex unrolling can only handle two terminators. One with an unknown
516 * iteration count and one with a known iteration count. We have already
517 * made sure we have a known iteration count above and removed any
518 * unreachable terminators with a known count. Here we make sure there
519 * isn't any additional unknown terminators, or any other jumps nested
522 if (ls->num_loop_jumps != 2 || ls->terminators.length() != 2)
523 return visit_continue;
525 ir_instruction *first_ir =
526 (ir_instruction *) ir->body_instructions.get_head();
528 unsigned term_count = 0;
529 bool first_term_then_continue = false;
530 foreach_in_list(loop_terminator, t, &ls->terminators) {
531 ir_if *ir_if = t->ir->as_if();
532 assert(ir_if != NULL);
534 ir_instruction *ir_if_last =
535 (ir_instruction *) ir_if->then_instructions.get_tail();
537 if (is_break(ir_if_last)) {
538 splice_post_if_instructions(ir_if, &ir_if->else_instructions);
539 ir_if_last->remove();
540 if (term_count == 1) {
542 exit_branch_has_instructions(ls->limiting_terminator->ir,
543 first_term_then_continue);
544 complex_unroll(ir, iterations, false,
545 first_ir->as_if() != ls->limiting_terminator->ir ||
547 first_term_then_continue);
548 return visit_continue;
552 (ir_instruction *) ir_if->else_instructions.get_tail();
554 assert(is_break(ir_if_last));
555 if (is_break(ir_if_last)) {
556 splice_post_if_instructions(ir_if, &ir_if->then_instructions);
557 ir_if_last->remove();
558 if (term_count == 1) {
560 exit_branch_has_instructions(ls->limiting_terminator->ir,
561 first_term_then_continue);
562 complex_unroll(ir, iterations, true,
563 first_ir->as_if() != ls->limiting_terminator->ir ||
565 first_term_then_continue);
566 return visit_continue;
568 first_term_then_continue = true;
576 /* Did not find the break statement. It must be in a complex if-nesting,
577 * so don't try to unroll.
579 return visit_continue;
584 unroll_loops(exec_list *instructions, loop_state *ls,
585 const struct gl_shader_compiler_options *options)
587 loop_unroll_visitor v(ls, options);