2 * Tiny Code Generator for QEMU
4 * Copyright (c) 2018 Linaro, Inc.
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2.1 of the License, or (at your option) any later version.
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
20 #include "qemu/osdep.h"
23 #include "tcg/tcg-op.h"
24 #include "tcg/tcg-mo.h"
26 /* Reduce the number of ifdefs below. This assumes that all uses of
27 TCGV_HIGH and TCGV_LOW are properly protected by a conditional that
28 the compiler can eliminate. */
29 #if TCG_TARGET_REG_BITS == 64
30 extern TCGv_i32 TCGV_LOW_link_error(TCGv_i64);
31 extern TCGv_i32 TCGV_HIGH_link_error(TCGv_i64);
32 #define TCGV_LOW TCGV_LOW_link_error
33 #define TCGV_HIGH TCGV_HIGH_link_error
37 * Vector optional opcode tracking.
38 * Except for the basic logical operations (and, or, xor), and
39 * data movement (mov, ld, st, dupi), many vector opcodes are
40 * optional and may not be supported on the host. Thank Intel
41 * for the irregularity in their instruction set.
43 * The gvec expanders allow custom vector operations to be composed,
44 * generally via the .fniv callback in the GVecGen* structures. At
45 * the same time, in deciding whether to use this hook we need to
46 * know if the host supports the required operations. This is
47 * presented as an array of opcodes, terminated by 0. Each opcode
48 * is assumed to be expanded with the given VECE.
50 * For debugging, we want to validate this array. Therefore, when
51 * tcg_ctx->vec_opt_opc is non-NULL, the tcg_gen_*_vec expanders
52 * will validate that their opcode is present in the list.
54 #ifdef CONFIG_DEBUG_TCG
55 void tcg_assert_listed_vecop(TCGOpcode op)
57 const TCGOpcode *p = tcg_ctx->vecop_list;
64 g_assert_not_reached();
69 bool tcg_can_emit_vecop_list(const TCGOpcode *list,
70 TCGType type, unsigned vece)
76 for (; *list; ++list) {
77 TCGOpcode opc = *list;
79 #ifdef CONFIG_DEBUG_TCG
81 case INDEX_op_and_vec:
83 case INDEX_op_xor_vec:
84 case INDEX_op_mov_vec:
85 case INDEX_op_dup_vec:
86 case INDEX_op_dupi_vec:
87 case INDEX_op_dup2_vec:
90 case INDEX_op_bitsel_vec:
91 /* These opcodes are mandatory and should not be listed. */
92 g_assert_not_reached();
93 case INDEX_op_not_vec:
94 /* These opcodes have generic expansions using the above. */
95 g_assert_not_reached();
101 if (tcg_can_emit_vec_op(opc, type, vece)) {
106 * The opcode list is created by front ends based on what they
107 * actually invoke. We must mirror the logic in the routines
108 * below for generic expansions using other opcodes.
111 case INDEX_op_neg_vec:
112 if (tcg_can_emit_vec_op(INDEX_op_sub_vec, type, vece)) {
116 case INDEX_op_abs_vec:
117 if (tcg_can_emit_vec_op(INDEX_op_sub_vec, type, vece)
118 && (tcg_can_emit_vec_op(INDEX_op_smax_vec, type, vece) > 0
119 || tcg_can_emit_vec_op(INDEX_op_sari_vec, type, vece) > 0
120 || tcg_can_emit_vec_op(INDEX_op_cmp_vec, type, vece))) {
124 case INDEX_op_cmpsel_vec:
125 case INDEX_op_smin_vec:
126 case INDEX_op_smax_vec:
127 case INDEX_op_umin_vec:
128 case INDEX_op_umax_vec:
129 if (tcg_can_emit_vec_op(INDEX_op_cmp_vec, type, vece)) {
141 void vec_gen_2(TCGOpcode opc, TCGType type, unsigned vece, TCGArg r, TCGArg a)
143 TCGOp *op = tcg_emit_op(opc);
144 TCGOP_VECL(op) = type - TCG_TYPE_V64;
145 TCGOP_VECE(op) = vece;
150 void vec_gen_3(TCGOpcode opc, TCGType type, unsigned vece,
151 TCGArg r, TCGArg a, TCGArg b)
153 TCGOp *op = tcg_emit_op(opc);
154 TCGOP_VECL(op) = type - TCG_TYPE_V64;
155 TCGOP_VECE(op) = vece;
161 void vec_gen_4(TCGOpcode opc, TCGType type, unsigned vece,
162 TCGArg r, TCGArg a, TCGArg b, TCGArg c)
164 TCGOp *op = tcg_emit_op(opc);
165 TCGOP_VECL(op) = type - TCG_TYPE_V64;
166 TCGOP_VECE(op) = vece;
173 static void vec_gen_6(TCGOpcode opc, TCGType type, unsigned vece, TCGArg r,
174 TCGArg a, TCGArg b, TCGArg c, TCGArg d, TCGArg e)
176 TCGOp *op = tcg_emit_op(opc);
177 TCGOP_VECL(op) = type - TCG_TYPE_V64;
178 TCGOP_VECE(op) = vece;
187 static void vec_gen_op2(TCGOpcode opc, unsigned vece, TCGv_vec r, TCGv_vec a)
189 TCGTemp *rt = tcgv_vec_temp(r);
190 TCGTemp *at = tcgv_vec_temp(a);
191 TCGType type = rt->base_type;
193 /* Must enough inputs for the output. */
194 tcg_debug_assert(at->base_type >= type);
195 vec_gen_2(opc, type, vece, temp_arg(rt), temp_arg(at));
198 static void vec_gen_op3(TCGOpcode opc, unsigned vece,
199 TCGv_vec r, TCGv_vec a, TCGv_vec b)
201 TCGTemp *rt = tcgv_vec_temp(r);
202 TCGTemp *at = tcgv_vec_temp(a);
203 TCGTemp *bt = tcgv_vec_temp(b);
204 TCGType type = rt->base_type;
206 /* Must enough inputs for the output. */
207 tcg_debug_assert(at->base_type >= type);
208 tcg_debug_assert(bt->base_type >= type);
209 vec_gen_3(opc, type, vece, temp_arg(rt), temp_arg(at), temp_arg(bt));
212 void tcg_gen_mov_vec(TCGv_vec r, TCGv_vec a)
215 vec_gen_op2(INDEX_op_mov_vec, 0, r, a);
219 #define MO_REG (TCG_TARGET_REG_BITS == 64 ? MO_64 : MO_32)
221 static void do_dupi_vec(TCGv_vec r, unsigned vece, TCGArg a)
223 TCGTemp *rt = tcgv_vec_temp(r);
224 vec_gen_2(INDEX_op_dupi_vec, rt->base_type, vece, temp_arg(rt), a);
227 TCGv_vec tcg_const_zeros_vec(TCGType type)
229 TCGv_vec ret = tcg_temp_new_vec(type);
230 do_dupi_vec(ret, MO_REG, 0);
234 TCGv_vec tcg_const_ones_vec(TCGType type)
236 TCGv_vec ret = tcg_temp_new_vec(type);
237 do_dupi_vec(ret, MO_REG, -1);
241 TCGv_vec tcg_const_zeros_vec_matching(TCGv_vec m)
243 TCGTemp *t = tcgv_vec_temp(m);
244 return tcg_const_zeros_vec(t->base_type);
247 TCGv_vec tcg_const_ones_vec_matching(TCGv_vec m)
249 TCGTemp *t = tcgv_vec_temp(m);
250 return tcg_const_ones_vec(t->base_type);
253 void tcg_gen_dup64i_vec(TCGv_vec r, uint64_t a)
255 if (TCG_TARGET_REG_BITS == 64) {
256 do_dupi_vec(r, MO_64, a);
257 } else if (a == dup_const(MO_32, a)) {
258 do_dupi_vec(r, MO_32, a);
260 TCGv_i64 c = tcg_const_i64(a);
261 tcg_gen_dup_i64_vec(MO_64, r, c);
262 tcg_temp_free_i64(c);
266 void tcg_gen_dup32i_vec(TCGv_vec r, uint32_t a)
268 do_dupi_vec(r, MO_REG, dup_const(MO_32, a));
271 void tcg_gen_dup16i_vec(TCGv_vec r, uint32_t a)
273 do_dupi_vec(r, MO_REG, dup_const(MO_16, a));
276 void tcg_gen_dup8i_vec(TCGv_vec r, uint32_t a)
278 do_dupi_vec(r, MO_REG, dup_const(MO_8, a));
281 void tcg_gen_dupi_vec(unsigned vece, TCGv_vec r, uint64_t a)
284 tcg_gen_dup64i_vec(r, a);
286 do_dupi_vec(r, MO_REG, dup_const(vece, a));
290 void tcg_gen_dup_i64_vec(unsigned vece, TCGv_vec r, TCGv_i64 a)
292 TCGArg ri = tcgv_vec_arg(r);
293 TCGTemp *rt = arg_temp(ri);
294 TCGType type = rt->base_type;
296 if (TCG_TARGET_REG_BITS == 64) {
297 TCGArg ai = tcgv_i64_arg(a);
298 vec_gen_2(INDEX_op_dup_vec, type, vece, ri, ai);
299 } else if (vece == MO_64) {
300 TCGArg al = tcgv_i32_arg(TCGV_LOW(a));
301 TCGArg ah = tcgv_i32_arg(TCGV_HIGH(a));
302 vec_gen_3(INDEX_op_dup2_vec, type, MO_64, ri, al, ah);
304 TCGArg ai = tcgv_i32_arg(TCGV_LOW(a));
305 vec_gen_2(INDEX_op_dup_vec, type, vece, ri, ai);
309 void tcg_gen_dup_i32_vec(unsigned vece, TCGv_vec r, TCGv_i32 a)
311 TCGArg ri = tcgv_vec_arg(r);
312 TCGArg ai = tcgv_i32_arg(a);
313 TCGTemp *rt = arg_temp(ri);
314 TCGType type = rt->base_type;
316 vec_gen_2(INDEX_op_dup_vec, type, vece, ri, ai);
319 void tcg_gen_dup_mem_vec(unsigned vece, TCGv_vec r, TCGv_ptr b,
322 TCGArg ri = tcgv_vec_arg(r);
323 TCGArg bi = tcgv_ptr_arg(b);
324 TCGTemp *rt = arg_temp(ri);
325 TCGType type = rt->base_type;
327 vec_gen_3(INDEX_op_dupm_vec, type, vece, ri, bi, ofs);
330 static void vec_gen_ldst(TCGOpcode opc, TCGv_vec r, TCGv_ptr b, TCGArg o)
332 TCGArg ri = tcgv_vec_arg(r);
333 TCGArg bi = tcgv_ptr_arg(b);
334 TCGTemp *rt = arg_temp(ri);
335 TCGType type = rt->base_type;
337 vec_gen_3(opc, type, 0, ri, bi, o);
340 void tcg_gen_ld_vec(TCGv_vec r, TCGv_ptr b, TCGArg o)
342 vec_gen_ldst(INDEX_op_ld_vec, r, b, o);
345 void tcg_gen_st_vec(TCGv_vec r, TCGv_ptr b, TCGArg o)
347 vec_gen_ldst(INDEX_op_st_vec, r, b, o);
350 void tcg_gen_stl_vec(TCGv_vec r, TCGv_ptr b, TCGArg o, TCGType low_type)
352 TCGArg ri = tcgv_vec_arg(r);
353 TCGArg bi = tcgv_ptr_arg(b);
354 TCGTemp *rt = arg_temp(ri);
355 TCGType type = rt->base_type;
357 tcg_debug_assert(low_type >= TCG_TYPE_V64);
358 tcg_debug_assert(low_type <= type);
359 vec_gen_3(INDEX_op_st_vec, low_type, 0, ri, bi, o);
362 void tcg_gen_and_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
364 vec_gen_op3(INDEX_op_and_vec, 0, r, a, b);
367 void tcg_gen_or_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
369 vec_gen_op3(INDEX_op_or_vec, 0, r, a, b);
372 void tcg_gen_xor_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
374 vec_gen_op3(INDEX_op_xor_vec, 0, r, a, b);
377 void tcg_gen_andc_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
379 if (TCG_TARGET_HAS_andc_vec) {
380 vec_gen_op3(INDEX_op_andc_vec, 0, r, a, b);
382 TCGv_vec t = tcg_temp_new_vec_matching(r);
383 tcg_gen_not_vec(0, t, b);
384 tcg_gen_and_vec(0, r, a, t);
385 tcg_temp_free_vec(t);
389 void tcg_gen_orc_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
391 if (TCG_TARGET_HAS_orc_vec) {
392 vec_gen_op3(INDEX_op_orc_vec, 0, r, a, b);
394 TCGv_vec t = tcg_temp_new_vec_matching(r);
395 tcg_gen_not_vec(0, t, b);
396 tcg_gen_or_vec(0, r, a, t);
397 tcg_temp_free_vec(t);
401 void tcg_gen_nand_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
403 /* TODO: Add TCG_TARGET_HAS_nand_vec when adding a backend supports it. */
404 tcg_gen_and_vec(0, r, a, b);
405 tcg_gen_not_vec(0, r, r);
408 void tcg_gen_nor_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
410 /* TODO: Add TCG_TARGET_HAS_nor_vec when adding a backend supports it. */
411 tcg_gen_or_vec(0, r, a, b);
412 tcg_gen_not_vec(0, r, r);
415 void tcg_gen_eqv_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
417 /* TODO: Add TCG_TARGET_HAS_eqv_vec when adding a backend supports it. */
418 tcg_gen_xor_vec(0, r, a, b);
419 tcg_gen_not_vec(0, r, r);
422 static bool do_op2(unsigned vece, TCGv_vec r, TCGv_vec a, TCGOpcode opc)
424 TCGTemp *rt = tcgv_vec_temp(r);
425 TCGTemp *at = tcgv_vec_temp(a);
426 TCGArg ri = temp_arg(rt);
427 TCGArg ai = temp_arg(at);
428 TCGType type = rt->base_type;
431 tcg_debug_assert(at->base_type >= type);
432 tcg_assert_listed_vecop(opc);
433 can = tcg_can_emit_vec_op(opc, type, vece);
435 vec_gen_2(opc, type, vece, ri, ai);
436 } else if (can < 0) {
437 const TCGOpcode *hold_list = tcg_swap_vecop_list(NULL);
438 tcg_expand_vec_op(opc, type, vece, ri, ai);
439 tcg_swap_vecop_list(hold_list);
446 void tcg_gen_not_vec(unsigned vece, TCGv_vec r, TCGv_vec a)
448 const TCGOpcode *hold_list = tcg_swap_vecop_list(NULL);
450 if (!TCG_TARGET_HAS_not_vec || !do_op2(vece, r, a, INDEX_op_not_vec)) {
451 TCGv_vec t = tcg_const_ones_vec_matching(r);
452 tcg_gen_xor_vec(0, r, a, t);
453 tcg_temp_free_vec(t);
455 tcg_swap_vecop_list(hold_list);
458 void tcg_gen_neg_vec(unsigned vece, TCGv_vec r, TCGv_vec a)
460 const TCGOpcode *hold_list;
462 tcg_assert_listed_vecop(INDEX_op_neg_vec);
463 hold_list = tcg_swap_vecop_list(NULL);
465 if (!TCG_TARGET_HAS_neg_vec || !do_op2(vece, r, a, INDEX_op_neg_vec)) {
466 TCGv_vec t = tcg_const_zeros_vec_matching(r);
467 tcg_gen_sub_vec(vece, r, t, a);
468 tcg_temp_free_vec(t);
470 tcg_swap_vecop_list(hold_list);
473 void tcg_gen_abs_vec(unsigned vece, TCGv_vec r, TCGv_vec a)
475 const TCGOpcode *hold_list;
477 tcg_assert_listed_vecop(INDEX_op_abs_vec);
478 hold_list = tcg_swap_vecop_list(NULL);
480 if (!do_op2(vece, r, a, INDEX_op_abs_vec)) {
481 TCGType type = tcgv_vec_temp(r)->base_type;
482 TCGv_vec t = tcg_temp_new_vec(type);
484 tcg_debug_assert(tcg_can_emit_vec_op(INDEX_op_sub_vec, type, vece));
485 if (tcg_can_emit_vec_op(INDEX_op_smax_vec, type, vece) > 0) {
486 tcg_gen_neg_vec(vece, t, a);
487 tcg_gen_smax_vec(vece, r, a, t);
489 if (tcg_can_emit_vec_op(INDEX_op_sari_vec, type, vece) > 0) {
490 tcg_gen_sari_vec(vece, t, a, (8 << vece) - 1);
492 do_dupi_vec(t, MO_REG, 0);
493 tcg_gen_cmp_vec(TCG_COND_LT, vece, t, a, t);
495 tcg_gen_xor_vec(vece, r, a, t);
496 tcg_gen_sub_vec(vece, r, r, t);
499 tcg_temp_free_vec(t);
501 tcg_swap_vecop_list(hold_list);
504 static void do_shifti(TCGOpcode opc, unsigned vece,
505 TCGv_vec r, TCGv_vec a, int64_t i)
507 TCGTemp *rt = tcgv_vec_temp(r);
508 TCGTemp *at = tcgv_vec_temp(a);
509 TCGArg ri = temp_arg(rt);
510 TCGArg ai = temp_arg(at);
511 TCGType type = rt->base_type;
514 tcg_debug_assert(at->base_type == type);
515 tcg_debug_assert(i >= 0 && i < (8 << vece));
516 tcg_assert_listed_vecop(opc);
519 tcg_gen_mov_vec(r, a);
523 can = tcg_can_emit_vec_op(opc, type, vece);
525 vec_gen_3(opc, type, vece, ri, ai, i);
527 /* We leave the choice of expansion via scalar or vector shift
528 to the target. Often, but not always, dupi can feed a vector
529 shift easier than a scalar. */
530 const TCGOpcode *hold_list = tcg_swap_vecop_list(NULL);
531 tcg_debug_assert(can < 0);
532 tcg_expand_vec_op(opc, type, vece, ri, ai, i);
533 tcg_swap_vecop_list(hold_list);
537 void tcg_gen_shli_vec(unsigned vece, TCGv_vec r, TCGv_vec a, int64_t i)
539 do_shifti(INDEX_op_shli_vec, vece, r, a, i);
542 void tcg_gen_shri_vec(unsigned vece, TCGv_vec r, TCGv_vec a, int64_t i)
544 do_shifti(INDEX_op_shri_vec, vece, r, a, i);
547 void tcg_gen_sari_vec(unsigned vece, TCGv_vec r, TCGv_vec a, int64_t i)
549 do_shifti(INDEX_op_sari_vec, vece, r, a, i);
552 void tcg_gen_rotli_vec(unsigned vece, TCGv_vec r, TCGv_vec a, int64_t i)
554 do_shifti(INDEX_op_rotli_vec, vece, r, a, i);
557 void tcg_gen_rotri_vec(unsigned vece, TCGv_vec r, TCGv_vec a, int64_t i)
559 int bits = 8 << vece;
560 tcg_debug_assert(i >= 0 && i < bits);
561 do_shifti(INDEX_op_rotli_vec, vece, r, a, -i & (bits - 1));
564 void tcg_gen_cmp_vec(TCGCond cond, unsigned vece,
565 TCGv_vec r, TCGv_vec a, TCGv_vec b)
567 TCGTemp *rt = tcgv_vec_temp(r);
568 TCGTemp *at = tcgv_vec_temp(a);
569 TCGTemp *bt = tcgv_vec_temp(b);
570 TCGArg ri = temp_arg(rt);
571 TCGArg ai = temp_arg(at);
572 TCGArg bi = temp_arg(bt);
573 TCGType type = rt->base_type;
576 tcg_debug_assert(at->base_type >= type);
577 tcg_debug_assert(bt->base_type >= type);
578 tcg_assert_listed_vecop(INDEX_op_cmp_vec);
579 can = tcg_can_emit_vec_op(INDEX_op_cmp_vec, type, vece);
581 vec_gen_4(INDEX_op_cmp_vec, type, vece, ri, ai, bi, cond);
583 const TCGOpcode *hold_list = tcg_swap_vecop_list(NULL);
584 tcg_debug_assert(can < 0);
585 tcg_expand_vec_op(INDEX_op_cmp_vec, type, vece, ri, ai, bi, cond);
586 tcg_swap_vecop_list(hold_list);
590 static bool do_op3(unsigned vece, TCGv_vec r, TCGv_vec a,
591 TCGv_vec b, TCGOpcode opc)
593 TCGTemp *rt = tcgv_vec_temp(r);
594 TCGTemp *at = tcgv_vec_temp(a);
595 TCGTemp *bt = tcgv_vec_temp(b);
596 TCGArg ri = temp_arg(rt);
597 TCGArg ai = temp_arg(at);
598 TCGArg bi = temp_arg(bt);
599 TCGType type = rt->base_type;
602 tcg_debug_assert(at->base_type >= type);
603 tcg_debug_assert(bt->base_type >= type);
604 tcg_assert_listed_vecop(opc);
605 can = tcg_can_emit_vec_op(opc, type, vece);
607 vec_gen_3(opc, type, vece, ri, ai, bi);
608 } else if (can < 0) {
609 const TCGOpcode *hold_list = tcg_swap_vecop_list(NULL);
610 tcg_expand_vec_op(opc, type, vece, ri, ai, bi);
611 tcg_swap_vecop_list(hold_list);
618 static void do_op3_nofail(unsigned vece, TCGv_vec r, TCGv_vec a,
619 TCGv_vec b, TCGOpcode opc)
621 bool ok = do_op3(vece, r, a, b, opc);
622 tcg_debug_assert(ok);
625 void tcg_gen_add_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
627 do_op3_nofail(vece, r, a, b, INDEX_op_add_vec);
630 void tcg_gen_sub_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
632 do_op3_nofail(vece, r, a, b, INDEX_op_sub_vec);
635 void tcg_gen_mul_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
637 do_op3_nofail(vece, r, a, b, INDEX_op_mul_vec);
640 void tcg_gen_ssadd_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
642 do_op3_nofail(vece, r, a, b, INDEX_op_ssadd_vec);
645 void tcg_gen_usadd_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
647 do_op3_nofail(vece, r, a, b, INDEX_op_usadd_vec);
650 void tcg_gen_sssub_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
652 do_op3_nofail(vece, r, a, b, INDEX_op_sssub_vec);
655 void tcg_gen_ussub_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
657 do_op3_nofail(vece, r, a, b, INDEX_op_ussub_vec);
660 static void do_minmax(unsigned vece, TCGv_vec r, TCGv_vec a,
661 TCGv_vec b, TCGOpcode opc, TCGCond cond)
663 if (!do_op3(vece, r, a, b, opc)) {
664 const TCGOpcode *hold_list = tcg_swap_vecop_list(NULL);
665 tcg_gen_cmpsel_vec(cond, vece, r, a, b, a, b);
666 tcg_swap_vecop_list(hold_list);
670 void tcg_gen_smin_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
672 do_minmax(vece, r, a, b, INDEX_op_smin_vec, TCG_COND_LT);
675 void tcg_gen_umin_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
677 do_minmax(vece, r, a, b, INDEX_op_umin_vec, TCG_COND_LTU);
680 void tcg_gen_smax_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
682 do_minmax(vece, r, a, b, INDEX_op_smax_vec, TCG_COND_GT);
685 void tcg_gen_umax_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
687 do_minmax(vece, r, a, b, INDEX_op_umax_vec, TCG_COND_GTU);
690 void tcg_gen_shlv_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
692 do_op3_nofail(vece, r, a, b, INDEX_op_shlv_vec);
695 void tcg_gen_shrv_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
697 do_op3_nofail(vece, r, a, b, INDEX_op_shrv_vec);
700 void tcg_gen_sarv_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
702 do_op3_nofail(vece, r, a, b, INDEX_op_sarv_vec);
705 void tcg_gen_rotlv_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
707 do_op3_nofail(vece, r, a, b, INDEX_op_rotlv_vec);
710 void tcg_gen_rotrv_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
712 do_op3_nofail(vece, r, a, b, INDEX_op_rotrv_vec);
715 static void do_shifts(unsigned vece, TCGv_vec r, TCGv_vec a,
716 TCGv_i32 s, TCGOpcode opc)
718 TCGTemp *rt = tcgv_vec_temp(r);
719 TCGTemp *at = tcgv_vec_temp(a);
720 TCGTemp *st = tcgv_i32_temp(s);
721 TCGArg ri = temp_arg(rt);
722 TCGArg ai = temp_arg(at);
723 TCGArg si = temp_arg(st);
724 TCGType type = rt->base_type;
727 tcg_debug_assert(at->base_type >= type);
728 tcg_assert_listed_vecop(opc);
729 can = tcg_can_emit_vec_op(opc, type, vece);
731 vec_gen_3(opc, type, vece, ri, ai, si);
732 } else if (can < 0) {
733 const TCGOpcode *hold_list = tcg_swap_vecop_list(NULL);
734 tcg_expand_vec_op(opc, type, vece, ri, ai, si);
735 tcg_swap_vecop_list(hold_list);
737 g_assert_not_reached();
741 void tcg_gen_shls_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_i32 b)
743 do_shifts(vece, r, a, b, INDEX_op_shls_vec);
746 void tcg_gen_shrs_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_i32 b)
748 do_shifts(vece, r, a, b, INDEX_op_shrs_vec);
751 void tcg_gen_sars_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_i32 b)
753 do_shifts(vece, r, a, b, INDEX_op_sars_vec);
756 void tcg_gen_rotls_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_i32 s)
758 do_shifts(vece, r, a, s, INDEX_op_rotls_vec);
761 void tcg_gen_bitsel_vec(unsigned vece, TCGv_vec r, TCGv_vec a,
762 TCGv_vec b, TCGv_vec c)
764 TCGTemp *rt = tcgv_vec_temp(r);
765 TCGTemp *at = tcgv_vec_temp(a);
766 TCGTemp *bt = tcgv_vec_temp(b);
767 TCGTemp *ct = tcgv_vec_temp(c);
768 TCGType type = rt->base_type;
770 tcg_debug_assert(at->base_type >= type);
771 tcg_debug_assert(bt->base_type >= type);
772 tcg_debug_assert(ct->base_type >= type);
774 if (TCG_TARGET_HAS_bitsel_vec) {
775 vec_gen_4(INDEX_op_bitsel_vec, type, MO_8,
776 temp_arg(rt), temp_arg(at), temp_arg(bt), temp_arg(ct));
778 TCGv_vec t = tcg_temp_new_vec(type);
779 tcg_gen_and_vec(MO_8, t, a, b);
780 tcg_gen_andc_vec(MO_8, r, c, a);
781 tcg_gen_or_vec(MO_8, r, r, t);
782 tcg_temp_free_vec(t);
786 void tcg_gen_cmpsel_vec(TCGCond cond, unsigned vece, TCGv_vec r,
787 TCGv_vec a, TCGv_vec b, TCGv_vec c, TCGv_vec d)
789 TCGTemp *rt = tcgv_vec_temp(r);
790 TCGTemp *at = tcgv_vec_temp(a);
791 TCGTemp *bt = tcgv_vec_temp(b);
792 TCGTemp *ct = tcgv_vec_temp(c);
793 TCGTemp *dt = tcgv_vec_temp(d);
794 TCGArg ri = temp_arg(rt);
795 TCGArg ai = temp_arg(at);
796 TCGArg bi = temp_arg(bt);
797 TCGArg ci = temp_arg(ct);
798 TCGArg di = temp_arg(dt);
799 TCGType type = rt->base_type;
800 const TCGOpcode *hold_list;
803 tcg_debug_assert(at->base_type >= type);
804 tcg_debug_assert(bt->base_type >= type);
805 tcg_debug_assert(ct->base_type >= type);
806 tcg_debug_assert(dt->base_type >= type);
808 tcg_assert_listed_vecop(INDEX_op_cmpsel_vec);
809 hold_list = tcg_swap_vecop_list(NULL);
810 can = tcg_can_emit_vec_op(INDEX_op_cmpsel_vec, type, vece);
813 vec_gen_6(INDEX_op_cmpsel_vec, type, vece, ri, ai, bi, ci, di, cond);
814 } else if (can < 0) {
815 tcg_expand_vec_op(INDEX_op_cmpsel_vec, type, vece,
816 ri, ai, bi, ci, di, cond);
818 TCGv_vec t = tcg_temp_new_vec(type);
819 tcg_gen_cmp_vec(cond, vece, t, a, b);
820 tcg_gen_bitsel_vec(vece, r, t, c, d);
821 tcg_temp_free_vec(t);
823 tcg_swap_vecop_list(hold_list);