2 * QEMU TCG support -- s390x vector integer instruction support
4 * Copyright (C) 2019 Red Hat Inc
7 * David Hildenbrand <david@redhat.com>
9 * This work is licensed under the terms of the GNU GPL, version 2 or later.
10 * See the COPYING file in the top-level directory.
12 #include "qemu/osdep.h"
15 #include "exec/helper-proto.h"
16 #include "tcg/tcg-gvec-desc.h"
17 #include "crypto/clmul.h"
19 static bool s390_vec_is_zero(const S390Vector *v)
21 return !v->doubleword[0] && !v->doubleword[1];
24 static void s390_vec_xor(S390Vector *res, const S390Vector *a,
27 res->doubleword[0] = a->doubleword[0] ^ b->doubleword[0];
28 res->doubleword[1] = a->doubleword[1] ^ b->doubleword[1];
31 static void s390_vec_and(S390Vector *res, const S390Vector *a,
34 res->doubleword[0] = a->doubleword[0] & b->doubleword[0];
35 res->doubleword[1] = a->doubleword[1] & b->doubleword[1];
38 static bool s390_vec_equal(const S390Vector *a, const S390Vector *b)
40 return a->doubleword[0] == b->doubleword[0] &&
41 a->doubleword[1] == b->doubleword[1];
44 static void s390_vec_shl(S390Vector *d, const S390Vector *a, uint64_t count)
48 g_assert(count < 128);
50 d->doubleword[0] = a->doubleword[0];
51 d->doubleword[1] = a->doubleword[1];
52 } else if (count == 64) {
53 d->doubleword[0] = a->doubleword[1];
55 } else if (count < 64) {
56 tmp = extract64(a->doubleword[1], 64 - count, count);
57 d->doubleword[1] = a->doubleword[1] << count;
58 d->doubleword[0] = (a->doubleword[0] << count) | tmp;
60 d->doubleword[0] = a->doubleword[1] << (count - 64);
65 static void s390_vec_sar(S390Vector *d, const S390Vector *a, uint64_t count)
70 d->doubleword[0] = a->doubleword[0];
71 d->doubleword[1] = a->doubleword[1];
72 } else if (count == 64) {
73 tmp = (int64_t)a->doubleword[0] >> 63;
74 d->doubleword[1] = a->doubleword[0];
75 d->doubleword[0] = tmp;
76 } else if (count < 64) {
77 tmp = a->doubleword[1] >> count;
78 d->doubleword[1] = deposit64(tmp, 64 - count, count, a->doubleword[0]);
79 d->doubleword[0] = (int64_t)a->doubleword[0] >> count;
81 tmp = (int64_t)a->doubleword[0] >> 63;
82 d->doubleword[1] = (int64_t)a->doubleword[0] >> (count - 64);
83 d->doubleword[0] = tmp;
87 static void s390_vec_shr(S390Vector *d, const S390Vector *a, uint64_t count)
91 g_assert(count < 128);
93 d->doubleword[0] = a->doubleword[0];
94 d->doubleword[1] = a->doubleword[1];
95 } else if (count == 64) {
96 d->doubleword[1] = a->doubleword[0];
98 } else if (count < 64) {
99 tmp = a->doubleword[1] >> count;
100 d->doubleword[1] = deposit64(tmp, 64 - count, count, a->doubleword[0]);
101 d->doubleword[0] = a->doubleword[0] >> count;
103 d->doubleword[1] = a->doubleword[0] >> (count - 64);
104 d->doubleword[0] = 0;
107 #define DEF_VAVG(BITS) \
108 void HELPER(gvec_vavg##BITS)(void *v1, const void *v2, const void *v3, \
113 for (i = 0; i < (128 / BITS); i++) { \
114 const int32_t a = (int##BITS##_t)s390_vec_read_element##BITS(v2, i); \
115 const int32_t b = (int##BITS##_t)s390_vec_read_element##BITS(v3, i); \
117 s390_vec_write_element##BITS(v1, i, (a + b + 1) >> 1); \
123 #define DEF_VAVGL(BITS) \
124 void HELPER(gvec_vavgl##BITS)(void *v1, const void *v2, const void *v3, \
129 for (i = 0; i < (128 / BITS); i++) { \
130 const uint##BITS##_t a = s390_vec_read_element##BITS(v2, i); \
131 const uint##BITS##_t b = s390_vec_read_element##BITS(v3, i); \
133 s390_vec_write_element##BITS(v1, i, (a + b + 1) >> 1); \
139 #define DEF_VCLZ(BITS) \
140 void HELPER(gvec_vclz##BITS)(void *v1, const void *v2, uint32_t desc) \
144 for (i = 0; i < (128 / BITS); i++) { \
145 const uint##BITS##_t a = s390_vec_read_element##BITS(v2, i); \
147 s390_vec_write_element##BITS(v1, i, clz32(a) - 32 + BITS); \
153 #define DEF_VCTZ(BITS) \
154 void HELPER(gvec_vctz##BITS)(void *v1, const void *v2, uint32_t desc) \
158 for (i = 0; i < (128 / BITS); i++) { \
159 const uint##BITS##_t a = s390_vec_read_element##BITS(v2, i); \
161 s390_vec_write_element##BITS(v1, i, a ? ctz32(a) : BITS); \
167 /* like binary multiplication, but XOR instead of addition */
168 #define DEF_GALOIS_MULTIPLY(BITS, TBITS) \
169 static uint##TBITS##_t galois_multiply##BITS(uint##TBITS##_t a, \
172 uint##TBITS##_t res = 0; \
183 DEF_GALOIS_MULTIPLY(32, 64)
185 static S390Vector galois_multiply64(uint64_t a, uint64_t b)
195 while (!s390_vec_is_zero(&vb)) {
196 if (vb.doubleword[1] & 0x1) {
197 s390_vec_xor(&res, &res, &va);
199 s390_vec_shl(&va, &va, 1);
200 s390_vec_shr(&vb, &vb, 1);
206 * There is no carry across the two doublewords, so their order does
207 * not matter. Nor is there partial overlap between registers.
209 static inline uint64_t do_gfma8(uint64_t n, uint64_t m, uint64_t a)
211 return clmul_8x4_even(n, m) ^ clmul_8x4_odd(n, m) ^ a;
214 void HELPER(gvec_vgfm8)(void *v1, const void *v2, const void *v3, uint32_t d)
217 const uint64_t *q2 = v2, *q3 = v3;
219 q1[0] = do_gfma8(q2[0], q3[0], 0);
220 q1[1] = do_gfma8(q2[1], q3[1], 0);
223 void HELPER(gvec_vgfma8)(void *v1, const void *v2, const void *v3,
224 const void *v4, uint32_t desc)
227 const uint64_t *q2 = v2, *q3 = v3, *q4 = v4;
229 q1[0] = do_gfma8(q2[0], q3[0], q4[0]);
230 q1[1] = do_gfma8(q2[1], q3[1], q4[1]);
233 static inline uint64_t do_gfma16(uint64_t n, uint64_t m, uint64_t a)
235 return clmul_16x2_even(n, m) ^ clmul_16x2_odd(n, m) ^ a;
238 void HELPER(gvec_vgfm16)(void *v1, const void *v2, const void *v3, uint32_t d)
241 const uint64_t *q2 = v2, *q3 = v3;
243 q1[0] = do_gfma16(q2[0], q3[0], 0);
244 q1[1] = do_gfma16(q2[1], q3[1], 0);
247 void HELPER(gvec_vgfma16)(void *v1, const void *v2, const void *v3,
248 const void *v4, uint32_t d)
251 const uint64_t *q2 = v2, *q3 = v3, *q4 = v4;
253 q1[0] = do_gfma16(q2[0], q3[0], q4[0]);
254 q1[1] = do_gfma16(q2[1], q3[1], q4[1]);
257 #define DEF_VGFM(BITS, TBITS) \
258 void HELPER(gvec_vgfm##BITS)(void *v1, const void *v2, const void *v3, \
263 for (i = 0; i < (128 / TBITS); i++) { \
264 uint##BITS##_t a = s390_vec_read_element##BITS(v2, i * 2); \
265 uint##BITS##_t b = s390_vec_read_element##BITS(v3, i * 2); \
266 uint##TBITS##_t d = galois_multiply##BITS(a, b); \
268 a = s390_vec_read_element##BITS(v2, i * 2 + 1); \
269 b = s390_vec_read_element##BITS(v3, i * 2 + 1); \
270 d = d ^ galois_multiply32(a, b); \
271 s390_vec_write_element##TBITS(v1, i, d); \
276 void HELPER(gvec_vgfm64)(void *v1, const void *v2, const void *v3,
279 S390Vector tmp1, tmp2;
282 a = s390_vec_read_element64(v2, 0);
283 b = s390_vec_read_element64(v3, 0);
284 tmp1 = galois_multiply64(a, b);
285 a = s390_vec_read_element64(v2, 1);
286 b = s390_vec_read_element64(v3, 1);
287 tmp2 = galois_multiply64(a, b);
288 s390_vec_xor(v1, &tmp1, &tmp2);
291 #define DEF_VGFMA(BITS, TBITS) \
292 void HELPER(gvec_vgfma##BITS)(void *v1, const void *v2, const void *v3, \
293 const void *v4, uint32_t desc) \
297 for (i = 0; i < (128 / TBITS); i++) { \
298 uint##BITS##_t a = s390_vec_read_element##BITS(v2, i * 2); \
299 uint##BITS##_t b = s390_vec_read_element##BITS(v3, i * 2); \
300 uint##TBITS##_t d = galois_multiply##BITS(a, b); \
302 a = s390_vec_read_element##BITS(v2, i * 2 + 1); \
303 b = s390_vec_read_element##BITS(v3, i * 2 + 1); \
304 d = d ^ galois_multiply32(a, b); \
305 d = d ^ s390_vec_read_element##TBITS(v4, i); \
306 s390_vec_write_element##TBITS(v1, i, d); \
311 void HELPER(gvec_vgfma64)(void *v1, const void *v2, const void *v3,
312 const void *v4, uint32_t desc)
314 S390Vector tmp1, tmp2;
317 a = s390_vec_read_element64(v2, 0);
318 b = s390_vec_read_element64(v3, 0);
319 tmp1 = galois_multiply64(a, b);
320 a = s390_vec_read_element64(v2, 1);
321 b = s390_vec_read_element64(v3, 1);
322 tmp2 = galois_multiply64(a, b);
323 s390_vec_xor(&tmp1, &tmp1, &tmp2);
324 s390_vec_xor(v1, &tmp1, v4);
327 #define DEF_VMAL(BITS) \
328 void HELPER(gvec_vmal##BITS)(void *v1, const void *v2, const void *v3, \
329 const void *v4, uint32_t desc) \
333 for (i = 0; i < (128 / BITS); i++) { \
334 const uint##BITS##_t a = s390_vec_read_element##BITS(v2, i); \
335 const uint##BITS##_t b = s390_vec_read_element##BITS(v3, i); \
336 const uint##BITS##_t c = s390_vec_read_element##BITS(v4, i); \
338 s390_vec_write_element##BITS(v1, i, a * b + c); \
344 #define DEF_VMAH(BITS) \
345 void HELPER(gvec_vmah##BITS)(void *v1, const void *v2, const void *v3, \
346 const void *v4, uint32_t desc) \
350 for (i = 0; i < (128 / BITS); i++) { \
351 const int32_t a = (int##BITS##_t)s390_vec_read_element##BITS(v2, i); \
352 const int32_t b = (int##BITS##_t)s390_vec_read_element##BITS(v3, i); \
353 const int32_t c = (int##BITS##_t)s390_vec_read_element##BITS(v4, i); \
355 s390_vec_write_element##BITS(v1, i, (a * b + c) >> BITS); \
361 #define DEF_VMALH(BITS) \
362 void HELPER(gvec_vmalh##BITS)(void *v1, const void *v2, const void *v3, \
363 const void *v4, uint32_t desc) \
367 for (i = 0; i < (128 / BITS); i++) { \
368 const uint##BITS##_t a = s390_vec_read_element##BITS(v2, i); \
369 const uint##BITS##_t b = s390_vec_read_element##BITS(v3, i); \
370 const uint##BITS##_t c = s390_vec_read_element##BITS(v4, i); \
372 s390_vec_write_element##BITS(v1, i, (a * b + c) >> BITS); \
378 #define DEF_VMAE(BITS, TBITS) \
379 void HELPER(gvec_vmae##BITS)(void *v1, const void *v2, const void *v3, \
380 const void *v4, uint32_t desc) \
384 for (i = 0, j = 0; i < (128 / TBITS); i++, j += 2) { \
385 int##TBITS##_t a = (int##BITS##_t)s390_vec_read_element##BITS(v2, j); \
386 int##TBITS##_t b = (int##BITS##_t)s390_vec_read_element##BITS(v3, j); \
387 int##TBITS##_t c = s390_vec_read_element##TBITS(v4, i); \
389 s390_vec_write_element##TBITS(v1, i, a * b + c); \
396 #define DEF_VMALE(BITS, TBITS) \
397 void HELPER(gvec_vmale##BITS)(void *v1, const void *v2, const void *v3, \
398 const void *v4, uint32_t desc) \
402 for (i = 0, j = 0; i < (128 / TBITS); i++, j += 2) { \
403 uint##TBITS##_t a = s390_vec_read_element##BITS(v2, j); \
404 uint##TBITS##_t b = s390_vec_read_element##BITS(v3, j); \
405 uint##TBITS##_t c = s390_vec_read_element##TBITS(v4, i); \
407 s390_vec_write_element##TBITS(v1, i, a * b + c); \
414 #define DEF_VMAO(BITS, TBITS) \
415 void HELPER(gvec_vmao##BITS)(void *v1, const void *v2, const void *v3, \
416 const void *v4, uint32_t desc) \
420 for (i = 0, j = 1; i < (128 / TBITS); i++, j += 2) { \
421 int##TBITS##_t a = (int##BITS##_t)s390_vec_read_element##BITS(v2, j); \
422 int##TBITS##_t b = (int##BITS##_t)s390_vec_read_element##BITS(v3, j); \
423 int##TBITS##_t c = s390_vec_read_element##TBITS(v4, i); \
425 s390_vec_write_element##TBITS(v1, i, a * b + c); \
432 #define DEF_VMALO(BITS, TBITS) \
433 void HELPER(gvec_vmalo##BITS)(void *v1, const void *v2, const void *v3, \
434 const void *v4, uint32_t desc) \
438 for (i = 0, j = 1; i < (128 / TBITS); i++, j += 2) { \
439 uint##TBITS##_t a = s390_vec_read_element##BITS(v2, j); \
440 uint##TBITS##_t b = s390_vec_read_element##BITS(v3, j); \
441 uint##TBITS##_t c = s390_vec_read_element##TBITS(v4, i); \
443 s390_vec_write_element##TBITS(v1, i, a * b + c); \
450 #define DEF_VMH(BITS) \
451 void HELPER(gvec_vmh##BITS)(void *v1, const void *v2, const void *v3, \
456 for (i = 0; i < (128 / BITS); i++) { \
457 const int32_t a = (int##BITS##_t)s390_vec_read_element##BITS(v2, i); \
458 const int32_t b = (int##BITS##_t)s390_vec_read_element##BITS(v3, i); \
460 s390_vec_write_element##BITS(v1, i, (a * b) >> BITS); \
466 #define DEF_VMLH(BITS) \
467 void HELPER(gvec_vmlh##BITS)(void *v1, const void *v2, const void *v3, \
472 for (i = 0; i < (128 / BITS); i++) { \
473 const uint##BITS##_t a = s390_vec_read_element##BITS(v2, i); \
474 const uint##BITS##_t b = s390_vec_read_element##BITS(v3, i); \
476 s390_vec_write_element##BITS(v1, i, (a * b) >> BITS); \
482 #define DEF_VME(BITS, TBITS) \
483 void HELPER(gvec_vme##BITS)(void *v1, const void *v2, const void *v3, \
488 for (i = 0, j = 0; i < (128 / TBITS); i++, j += 2) { \
489 int##TBITS##_t a = (int##BITS##_t)s390_vec_read_element##BITS(v2, j); \
490 int##TBITS##_t b = (int##BITS##_t)s390_vec_read_element##BITS(v3, j); \
492 s390_vec_write_element##TBITS(v1, i, a * b); \
499 #define DEF_VMLE(BITS, TBITS) \
500 void HELPER(gvec_vmle##BITS)(void *v1, const void *v2, const void *v3, \
505 for (i = 0, j = 0; i < (128 / TBITS); i++, j += 2) { \
506 const uint##TBITS##_t a = s390_vec_read_element##BITS(v2, j); \
507 const uint##TBITS##_t b = s390_vec_read_element##BITS(v3, j); \
509 s390_vec_write_element##TBITS(v1, i, a * b); \
516 #define DEF_VMO(BITS, TBITS) \
517 void HELPER(gvec_vmo##BITS)(void *v1, const void *v2, const void *v3, \
522 for (i = 0, j = 1; i < (128 / TBITS); i++, j += 2) { \
523 int##TBITS##_t a = (int##BITS##_t)s390_vec_read_element##BITS(v2, j); \
524 int##TBITS##_t b = (int##BITS##_t)s390_vec_read_element##BITS(v3, j); \
526 s390_vec_write_element##TBITS(v1, i, a * b); \
533 #define DEF_VMLO(BITS, TBITS) \
534 void HELPER(gvec_vmlo##BITS)(void *v1, const void *v2, const void *v3, \
539 for (i = 0, j = 1; i < (128 / TBITS); i++, j += 2) { \
540 const uint##TBITS##_t a = s390_vec_read_element##BITS(v2, j); \
541 const uint##TBITS##_t b = s390_vec_read_element##BITS(v3, j); \
543 s390_vec_write_element##TBITS(v1, i, a * b); \
550 #define DEF_VPOPCT(BITS) \
551 void HELPER(gvec_vpopct##BITS)(void *v1, const void *v2, uint32_t desc) \
555 for (i = 0; i < (128 / BITS); i++) { \
556 const uint##BITS##_t a = s390_vec_read_element##BITS(v2, i); \
558 s390_vec_write_element##BITS(v1, i, ctpop32(a)); \
564 #define DEF_VERIM(BITS) \
565 void HELPER(gvec_verim##BITS)(void *v1, const void *v2, const void *v3, \
568 const uint8_t count = simd_data(desc); \
571 for (i = 0; i < (128 / BITS); i++) { \
572 const uint##BITS##_t a = s390_vec_read_element##BITS(v1, i); \
573 const uint##BITS##_t b = s390_vec_read_element##BITS(v2, i); \
574 const uint##BITS##_t mask = s390_vec_read_element##BITS(v3, i); \
575 const uint##BITS##_t d = (a & ~mask) | (rol##BITS(b, count) & mask); \
577 s390_vec_write_element##BITS(v1, i, d); \
583 void HELPER(gvec_vsl)(void *v1, const void *v2, uint64_t count,
586 s390_vec_shl(v1, v2, count);
589 void HELPER(gvec_vsl_ve2)(void *v1, const void *v2, const void *v3,
593 uint32_t sh, e0, e1 = 0;
596 for (i = 15; i >= 0; --i, e1 = e0) {
597 e0 = s390_vec_read_element8(v2, i);
598 sh = s390_vec_read_element8(v3, i) & 7;
600 s390_vec_write_element8(&tmp, i, rol32(e0 | (e1 << 24), sh));
603 *(S390Vector *)v1 = tmp;
606 void HELPER(gvec_vsra)(void *v1, const void *v2, uint64_t count,
609 s390_vec_sar(v1, v2, count);
612 void HELPER(gvec_vsra_ve2)(void *v1, const void *v2, const void *v3,
616 uint32_t sh, e0, e1 = 0;
619 /* Byte 0 is special only. */
620 e0 = (int32_t)(int8_t)s390_vec_read_element8(v2, i);
621 sh = s390_vec_read_element8(v3, i) & 7;
622 s390_vec_write_element8(&tmp, i, e0 >> sh);
625 for (i = 1; i < 16; ++i, e1 = e0) {
626 e0 = s390_vec_read_element8(v2, i);
627 sh = s390_vec_read_element8(v3, i) & 7;
628 s390_vec_write_element8(&tmp, i, (e0 | e1 << 8) >> sh);
631 *(S390Vector *)v1 = tmp;
634 void HELPER(gvec_vsrl)(void *v1, const void *v2, uint64_t count,
637 s390_vec_shr(v1, v2, count);
640 void HELPER(gvec_vsrl_ve2)(void *v1, const void *v2, const void *v3,
644 uint32_t sh, e0, e1 = 0;
646 for (int i = 0; i < 16; ++i, e1 = e0) {
647 e0 = s390_vec_read_element8(v2, i);
648 sh = s390_vec_read_element8(v3, i) & 7;
650 s390_vec_write_element8(&tmp, i, (e0 | (e1 << 8)) >> sh);
653 *(S390Vector *)v1 = tmp;
656 #define DEF_VSCBI(BITS) \
657 void HELPER(gvec_vscbi##BITS)(void *v1, const void *v2, const void *v3, \
662 for (i = 0; i < (128 / BITS); i++) { \
663 const uint##BITS##_t a = s390_vec_read_element##BITS(v2, i); \
664 const uint##BITS##_t b = s390_vec_read_element##BITS(v3, i); \
666 s390_vec_write_element##BITS(v1, i, a >= b); \
672 void HELPER(gvec_vtm)(void *v1, const void *v2, CPUS390XState *env,
677 s390_vec_and(&tmp, v1, v2);
678 if (s390_vec_is_zero(&tmp)) {
679 /* Selected bits all zeros; or all mask bits zero */
681 } else if (s390_vec_equal(&tmp, v2)) {
682 /* Selected bits all ones */
685 /* Selected bits a mix of zeros and ones */