1 // Copyright ©2016 The Gonum Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style
3 // license that can be found in the LICENSE file.
5 //+build !noasm,!appengine
10 #define MOVSHDUP_X3_X2 BYTE $0xF3; BYTE $0x0F; BYTE $0x16; BYTE $0xD3
12 #define MOVSLDUP_X3_X3 BYTE $0xF3; BYTE $0x0F; BYTE $0x12; BYTE $0xDB
14 #define ADDSUBPS_X2_X3 BYTE $0xF2; BYTE $0x0F; BYTE $0xD0; BYTE $0xDA
17 #define MOVSHDUP_X5_X4 BYTE $0xF3; BYTE $0x0F; BYTE $0x16; BYTE $0xE5
19 #define MOVSLDUP_X5_X5 BYTE $0xF3; BYTE $0x0F; BYTE $0x12; BYTE $0xED
21 #define ADDSUBPS_X4_X5 BYTE $0xF2; BYTE $0x0F; BYTE $0xD0; BYTE $0xEC
24 #define MOVSHDUP_X7_X6 BYTE $0xF3; BYTE $0x0F; BYTE $0x16; BYTE $0xF7
26 #define MOVSLDUP_X7_X7 BYTE $0xF3; BYTE $0x0F; BYTE $0x12; BYTE $0xFF
28 #define ADDSUBPS_X6_X7 BYTE $0xF2; BYTE $0x0F; BYTE $0xD0; BYTE $0xFE
31 #define MOVSHDUP_X9_X8 BYTE $0xF3; BYTE $0x45; BYTE $0x0F; BYTE $0x16; BYTE $0xC1
33 #define MOVSLDUP_X9_X9 BYTE $0xF3; BYTE $0x45; BYTE $0x0F; BYTE $0x12; BYTE $0xC9
35 #define ADDSUBPS_X8_X9 BYTE $0xF2; BYTE $0x45; BYTE $0x0F; BYTE $0xD0; BYTE $0xC8
37 // func AxpyUnitary(alpha complex64, x, y []complex64)
38 TEXT ·AxpyUnitary(SB), NOSPLIT, $0
39 MOVQ x_base+8(FP), SI // SI = &x
40 MOVQ y_base+32(FP), DI // DI = &y
41 MOVQ x_len+16(FP), CX // CX = min( len(x), len(y) )
43 CMOVQLE y_len+40(FP), CX
44 CMPQ CX, $0 // if CX == 0 { return }
46 PXOR X0, X0 // Clear work registers and cache-align loop
48 MOVSD alpha+0(FP), X0 // X0 = { 0, 0, imag(a), real(a) }
49 SHUFPD $0, X0, X0 // X0 = { imag(a), real(a), imag(a), real(a) }
51 SHUFPS $0x11, X1, X1 // X1 = { real(a), imag(a), real(a), imag(a) }
53 MOVQ DI, BX // Align on 16-byte boundary for ADDPS
54 ANDQ $15, BX // BX = &y & 15
55 JZ caxy_no_trim // if BX == 0 { goto caxy_no_trim }
57 // Trim first value in unaligned buffer
58 XORPS X2, X2 // Clear work registers and cache-align loop
61 MOVSD (SI)(AX*8), X3 // X3 = { imag(x[i]), real(x[i]) }
62 MOVSHDUP_X3_X2 // X2 = { imag(x[i]), imag(x[i]) }
63 MOVSLDUP_X3_X3 // X3 = { real(x[i]), real(x[i]) }
64 MULPS X1, X2 // X2 = { real(a) * imag(x[i]), imag(a) * imag(x[i]) }
65 MULPS X0, X3 // X3 = { imag(a) * real(x[i]), real(a) * real(x[i]) }
67 // X3 = { imag(a)*real(x[i]) + real(a)*imag(x[i]), real(a)*real(x[i]) - imag(a)*imag(x[i]) }
69 MOVSD (DI)(AX*8), X4 // X3 += y[i]
71 MOVSD X3, (DI)(AX*8) // y[i] = X3
74 JZ caxy_end // if CX == 0 { return }
77 MOVAPS X0, X10 // Copy X0 and X1 for pipelineing
80 ANDQ $7, CX // CX = n % 8
81 SHRQ $3, BX // BX = floor( n / 8 )
82 JZ caxy_tail // if BX == 0 { goto caxy_tail }
85 // X_i = { imag(x[i]), real(x[i]), imag(x[i+1]), real(x[i+1]) }
87 MOVUPS 16(SI)(AX*8), X5
88 MOVUPS 32(SI)(AX*8), X7
89 MOVUPS 48(SI)(AX*8), X9
91 // X_(i-1) = { imag(x[i]), imag(x[i]), imag(x[i]+1), imag(x[i]+1) }
97 // X_i = { real(x[i]), real(x[i]), real(x[i+1]), real(x[i+1]) }
103 // X_i = { imag(a) * real(x[i]), real(a) * real(x[i]),
104 // imag(a) * real(x[i+1]), real(a) * real(x[i+1]) }
105 // X_(i-1) = { real(a) * imag(x[i]), imag(a) * imag(x[i]),
106 // real(a) * imag(x[i+1]), imag(a) * imag(x[i+1]) }
117 // imag(result[i]): imag(a)*real(x[i]) + real(a)*imag(x[i]),
118 // real(result[i]): real(a)*real(x[i]) - imag(a)*imag(x[i]),
119 // imag(result[i+1]): imag(a)*real(x[i+1]) + real(a)*imag(x[i+1]),
120 // real(result[i+1]): real(a)*real(x[i+1]) - imag(a)*imag(x[i+1]),
127 // X_i = { imag(result[i]) + imag(y[i]), real(result[i]) + real(y[i]),
128 // imag(result[i+1]) + imag(y[i+1]), real(result[i+1]) + real(y[i+1]) }
130 ADDPS 16(DI)(AX*8), X5
131 ADDPS 32(DI)(AX*8), X7
132 ADDPS 48(DI)(AX*8), X9
133 MOVUPS X3, (DI)(AX*8) // y[i:i+1] = X_i
134 MOVUPS X5, 16(DI)(AX*8)
135 MOVUPS X7, 32(DI)(AX*8)
136 MOVUPS X9, 48(DI)(AX*8)
137 ADDQ $8, AX // i += 8
139 JNZ caxy_loop // } while BX > 0
140 CMPQ CX, $0 // if CX == 0 { return }
144 MOVSD (SI)(AX*8), X3 // X3 = { imag(x[i]), real(x[i]) }
145 MOVSHDUP_X3_X2 // X2 = { imag(x[i]), imag(x[i]) }
146 MOVSLDUP_X3_X3 // X3 = { real(x[i]), real(x[i]) }
147 MULPS X1, X2 // X2 = { real(a) * imag(x[i]), imag(a) * imag(x[i]) }
148 MULPS X0, X3 // X3 = { imag(a) * real(x[i]), real(a) * real(x[i]) }
150 // X3 = { imag(a)*real(x[i]) + real(a)*imag(x[i]),
151 // real(a)*real(x[i]) - imag(a)*imag(x[i]) }
153 MOVSD (DI)(AX*8), X4 // X3 += y[i]
155 MOVSD X3, (DI)(AX*8) // y[i] = X3
157 LOOP caxy_tail // } while --CX > 0