vendor/gonum.org/v1/gonum/internal/asm/c64/dotuunitary_amd64.s

   1 // Copyright ©2017 The Gonum Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style
   3 // license that can be found in the LICENSE file.
   4
   5 //+build !noasm,!appengine
   6
   7 #include "textflag.h"
   8
   9 #define MOVSLDUP_XPTR_IDX_8__X3    LONG $0x1C120FF3; BYTE $0xC6 // MOVSLDUP (SI)(AX*8), X3
  10 #define MOVSLDUP_16_XPTR_IDX_8__X5    LONG $0x6C120FF3; WORD $0x10C6 // MOVSLDUP 16(SI)(AX*8), X5
  11 #define MOVSLDUP_32_XPTR_IDX_8__X7    LONG $0x7C120FF3; WORD $0x20C6 // MOVSLDUP 32(SI)(AX*8), X7
  12 #define MOVSLDUP_48_XPTR_IDX_8__X9    LONG $0x120F44F3; WORD $0xC64C; BYTE $0x30 // MOVSLDUP 48(SI)(AX*8), X9
  13
  14 #define MOVSHDUP_XPTR_IDX_8__X2    LONG $0x14160FF3; BYTE $0xC6 // MOVSHDUP (SI)(AX*8), X2
  15 #define MOVSHDUP_16_XPTR_IDX_8__X4    LONG $0x64160FF3; WORD $0x10C6 // MOVSHDUP 16(SI)(AX*8), X4
  16 #define MOVSHDUP_32_XPTR_IDX_8__X6    LONG $0x74160FF3; WORD $0x20C6 // MOVSHDUP 32(SI)(AX*8), X6
  17 #define MOVSHDUP_48_XPTR_IDX_8__X8    LONG $0x160F44F3; WORD $0xC644; BYTE $0x30 // MOVSHDUP 48(SI)(AX*8), X8
  18
  19 #define MOVSHDUP_X3_X2    LONG $0xD3160FF3 // MOVSHDUP X3, X2
  20 #define MOVSLDUP_X3_X3    LONG $0xDB120FF3 // MOVSLDUP X3, X3
  21
  22 #define ADDSUBPS_X2_X3    LONG $0xDAD00FF2 // ADDSUBPS X2, X3
  23 #define ADDSUBPS_X4_X5    LONG $0xECD00FF2 // ADDSUBPS X4, X5
  24 #define ADDSUBPS_X6_X7    LONG $0xFED00FF2 // ADDSUBPS X6, X7
  25 #define ADDSUBPS_X8_X9    LONG $0xD00F45F2; BYTE $0xC8 // ADDSUBPS X8, X9
  26
  27 #define X_PTR SI
  28 #define Y_PTR DI
  29 #define LEN CX
  30 #define TAIL BX
  31 #define SUM X0
  32 #define P_SUM X1
  33 #define IDX AX
  34 #define I_IDX DX
  35 #define NEG1 X15
  36 #define P_NEG1 X14
  37
  38 // func DotuUnitary(x, y []complex64) (sum complex64)
  39 TEXT ·DotuUnitary(SB), NOSPLIT, $0
  40         MOVQ    x_base+0(FP), X_PTR  // X_PTR = &x
  41         MOVQ    y_base+24(FP), Y_PTR // Y_PTR = &y
  42         PXOR    SUM, SUM             // SUM = 0
  43         PXOR    P_SUM, P_SUM         // P_SUM = 0
  44         MOVQ    x_len+8(FP), LEN     // LEN = min( len(x), len(y) )
  45         CMPQ    y_len+32(FP), LEN
  46         CMOVQLE y_len+32(FP), LEN
  47         CMPQ    LEN, $0              // if LEN == 0 { return }
  48         JE      dotu_end
  49         XORQ    IDX, IDX             // IDX = 0
  50
  51         MOVQ X_PTR, DX
  52         ANDQ $15, DX      // DX = &x & 15
  53         JZ   dotu_aligned // if DX == 0 { goto dotu_aligned }
  54
  55         MOVSD  (X_PTR)(IDX*8), X3  // X_i     = { imag(x[i]), real(x[i]) }
  56         MOVSHDUP_X3_X2             // X_(i-1) = { imag(x[i]), imag(x[i]) }
  57         MOVSLDUP_X3_X3             // X_i     = { real(x[i]), real(x[i]) }
  58         MOVSD  (Y_PTR)(IDX*8), X10 // X_j     = { imag(y[i]), real(y[i]) }
  59         MULPS  X10, X3             // X_i     = { imag(y[i]) * real(x[i]), real(y[i]) * real(x[i]) }
  60         SHUFPS $0x1, X10, X10      // X_j     = { real(y[i]), imag(y[i]) }
  61         MULPS  X10, X2             // X_(i-1) = { real(y[i]) * imag(x[i]), imag(y[i]) * imag(x[i]) }
  62
  63         // X_i = {
  64         //      imag(result[i]):  imag(y[i])*real(x[i]) + real(y[i])*imag(x[i]),
  65         //      real(result[i]):  real(y[i])*real(x[i]) - imag(y[i])*imag(x[i]) }
  66         ADDSUBPS_X2_X3
  67
  68         MOVAPS X3, SUM  // SUM = X_i
  69         INCQ   IDX      // IDX++
  70         DECQ   LEN      // LEN--
  71         JZ     dotu_end // if LEN == 0 { goto dotu_end }
  72
  73 dotu_aligned:
  74         MOVQ LEN, TAIL
  75         ANDQ $7, TAIL     // TAIL = LEN % 8
  76         SHRQ $3, LEN      // LEN = floor( LEN / 8 )
  77         JZ   dotu_tail    // if LEN == 0 { goto dotu_tail }
  78         PXOR P_SUM, P_SUM
  79
  80 dotu_loop: // do {
  81         MOVSLDUP_XPTR_IDX_8__X3    // X_i = { real(x[i]), real(x[i]), real(x[i+1]), real(x[i+1]) }
  82         MOVSLDUP_16_XPTR_IDX_8__X5
  83         MOVSLDUP_32_XPTR_IDX_8__X7
  84         MOVSLDUP_48_XPTR_IDX_8__X9
  85
  86         MOVSHDUP_XPTR_IDX_8__X2    // X_(i-1) = { imag(x[i]), imag(x[i]), imag(x[i]+1), imag(x[i]+1) }
  87         MOVSHDUP_16_XPTR_IDX_8__X4
  88         MOVSHDUP_32_XPTR_IDX_8__X6
  89         MOVSHDUP_48_XPTR_IDX_8__X8
  90
  91         // X_j = { imag(y[i]), real(y[i]), imag(y[i+1]), real(y[i+1]) }
  92         MOVUPS (Y_PTR)(IDX*8), X10
  93         MOVUPS 16(Y_PTR)(IDX*8), X11
  94         MOVUPS 32(Y_PTR)(IDX*8), X12
  95         MOVUPS 48(Y_PTR)(IDX*8), X13
  96
  97         // X_i     = {  imag(y[i])   * real(x[i]),   real(y[i])   * real(x[i]),
  98         //              imag(y[i+1]) * real(x[i+1]), real(y[i+1]) * real(x[i+1])  }
  99         MULPS X10, X3
 100         MULPS X11, X5
 101         MULPS X12, X7
 102         MULPS X13, X9
 103
 104         // X_j = { real(y[i]), imag(y[i]), real(y[i+1]), imag(y[i+1]) }
 105         SHUFPS $0xB1, X10, X10
 106         SHUFPS $0xB1, X11, X11
 107         SHUFPS $0xB1, X12, X12
 108         SHUFPS $0xB1, X13, X13
 109
 110         // X_(i-1) = {  real(y[i])   * imag(x[i]),   imag(y[i])   * imag(x[i]),
 111         //              real(y[i+1]) * imag(x[i+1]), imag(y[i+1]) * imag(x[i+1])  }
 112         MULPS X10, X2
 113         MULPS X11, X4
 114         MULPS X12, X6
 115         MULPS X13, X8
 116
 117         // X_i = {
 118         //      imag(result[i]):   imag(y[i])   * real(x[i])   + real(y[i])   * imag(x[i]),
 119         //      real(result[i]):   real(y[i])   * real(x[i])   - imag(y[i])   * imag(x[i]),
 120         //      imag(result[i+1]): imag(y[i+1]) * real(x[i+1]) + real(y[i+1]) * imag(x[i+1]),
 121         //      real(result[i+1]): real(y[i+1]) * real(x[i+1]) - imag(y[i+1]) * imag(x[i+1]),
 122         //  }
 123         ADDSUBPS_X2_X3
 124         ADDSUBPS_X4_X5
 125         ADDSUBPS_X6_X7
 126         ADDSUBPS_X8_X9
 127
 128         // SUM += X_i
 129         ADDPS X3, SUM
 130         ADDPS X5, P_SUM
 131         ADDPS X7, SUM
 132         ADDPS X9, P_SUM
 133
 134         ADDQ $8, IDX   // IDX += 8
 135         DECQ LEN
 136         JNZ  dotu_loop // } while --LEN > 0
 137
 138         ADDPS SUM, P_SUM // P_SUM = { P_SUM[1] + SUM[1], P_SUM[0] + SUM[0] }
 139         XORPS SUM, SUM   // SUM = 0
 140
 141         CMPQ TAIL, $0 // if TAIL == 0 { return }
 142         JE   dotu_end
 143
 144 dotu_tail:
 145         MOVQ TAIL, LEN
 146         SHRQ $1, LEN       // LEN = floor( LEN / 2 )
 147         JZ   dotu_tail_one // if LEN == 0 { goto dotc_tail_one }
 148
 149 dotu_tail_two: // do {
 150         MOVSLDUP_XPTR_IDX_8__X3    // X_i = { real(x[i]), real(x[i]), real(x[i+1]), real(x[i+1]) }
 151         MOVSHDUP_XPTR_IDX_8__X2    // X_(i-1) = { imag(x[i]), imag(x[i]), imag(x[i]+1), imag(x[i]+1) }
 152         MOVUPS (Y_PTR)(IDX*8), X10 // X_j = { imag(y[i]), real(y[i]) }
 153         MULPS  X10, X3             // X_i = { imag(y[i]) * real(x[i]), real(y[i]) * real(x[i]) }
 154         SHUFPS $0xB1, X10, X10     // X_j = { real(y[i]), imag(y[i]) }
 155         MULPS  X10, X2             // X_(i-1) = { real(y[i]) * imag(x[i]), imag(y[i]) * imag(x[i]) }
 156
 157         // X_i = {
 158         //      imag(result[i]):  imag(y[i])*real(x[i]) + real(y[i])*imag(x[i]),
 159         //      real(result[i]):  real(y[i])*real(x[i]) - imag(y[i])*imag(x[i]) }
 160         ADDSUBPS_X2_X3
 161
 162         ADDPS X3, SUM // SUM += X_i
 163
 164         ADDQ $2, IDX       // IDX += 2
 165         DECQ LEN
 166         JNZ  dotu_tail_two // } while --LEN > 0
 167
 168         ADDPS SUM, P_SUM // P_SUM = { P_SUM[1] + SUM[1], P_SUM[0] + SUM[0] }
 169         XORPS SUM, SUM   // SUM = 0
 170
 171         ANDQ $1, TAIL
 172         JZ   dotu_end
 173
 174 dotu_tail_one:
 175         MOVSD  (X_PTR)(IDX*8), X3  // X_i = { imag(x[i]), real(x[i]) }
 176         MOVSHDUP_X3_X2             // X_(i-1) = { imag(x[i]), imag(x[i]) }
 177         MOVSLDUP_X3_X3             // X_i = { real(x[i]), real(x[i]) }
 178         MOVSD  (Y_PTR)(IDX*8), X10 // X_j = { imag(y[i]), real(y[i]) }
 179         MULPS  X10, X3             // X_i = { imag(y[i]) * real(x[i]), real(y[i]) * real(x[i]) }
 180         SHUFPS $0x1, X10, X10      // X_j = { real(y[i]), imag(y[i]) }
 181         MULPS  X10, X2             // X_(i-1) = { real(y[i]) * imag(x[i]), imag(y[i]) * imag(x[i]) }
 182
 183         // X_i = {
 184         //      imag(result[i]):  imag(y[i])*real(x[i]) + real(y[i])*imag(x[i]),
 185         //      real(result[i]):  real(y[i])*real(x[i]) - imag(y[i])*imag(x[i]) }
 186         ADDSUBPS_X2_X3
 187
 188         ADDPS X3, SUM // SUM += X_i
 189
 190 dotu_end:
 191         ADDPS   P_SUM, SUM   // SUM = { P_SUM[0] + SUM[0] }
 192         MOVHLPS P_SUM, P_SUM // P_SUM = { P_SUM[1], P_SUM[1] }
 193         ADDPS   P_SUM, SUM   // SUM = { P_SUM[1] + SUM[0] }
 194
 195 dotu_ret:
 196         MOVSD SUM, sum+48(FP) // return SUM
 197         RET