libavcodec/x86/simple_idct_mmx.c

   1 /*
   2  * Simple IDCT MMX
   3  *
   4  * Copyright (c) 2001, 2002 Michael Niedermayer <michaelni@gmx.at>
   5  *
   6  * This file is part of Libav.
   7  *
   8  * Libav is free software; you can redistribute it and/or
   9  * modify it under the terms of the GNU Lesser General Public
  10  * License as published by the Free Software Foundation; either
  11  * version 2.1 of the License, or (at your option) any later version.
  12  *
  13  * Libav is distributed in the hope that it will be useful,
  14  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  15  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  16  * Lesser General Public License for more details.
  17  *
  18  * You should have received a copy of the GNU Lesser General Public
  19  * License along with Libav; if not, write to the Free Software
  20  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
  21  */
  22 #include "libavcodec/dsputil.h"
  23 #include "libavcodec/simple_idct.h"
  24 #include "dsputil_mmx.h"
  25
  26 /*
  27 23170.475006
  28 22725.260826
  29 21406.727617
  30 19265.545870
  31 16384.000000
  32 12872.826198
  33 8866.956905
  34 4520.335430
  35 */
  36 #define C0 23170 //cos(i*M_PI/16)*sqrt(2)*(1<<14) + 0.5
  37 #define C1 22725 //cos(i*M_PI/16)*sqrt(2)*(1<<14) + 0.5
  38 #define C2 21407 //cos(i*M_PI/16)*sqrt(2)*(1<<14) + 0.5
  39 #define C3 19266 //cos(i*M_PI/16)*sqrt(2)*(1<<14) + 0.5
  40 #if 0
  41 #define C4 16384 //cos(i*M_PI/16)*sqrt(2)*(1<<14) + 0.5
  42 #else
  43 #define C4 16383 //cos(i*M_PI/16)*sqrt(2)*(1<<14) - 0.5
  44 #endif
  45 #define C5 12873 //cos(i*M_PI/16)*sqrt(2)*(1<<14) + 0.5
  46 #define C6 8867  //cos(i*M_PI/16)*sqrt(2)*(1<<14) + 0.5
  47 #define C7 4520  //cos(i*M_PI/16)*sqrt(2)*(1<<14) + 0.5
  48
  49 #define ROW_SHIFT 11
  50 #define COL_SHIFT 20 // 6
  51
  52 DECLARE_ASM_CONST(8, uint64_t, wm1010)= 0xFFFF0000FFFF0000ULL;
  53 DECLARE_ASM_CONST(8, uint64_t, d40000)= 0x0000000000040000ULL;
  54
  55 DECLARE_ALIGNED(8, static const int16_t, coeffs)[]= {
  56         1<<(ROW_SHIFT-1), 0, 1<<(ROW_SHIFT-1), 0,
  57 //        1<<(COL_SHIFT-1), 0, 1<<(COL_SHIFT-1), 0,
  58 //        0, 1<<(COL_SHIFT-1-16), 0, 1<<(COL_SHIFT-1-16),
  59         1<<(ROW_SHIFT-1), 1, 1<<(ROW_SHIFT-1), 0,
  60         // the 1 = ((1<<(COL_SHIFT-1))/C4)<<ROW_SHIFT :)
  61 //        0, 0, 0, 0,
  62 //        0, 0, 0, 0,
  63
  64  C4,  C4,  C4,  C4,
  65  C4, -C4,  C4, -C4,
  66
  67  C2,  C6,  C2,  C6,
  68  C6, -C2,  C6, -C2,
  69
  70  C1,  C3,  C1,  C3,
  71  C5,  C7,  C5,  C7,
  72
  73  C3, -C7,  C3, -C7,
  74 -C1, -C5, -C1, -C5,
  75
  76  C5, -C1,  C5, -C1,
  77  C7,  C3,  C7,  C3,
  78
  79  C7, -C5,  C7, -C5,
  80  C3, -C1,  C3, -C1
  81 };
  82
  83 #if 0
  84 static void unused_var_killer(void)
  85 {
  86         int a= wm1010 + d40000;
  87         temp[0]=a;
  88 }
  89
  90 static void inline idctCol (int16_t * col, int16_t *input)
  91 {
  92 #undef C0
  93 #undef C1
  94 #undef C2
  95 #undef C3
  96 #undef C4
  97 #undef C5
  98 #undef C6
  99 #undef C7
 100         int a0, a1, a2, a3, b0, b1, b2, b3;
 101         const int C0 = 23170; //cos(i*M_PI/16)*sqrt(2)*(1<<14) + 0.5
 102         const int C1 = 22725; //cos(i*M_PI/16)*sqrt(2)*(1<<14) + 0.5
 103         const int C2 = 21407; //cos(i*M_PI/16)*sqrt(2)*(1<<14) + 0.5
 104         const int C3 = 19266; //cos(i*M_PI/16)*sqrt(2)*(1<<14) + 0.5
 105         const int C4 = 16383; //cos(i*M_PI/16)*sqrt(2)*(1<<14) + 0.5
 106         const int C5 = 12873; //cos(i*M_PI/16)*sqrt(2)*(1<<14) + 0.5
 107         const int C6 = 8867;  //cos(i*M_PI/16)*sqrt(2)*(1<<14) + 0.5
 108         const int C7 = 4520;  //cos(i*M_PI/16)*sqrt(2)*(1<<14) + 0.5
 109 /*
 110         if( !(col[8*1] | col[8*2] |col[8*3] |col[8*4] |col[8*5] |col[8*6] | col[8*7])) {
 111                 col[8*0] = col[8*1] = col[8*2] = col[8*3] = col[8*4] =
 112                         col[8*5] = col[8*6] = col[8*7] = col[8*0]<<3;
 113                 return;
 114         }*/
 115
 116 col[8*0] = input[8*0 + 0];
 117 col[8*1] = input[8*2 + 0];
 118 col[8*2] = input[8*0 + 1];
 119 col[8*3] = input[8*2 + 1];
 120 col[8*4] = input[8*4 + 0];
 121 col[8*5] = input[8*6 + 0];
 122 col[8*6] = input[8*4 + 1];
 123 col[8*7] = input[8*6 + 1];
 124
 125         a0 = C4*col[8*0] + C2*col[8*2] + C4*col[8*4] + C6*col[8*6] + (1<<(COL_SHIFT-1));
 126         a1 = C4*col[8*0] + C6*col[8*2] - C4*col[8*4] - C2*col[8*6] + (1<<(COL_SHIFT-1));
 127         a2 = C4*col[8*0] - C6*col[8*2] - C4*col[8*4] + C2*col[8*6] + (1<<(COL_SHIFT-1));
 128         a3 = C4*col[8*0] - C2*col[8*2] + C4*col[8*4] - C6*col[8*6] + (1<<(COL_SHIFT-1));
 129
 130         b0 = C1*col[8*1] + C3*col[8*3] + C5*col[8*5] + C7*col[8*7];
 131         b1 = C3*col[8*1] - C7*col[8*3] - C1*col[8*5] - C5*col[8*7];
 132         b2 = C5*col[8*1] - C1*col[8*3] + C7*col[8*5] + C3*col[8*7];
 133         b3 = C7*col[8*1] - C5*col[8*3] + C3*col[8*5] - C1*col[8*7];
 134
 135         col[8*0] = (a0 + b0) >> COL_SHIFT;
 136         col[8*1] = (a1 + b1) >> COL_SHIFT;
 137         col[8*2] = (a2 + b2) >> COL_SHIFT;
 138         col[8*3] = (a3 + b3) >> COL_SHIFT;
 139         col[8*4] = (a3 - b3) >> COL_SHIFT;
 140         col[8*5] = (a2 - b2) >> COL_SHIFT;
 141         col[8*6] = (a1 - b1) >> COL_SHIFT;
 142         col[8*7] = (a0 - b0) >> COL_SHIFT;
 143 }
 144
 145 static void inline idctRow (int16_t * output, int16_t * input)
 146 {
 147         int16_t row[8];
 148
 149         int a0, a1, a2, a3, b0, b1, b2, b3;
 150         const int C0 = 23170; //cos(i*M_PI/16)*sqrt(2)*(1<<14) + 0.5
 151         const int C1 = 22725; //cos(i*M_PI/16)*sqrt(2)*(1<<14) + 0.5
 152         const int C2 = 21407; //cos(i*M_PI/16)*sqrt(2)*(1<<14) + 0.5
 153         const int C3 = 19266; //cos(i*M_PI/16)*sqrt(2)*(1<<14) + 0.5
 154         const int C4 = 16383; //cos(i*M_PI/16)*sqrt(2)*(1<<14) + 0.5
 155         const int C5 = 12873; //cos(i*M_PI/16)*sqrt(2)*(1<<14) + 0.5
 156         const int C6 = 8867;  //cos(i*M_PI/16)*sqrt(2)*(1<<14) + 0.5
 157         const int C7 = 4520;  //cos(i*M_PI/16)*sqrt(2)*(1<<14) + 0.5
 158
 159 row[0] = input[0];
 160 row[2] = input[1];
 161 row[4] = input[4];
 162 row[6] = input[5];
 163 row[1] = input[8];
 164 row[3] = input[9];
 165 row[5] = input[12];
 166 row[7] = input[13];
 167
 168         if( !(row[1] | row[2] |row[3] |row[4] |row[5] |row[6] | row[7]) ) {
 169                 row[0] = row[1] = row[2] = row[3] = row[4] =
 170                         row[5] = row[6] = row[7] = row[0]<<3;
 171         output[0]  = row[0];
 172         output[2]  = row[1];
 173         output[4]  = row[2];
 174         output[6]  = row[3];
 175         output[8]  = row[4];
 176         output[10] = row[5];
 177         output[12] = row[6];
 178         output[14] = row[7];
 179                 return;
 180         }
 181
 182         a0 = C4*row[0] + C2*row[2] + C4*row[4] + C6*row[6] + (1<<(ROW_SHIFT-1));
 183         a1 = C4*row[0] + C6*row[2] - C4*row[4] - C2*row[6] + (1<<(ROW_SHIFT-1));
 184         a2 = C4*row[0] - C6*row[2] - C4*row[4] + C2*row[6] + (1<<(ROW_SHIFT-1));
 185         a3 = C4*row[0] - C2*row[2] + C4*row[4] - C6*row[6] + (1<<(ROW_SHIFT-1));
 186
 187         b0 = C1*row[1] + C3*row[3] + C5*row[5] + C7*row[7];
 188         b1 = C3*row[1] - C7*row[3] - C1*row[5] - C5*row[7];
 189         b2 = C5*row[1] - C1*row[3] + C7*row[5] + C3*row[7];
 190         b3 = C7*row[1] - C5*row[3] + C3*row[5] - C1*row[7];
 191
 192         row[0] = (a0 + b0) >> ROW_SHIFT;
 193         row[1] = (a1 + b1) >> ROW_SHIFT;
 194         row[2] = (a2 + b2) >> ROW_SHIFT;
 195         row[3] = (a3 + b3) >> ROW_SHIFT;
 196         row[4] = (a3 - b3) >> ROW_SHIFT;
 197         row[5] = (a2 - b2) >> ROW_SHIFT;
 198         row[6] = (a1 - b1) >> ROW_SHIFT;
 199         row[7] = (a0 - b0) >> ROW_SHIFT;
 200
 201         output[0]  = row[0];
 202         output[2]  = row[1];
 203         output[4]  = row[2];
 204         output[6]  = row[3];
 205         output[8]  = row[4];
 206         output[10] = row[5];
 207         output[12] = row[6];
 208         output[14] = row[7];
 209 }
 210 #endif
 211
 212 static inline void idct(int16_t *block)
 213 {
 214         DECLARE_ALIGNED(8, int64_t, align_tmp)[16];
 215         int16_t * const temp= (int16_t*)align_tmp;
 216
 217         __asm__ volatile(
 218 #if 0 //Alternative, simpler variant
 219
 220 #define ROW_IDCT(src0, src4, src1, src5, dst, rounder, shift) \
 221         "movq " #src0 ", %%mm0          \n\t" /* R4     R0      r4      r0 */\
 222         "movq " #src4 ", %%mm1          \n\t" /* R6     R2      r6      r2 */\
 223         "movq " #src1 ", %%mm2          \n\t" /* R3     R1      r3      r1 */\
 224         "movq " #src5 ", %%mm3          \n\t" /* R7     R5      r7      r5 */\
 225         "movq 16(%2), %%mm4             \n\t" /* C4     C4      C4      C4 */\
 226         "pmaddwd %%mm0, %%mm4           \n\t" /* C4R4+C4R0      C4r4+C4r0 */\
 227         "movq 24(%2), %%mm5             \n\t" /* -C4    C4      -C4     C4 */\
 228         "pmaddwd %%mm5, %%mm0           \n\t" /* -C4R4+C4R0     -C4r4+C4r0 */\
 229         "movq 32(%2), %%mm5             \n\t" /* C6     C2      C6      C2 */\
 230         "pmaddwd %%mm1, %%mm5           \n\t" /* C6R6+C2R2      C6r6+C2r2 */\
 231         "movq 40(%2), %%mm6             \n\t" /* -C2    C6      -C2     C6 */\
 232         "pmaddwd %%mm6, %%mm1           \n\t" /* -C2R6+C6R2     -C2r6+C6r2 */\
 233         "movq 48(%2), %%mm7             \n\t" /* C3     C1      C3      C1 */\
 234         "pmaddwd %%mm2, %%mm7           \n\t" /* C3R3+C1R1      C3r3+C1r1 */\
 235         #rounder ", %%mm4               \n\t"\
 236         "movq %%mm4, %%mm6              \n\t" /* C4R4+C4R0      C4r4+C4r0 */\
 237         "paddd %%mm5, %%mm4             \n\t" /* A0             a0 */\
 238         "psubd %%mm5, %%mm6             \n\t" /* A3             a3 */\
 239         "movq 56(%2), %%mm5             \n\t" /* C7     C5      C7      C5 */\
 240         "pmaddwd %%mm3, %%mm5           \n\t" /* C7R7+C5R5      C7r7+C5r5 */\
 241         #rounder ", %%mm0               \n\t"\
 242         "paddd %%mm0, %%mm1             \n\t" /* A1             a1 */\
 243         "paddd %%mm0, %%mm0             \n\t" \
 244         "psubd %%mm1, %%mm0             \n\t" /* A2             a2 */\
 245         "pmaddwd 64(%2), %%mm2          \n\t" /* -C7R3+C3R1     -C7r3+C3r1 */\
 246         "paddd %%mm5, %%mm7             \n\t" /* B0             b0 */\
 247         "movq 72(%2), %%mm5             \n\t" /* -C5    -C1     -C5     -C1 */\
 248         "pmaddwd %%mm3, %%mm5           \n\t" /* -C5R7-C1R5     -C5r7-C1r5 */\
 249         "paddd %%mm4, %%mm7             \n\t" /* A0+B0          a0+b0 */\
 250         "paddd %%mm4, %%mm4             \n\t" /* 2A0            2a0 */\
 251         "psubd %%mm7, %%mm4             \n\t" /* A0-B0          a0-b0 */\
 252         "paddd %%mm2, %%mm5             \n\t" /* B1             b1 */\
 253         "psrad $" #shift ", %%mm7       \n\t"\
 254         "psrad $" #shift ", %%mm4       \n\t"\
 255         "movq %%mm1, %%mm2              \n\t" /* A1             a1 */\
 256         "paddd %%mm5, %%mm1             \n\t" /* A1+B1          a1+b1 */\
 257         "psubd %%mm5, %%mm2             \n\t" /* A1-B1          a1-b1 */\
 258         "psrad $" #shift ", %%mm1       \n\t"\
 259         "psrad $" #shift ", %%mm2       \n\t"\
 260         "packssdw %%mm1, %%mm7          \n\t" /* A1+B1  a1+b1   A0+B0   a0+b0 */\
 261         "packssdw %%mm4, %%mm2          \n\t" /* A0-B0  a0-b0   A1-B1   a1-b1 */\
 262         "movq %%mm7, " #dst "           \n\t"\
 263         "movq " #src1 ", %%mm1          \n\t" /* R3     R1      r3      r1 */\
 264         "movq 80(%2), %%mm4             \n\t" /* -C1    C5      -C1     C5 */\
 265         "movq %%mm2, 24+" #dst "        \n\t"\
 266         "pmaddwd %%mm1, %%mm4           \n\t" /* -C1R3+C5R1     -C1r3+C5r1 */\
 267         "movq 88(%2), %%mm7             \n\t" /* C3     C7      C3      C7 */\
 268         "pmaddwd 96(%2), %%mm1          \n\t" /* -C5R3+C7R1     -C5r3+C7r1 */\
 269         "pmaddwd %%mm3, %%mm7           \n\t" /* C3R7+C7R5      C3r7+C7r5 */\
 270         "movq %%mm0, %%mm2              \n\t" /* A2             a2 */\
 271         "pmaddwd 104(%2), %%mm3         \n\t" /* -C1R7+C3R5     -C1r7+C3r5 */\
 272         "paddd %%mm7, %%mm4             \n\t" /* B2             b2 */\
 273         "paddd %%mm4, %%mm2             \n\t" /* A2+B2          a2+b2 */\
 274         "psubd %%mm4, %%mm0             \n\t" /* a2-B2          a2-b2 */\
 275         "psrad $" #shift ", %%mm2       \n\t"\
 276         "psrad $" #shift ", %%mm0       \n\t"\
 277         "movq %%mm6, %%mm4              \n\t" /* A3             a3 */\
 278         "paddd %%mm1, %%mm3             \n\t" /* B3             b3 */\
 279         "paddd %%mm3, %%mm6             \n\t" /* A3+B3          a3+b3 */\
 280         "psubd %%mm3, %%mm4             \n\t" /* a3-B3          a3-b3 */\
 281         "psrad $" #shift ", %%mm6       \n\t"\
 282         "packssdw %%mm6, %%mm2          \n\t" /* A3+B3  a3+b3   A2+B2   a2+b2 */\
 283         "movq %%mm2, 8+" #dst "         \n\t"\
 284         "psrad $" #shift ", %%mm4       \n\t"\
 285         "packssdw %%mm0, %%mm4          \n\t" /* A2-B2  a2-b2   A3-B3   a3-b3 */\
 286         "movq %%mm4, 16+" #dst "        \n\t"\
 287
 288 #define COL_IDCT(src0, src4, src1, src5, dst, shift) \
 289         "movq " #src0 ", %%mm0          \n\t" /* R4     R0      r4      r0 */\
 290         "movq " #src4 ", %%mm1          \n\t" /* R6     R2      r6      r2 */\
 291         "movq " #src1 ", %%mm2          \n\t" /* R3     R1      r3      r1 */\
 292         "movq " #src5 ", %%mm3          \n\t" /* R7     R5      r7      r5 */\
 293         "movq 16(%2), %%mm4             \n\t" /* C4     C4      C4      C4 */\
 294         "pmaddwd %%mm0, %%mm4           \n\t" /* C4R4+C4R0      C4r4+C4r0 */\
 295         "movq 24(%2), %%mm5             \n\t" /* -C4    C4      -C4     C4 */\
 296         "pmaddwd %%mm5, %%mm0           \n\t" /* -C4R4+C4R0     -C4r4+C4r0 */\
 297         "movq 32(%2), %%mm5             \n\t" /* C6     C2      C6      C2 */\
 298         "pmaddwd %%mm1, %%mm5           \n\t" /* C6R6+C2R2      C6r6+C2r2 */\
 299         "movq 40(%2), %%mm6             \n\t" /* -C2    C6      -C2     C6 */\
 300         "pmaddwd %%mm6, %%mm1           \n\t" /* -C2R6+C6R2     -C2r6+C6r2 */\
 301         "movq %%mm4, %%mm6              \n\t" /* C4R4+C4R0      C4r4+C4r0 */\
 302         "movq 48(%2), %%mm7             \n\t" /* C3     C1      C3      C1 */\
 303         "pmaddwd %%mm2, %%mm7           \n\t" /* C3R3+C1R1      C3r3+C1r1 */\
 304         "paddd %%mm5, %%mm4             \n\t" /* A0             a0 */\
 305         "psubd %%mm5, %%mm6             \n\t" /* A3             a3 */\
 306         "movq %%mm0, %%mm5              \n\t" /* -C4R4+C4R0     -C4r4+C4r0 */\
 307         "paddd %%mm1, %%mm0             \n\t" /* A1             a1 */\
 308         "psubd %%mm1, %%mm5             \n\t" /* A2             a2 */\
 309         "movq 56(%2), %%mm1             \n\t" /* C7     C5      C7      C5 */\
 310         "pmaddwd %%mm3, %%mm1           \n\t" /* C7R7+C5R5      C7r7+C5r5 */\
 311         "pmaddwd 64(%2), %%mm2          \n\t" /* -C7R3+C3R1     -C7r3+C3r1 */\
 312         "paddd %%mm1, %%mm7             \n\t" /* B0             b0 */\
 313         "movq 72(%2), %%mm1             \n\t" /* -C5    -C1     -C5     -C1 */\
 314         "pmaddwd %%mm3, %%mm1           \n\t" /* -C5R7-C1R5     -C5r7-C1r5 */\
 315         "paddd %%mm4, %%mm7             \n\t" /* A0+B0          a0+b0 */\
 316         "paddd %%mm4, %%mm4             \n\t" /* 2A0            2a0 */\
 317         "psubd %%mm7, %%mm4             \n\t" /* A0-B0          a0-b0 */\
 318         "paddd %%mm2, %%mm1             \n\t" /* B1             b1 */\
 319         "psrad $" #shift ", %%mm7       \n\t"\
 320         "psrad $" #shift ", %%mm4       \n\t"\
 321         "movq %%mm0, %%mm2              \n\t" /* A1             a1 */\
 322         "paddd %%mm1, %%mm0             \n\t" /* A1+B1          a1+b1 */\
 323         "psubd %%mm1, %%mm2             \n\t" /* A1-B1          a1-b1 */\
 324         "psrad $" #shift ", %%mm0       \n\t"\
 325         "psrad $" #shift ", %%mm2       \n\t"\
 326         "packssdw %%mm7, %%mm7          \n\t" /* A0+B0  a0+b0 */\
 327         "movd %%mm7, " #dst "           \n\t"\
 328         "packssdw %%mm0, %%mm0          \n\t" /* A1+B1  a1+b1 */\
 329         "movd %%mm0, 16+" #dst "        \n\t"\
 330         "packssdw %%mm2, %%mm2          \n\t" /* A1-B1  a1-b1 */\
 331         "movd %%mm2, 96+" #dst "        \n\t"\
 332         "packssdw %%mm4, %%mm4          \n\t" /* A0-B0  a0-b0 */\
 333         "movd %%mm4, 112+" #dst "       \n\t"\
 334         "movq " #src1 ", %%mm0          \n\t" /* R3     R1      r3      r1 */\
 335         "movq 80(%2), %%mm4             \n\t" /* -C1    C5      -C1     C5 */\
 336         "pmaddwd %%mm0, %%mm4           \n\t" /* -C1R3+C5R1     -C1r3+C5r1 */\
 337         "movq 88(%2), %%mm7             \n\t" /* C3     C7      C3      C7 */\
 338         "pmaddwd 96(%2), %%mm0          \n\t" /* -C5R3+C7R1     -C5r3+C7r1 */\
 339         "pmaddwd %%mm3, %%mm7           \n\t" /* C3R7+C7R5      C3r7+C7r5 */\
 340         "movq %%mm5, %%mm2              \n\t" /* A2             a2 */\
 341         "pmaddwd 104(%2), %%mm3         \n\t" /* -C1R7+C3R5     -C1r7+C3r5 */\
 342         "paddd %%mm7, %%mm4             \n\t" /* B2             b2 */\
 343         "paddd %%mm4, %%mm2             \n\t" /* A2+B2          a2+b2 */\
 344         "psubd %%mm4, %%mm5             \n\t" /* a2-B2          a2-b2 */\
 345         "psrad $" #shift ", %%mm2       \n\t"\
 346         "psrad $" #shift ", %%mm5       \n\t"\
 347         "movq %%mm6, %%mm4              \n\t" /* A3             a3 */\
 348         "paddd %%mm0, %%mm3             \n\t" /* B3             b3 */\
 349         "paddd %%mm3, %%mm6             \n\t" /* A3+B3          a3+b3 */\
 350         "psubd %%mm3, %%mm4             \n\t" /* a3-B3          a3-b3 */\
 351         "psrad $" #shift ", %%mm6       \n\t"\
 352         "psrad $" #shift ", %%mm4       \n\t"\
 353         "packssdw %%mm2, %%mm2          \n\t" /* A2+B2  a2+b2 */\
 354         "packssdw %%mm6, %%mm6          \n\t" /* A3+B3  a3+b3 */\
 355         "movd %%mm2, 32+" #dst "        \n\t"\
 356         "packssdw %%mm4, %%mm4          \n\t" /* A3-B3  a3-b3 */\
 357         "packssdw %%mm5, %%mm5          \n\t" /* A2-B2  a2-b2 */\
 358         "movd %%mm6, 48+" #dst "        \n\t"\
 359         "movd %%mm4, 64+" #dst "        \n\t"\
 360         "movd %%mm5, 80+" #dst "        \n\t"\
 361
 362
 363 #define DC_COND_ROW_IDCT(src0, src4, src1, src5, dst, rounder, shift) \
 364         "movq " #src0 ", %%mm0          \n\t" /* R4     R0      r4      r0 */\
 365         "movq " #src4 ", %%mm1          \n\t" /* R6     R2      r6      r2 */\
 366         "movq " #src1 ", %%mm2          \n\t" /* R3     R1      r3      r1 */\
 367         "movq " #src5 ", %%mm3          \n\t" /* R7     R5      r7      r5 */\
 368         "movq "MANGLE(wm1010)", %%mm4   \n\t"\
 369         "pand %%mm0, %%mm4              \n\t"\
 370         "por %%mm1, %%mm4               \n\t"\
 371         "por %%mm2, %%mm4               \n\t"\
 372         "por %%mm3, %%mm4               \n\t"\
 373         "packssdw %%mm4,%%mm4           \n\t"\
 374         "movd %%mm4, %%eax              \n\t"\
 375         "orl %%eax, %%eax               \n\t"\
 376         "jz 1f                          \n\t"\
 377         "movq 16(%2), %%mm4             \n\t" /* C4     C4      C4      C4 */\
 378         "pmaddwd %%mm0, %%mm4           \n\t" /* C4R4+C4R0      C4r4+C4r0 */\
 379         "movq 24(%2), %%mm5             \n\t" /* -C4    C4      -C4     C4 */\
 380         "pmaddwd %%mm5, %%mm0           \n\t" /* -C4R4+C4R0     -C4r4+C4r0 */\
 381         "movq 32(%2), %%mm5             \n\t" /* C6     C2      C6      C2 */\
 382         "pmaddwd %%mm1, %%mm5           \n\t" /* C6R6+C2R2      C6r6+C2r2 */\
 383         "movq 40(%2), %%mm6             \n\t" /* -C2    C6      -C2     C6 */\
 384         "pmaddwd %%mm6, %%mm1           \n\t" /* -C2R6+C6R2     -C2r6+C6r2 */\
 385         "movq 48(%2), %%mm7             \n\t" /* C3     C1      C3      C1 */\
 386         "pmaddwd %%mm2, %%mm7           \n\t" /* C3R3+C1R1      C3r3+C1r1 */\
 387         #rounder ", %%mm4               \n\t"\
 388         "movq %%mm4, %%mm6              \n\t" /* C4R4+C4R0      C4r4+C4r0 */\
 389         "paddd %%mm5, %%mm4             \n\t" /* A0             a0 */\
 390         "psubd %%mm5, %%mm6             \n\t" /* A3             a3 */\
 391         "movq 56(%2), %%mm5             \n\t" /* C7     C5      C7      C5 */\
 392         "pmaddwd %%mm3, %%mm5           \n\t" /* C7R7+C5R5      C7r7+C5r5 */\
 393         #rounder ", %%mm0               \n\t"\
 394         "paddd %%mm0, %%mm1             \n\t" /* A1             a1 */\
 395         "paddd %%mm0, %%mm0             \n\t" \
 396         "psubd %%mm1, %%mm0             \n\t" /* A2             a2 */\
 397         "pmaddwd 64(%2), %%mm2          \n\t" /* -C7R3+C3R1     -C7r3+C3r1 */\
 398         "paddd %%mm5, %%mm7             \n\t" /* B0             b0 */\
 399         "movq 72(%2), %%mm5             \n\t" /* -C5    -C1     -C5     -C1 */\
 400         "pmaddwd %%mm3, %%mm5           \n\t" /* -C5R7-C1R5     -C5r7-C1r5 */\
 401         "paddd %%mm4, %%mm7             \n\t" /* A0+B0          a0+b0 */\
 402         "paddd %%mm4, %%mm4             \n\t" /* 2A0            2a0 */\
 403         "psubd %%mm7, %%mm4             \n\t" /* A0-B0          a0-b0 */\
 404         "paddd %%mm2, %%mm5             \n\t" /* B1             b1 */\
 405         "psrad $" #shift ", %%mm7       \n\t"\
 406         "psrad $" #shift ", %%mm4       \n\t"\
 407         "movq %%mm1, %%mm2              \n\t" /* A1             a1 */\
 408         "paddd %%mm5, %%mm1             \n\t" /* A1+B1          a1+b1 */\
 409         "psubd %%mm5, %%mm2             \n\t" /* A1-B1          a1-b1 */\
 410         "psrad $" #shift ", %%mm1       \n\t"\
 411         "psrad $" #shift ", %%mm2       \n\t"\
 412         "packssdw %%mm1, %%mm7          \n\t" /* A1+B1  a1+b1   A0+B0   a0+b0 */\
 413         "packssdw %%mm4, %%mm2          \n\t" /* A0-B0  a0-b0   A1-B1   a1-b1 */\
 414         "movq %%mm7, " #dst "           \n\t"\
 415         "movq " #src1 ", %%mm1          \n\t" /* R3     R1      r3      r1 */\
 416         "movq 80(%2), %%mm4             \n\t" /* -C1    C5      -C1     C5 */\
 417         "movq %%mm2, 24+" #dst "        \n\t"\
 418         "pmaddwd %%mm1, %%mm4           \n\t" /* -C1R3+C5R1     -C1r3+C5r1 */\
 419         "movq 88(%2), %%mm7             \n\t" /* C3     C7      C3      C7 */\
 420         "pmaddwd 96(%2), %%mm1          \n\t" /* -C5R3+C7R1     -C5r3+C7r1 */\
 421         "pmaddwd %%mm3, %%mm7           \n\t" /* C3R7+C7R5      C3r7+C7r5 */\
 422         "movq %%mm0, %%mm2              \n\t" /* A2             a2 */\
 423         "pmaddwd 104(%2), %%mm3         \n\t" /* -C1R7+C3R5     -C1r7+C3r5 */\
 424         "paddd %%mm7, %%mm4             \n\t" /* B2             b2 */\
 425         "paddd %%mm4, %%mm2             \n\t" /* A2+B2          a2+b2 */\
 426         "psubd %%mm4, %%mm0             \n\t" /* a2-B2          a2-b2 */\
 427         "psrad $" #shift ", %%mm2       \n\t"\
 428         "psrad $" #shift ", %%mm0       \n\t"\
 429         "movq %%mm6, %%mm4              \n\t" /* A3             a3 */\
 430         "paddd %%mm1, %%mm3             \n\t" /* B3             b3 */\
 431         "paddd %%mm3, %%mm6             \n\t" /* A3+B3          a3+b3 */\
 432         "psubd %%mm3, %%mm4             \n\t" /* a3-B3          a3-b3 */\
 433         "psrad $" #shift ", %%mm6       \n\t"\
 434         "packssdw %%mm6, %%mm2          \n\t" /* A3+B3  a3+b3   A2+B2   a2+b2 */\
 435         "movq %%mm2, 8+" #dst "         \n\t"\
 436         "psrad $" #shift ", %%mm4       \n\t"\
 437         "packssdw %%mm0, %%mm4          \n\t" /* A2-B2  a2-b2   A3-B3   a3-b3 */\
 438         "movq %%mm4, 16+" #dst "        \n\t"\
 439         "jmp 2f                         \n\t"\
 440         "1:                             \n\t"\
 441         "pslld $16, %%mm0               \n\t"\
 442         "#paddd "MANGLE(d40000)", %%mm0 \n\t"\
 443         "psrad $13, %%mm0               \n\t"\
 444         "packssdw %%mm0, %%mm0          \n\t"\
 445         "movq %%mm0, " #dst "           \n\t"\
 446         "movq %%mm0, 8+" #dst "         \n\t"\
 447         "movq %%mm0, 16+" #dst "        \n\t"\
 448         "movq %%mm0, 24+" #dst "        \n\t"\
 449         "2:                             \n\t"
 450
 451
 452 //IDCT(      src0,   src4,   src1,   src5,    dst,    rounder, shift)
 453 ROW_IDCT(    (%0),  8(%0), 16(%0), 24(%0),  0(%1),paddd 8(%2), 11)
 454 /*ROW_IDCT(  32(%0), 40(%0), 48(%0), 56(%0), 32(%1), paddd (%2), 11)
 455 ROW_IDCT(  64(%0), 72(%0), 80(%0), 88(%0), 64(%1), paddd (%2), 11)
 456 ROW_IDCT(  96(%0),104(%0),112(%0),120(%0), 96(%1), paddd (%2), 11)*/
 457
 458 DC_COND_ROW_IDCT(  32(%0), 40(%0), 48(%0), 56(%0), 32(%1),paddd (%2), 11)
 459 DC_COND_ROW_IDCT(  64(%0), 72(%0), 80(%0), 88(%0), 64(%1),paddd (%2), 11)
 460 DC_COND_ROW_IDCT(  96(%0),104(%0),112(%0),120(%0), 96(%1),paddd (%2), 11)
 461
 462
 463 //IDCT(      src0,   src4,   src1,    src5,    dst, shift)
 464 COL_IDCT(    (%1), 64(%1), 32(%1),  96(%1),  0(%0), 20)
 465 COL_IDCT(   8(%1), 72(%1), 40(%1), 104(%1),  4(%0), 20)
 466 COL_IDCT(  16(%1), 80(%1), 48(%1), 112(%1),  8(%0), 20)
 467 COL_IDCT(  24(%1), 88(%1), 56(%1), 120(%1), 12(%0), 20)
 468
 469 #else
 470
 471 #define DC_COND_IDCT(src0, src4, src1, src5, dst, rounder, shift) \
 472         "movq " #src0 ", %%mm0          \n\t" /* R4     R0      r4      r0 */\
 473         "movq " #src4 ", %%mm1          \n\t" /* R6     R2      r6      r2 */\
 474         "movq " #src1 ", %%mm2          \n\t" /* R3     R1      r3      r1 */\
 475         "movq " #src5 ", %%mm3          \n\t" /* R7     R5      r7      r5 */\
 476         "movq "MANGLE(wm1010)", %%mm4   \n\t"\
 477         "pand %%mm0, %%mm4              \n\t"\
 478         "por %%mm1, %%mm4               \n\t"\
 479         "por %%mm2, %%mm4               \n\t"\
 480         "por %%mm3, %%mm4               \n\t"\
 481         "packssdw %%mm4,%%mm4           \n\t"\
 482         "movd %%mm4, %%eax              \n\t"\
 483         "orl %%eax, %%eax               \n\t"\
 484         "jz 1f                          \n\t"\
 485         "movq 16(%2), %%mm4             \n\t" /* C4     C4      C4      C4 */\
 486         "pmaddwd %%mm0, %%mm4           \n\t" /* C4R4+C4R0      C4r4+C4r0 */\
 487         "movq 24(%2), %%mm5             \n\t" /* -C4    C4      -C4     C4 */\
 488         "pmaddwd %%mm5, %%mm0           \n\t" /* -C4R4+C4R0     -C4r4+C4r0 */\
 489         "movq 32(%2), %%mm5             \n\t" /* C6     C2      C6      C2 */\
 490         "pmaddwd %%mm1, %%mm5           \n\t" /* C6R6+C2R2      C6r6+C2r2 */\
 491         "movq 40(%2), %%mm6             \n\t" /* -C2    C6      -C2     C6 */\
 492         "pmaddwd %%mm6, %%mm1           \n\t" /* -C2R6+C6R2     -C2r6+C6r2 */\
 493         "movq 48(%2), %%mm7             \n\t" /* C3     C1      C3      C1 */\
 494         "pmaddwd %%mm2, %%mm7           \n\t" /* C3R3+C1R1      C3r3+C1r1 */\
 495         #rounder ", %%mm4               \n\t"\
 496         "movq %%mm4, %%mm6              \n\t" /* C4R4+C4R0      C4r4+C4r0 */\
 497         "paddd %%mm5, %%mm4             \n\t" /* A0             a0 */\
 498         "psubd %%mm5, %%mm6             \n\t" /* A3             a3 */\
 499         "movq 56(%2), %%mm5             \n\t" /* C7     C5      C7      C5 */\
 500         "pmaddwd %%mm3, %%mm5           \n\t" /* C7R7+C5R5      C7r7+C5r5 */\
 501         #rounder ", %%mm0               \n\t"\
 502         "paddd %%mm0, %%mm1             \n\t" /* A1             a1 */\
 503         "paddd %%mm0, %%mm0             \n\t" \
 504         "psubd %%mm1, %%mm0             \n\t" /* A2             a2 */\
 505         "pmaddwd 64(%2), %%mm2          \n\t" /* -C7R3+C3R1     -C7r3+C3r1 */\
 506         "paddd %%mm5, %%mm7             \n\t" /* B0             b0 */\
 507         "movq 72(%2), %%mm5             \n\t" /* -C5    -C1     -C5     -C1 */\
 508         "pmaddwd %%mm3, %%mm5           \n\t" /* -C5R7-C1R5     -C5r7-C1r5 */\
 509         "paddd %%mm4, %%mm7             \n\t" /* A0+B0          a0+b0 */\
 510         "paddd %%mm4, %%mm4             \n\t" /* 2A0            2a0 */\
 511         "psubd %%mm7, %%mm4             \n\t" /* A0-B0          a0-b0 */\
 512         "paddd %%mm2, %%mm5             \n\t" /* B1             b1 */\
 513         "psrad $" #shift ", %%mm7       \n\t"\
 514         "psrad $" #shift ", %%mm4       \n\t"\
 515         "movq %%mm1, %%mm2              \n\t" /* A1             a1 */\
 516         "paddd %%mm5, %%mm1             \n\t" /* A1+B1          a1+b1 */\
 517         "psubd %%mm5, %%mm2             \n\t" /* A1-B1          a1-b1 */\
 518         "psrad $" #shift ", %%mm1       \n\t"\
 519         "psrad $" #shift ", %%mm2       \n\t"\
 520         "packssdw %%mm1, %%mm7          \n\t" /* A1+B1  a1+b1   A0+B0   a0+b0 */\
 521         "packssdw %%mm4, %%mm2          \n\t" /* A0-B0  a0-b0   A1-B1   a1-b1 */\
 522         "movq %%mm7, " #dst "           \n\t"\
 523         "movq " #src1 ", %%mm1          \n\t" /* R3     R1      r3      r1 */\
 524         "movq 80(%2), %%mm4             \n\t" /* -C1    C5      -C1     C5 */\
 525         "movq %%mm2, 24+" #dst "        \n\t"\
 526         "pmaddwd %%mm1, %%mm4           \n\t" /* -C1R3+C5R1     -C1r3+C5r1 */\
 527         "movq 88(%2), %%mm7             \n\t" /* C3     C7      C3      C7 */\
 528         "pmaddwd 96(%2), %%mm1          \n\t" /* -C5R3+C7R1     -C5r3+C7r1 */\
 529         "pmaddwd %%mm3, %%mm7           \n\t" /* C3R7+C7R5      C3r7+C7r5 */\
 530         "movq %%mm0, %%mm2              \n\t" /* A2             a2 */\
 531         "pmaddwd 104(%2), %%mm3         \n\t" /* -C1R7+C3R5     -C1r7+C3r5 */\
 532         "paddd %%mm7, %%mm4             \n\t" /* B2             b2 */\
 533         "paddd %%mm4, %%mm2             \n\t" /* A2+B2          a2+b2 */\
 534         "psubd %%mm4, %%mm0             \n\t" /* a2-B2          a2-b2 */\
 535         "psrad $" #shift ", %%mm2       \n\t"\
 536         "psrad $" #shift ", %%mm0       \n\t"\
 537         "movq %%mm6, %%mm4              \n\t" /* A3             a3 */\
 538         "paddd %%mm1, %%mm3             \n\t" /* B3             b3 */\
 539         "paddd %%mm3, %%mm6             \n\t" /* A3+B3          a3+b3 */\
 540         "psubd %%mm3, %%mm4             \n\t" /* a3-B3          a3-b3 */\
 541         "psrad $" #shift ", %%mm6       \n\t"\
 542         "packssdw %%mm6, %%mm2          \n\t" /* A3+B3  a3+b3   A2+B2   a2+b2 */\
 543         "movq %%mm2, 8+" #dst "         \n\t"\
 544         "psrad $" #shift ", %%mm4       \n\t"\
 545         "packssdw %%mm0, %%mm4          \n\t" /* A2-B2  a2-b2   A3-B3   a3-b3 */\
 546         "movq %%mm4, 16+" #dst "        \n\t"\
 547         "jmp 2f                         \n\t"\
 548         "1:                             \n\t"\
 549         "pslld $16, %%mm0               \n\t"\
 550         "paddd "MANGLE(d40000)", %%mm0  \n\t"\
 551         "psrad $13, %%mm0               \n\t"\
 552         "packssdw %%mm0, %%mm0          \n\t"\
 553         "movq %%mm0, " #dst "           \n\t"\
 554         "movq %%mm0, 8+" #dst "         \n\t"\
 555         "movq %%mm0, 16+" #dst "        \n\t"\
 556         "movq %%mm0, 24+" #dst "        \n\t"\
 557         "2:                             \n\t"
 558
 559 #define Z_COND_IDCT(src0, src4, src1, src5, dst, rounder, shift, bt) \
 560         "movq " #src0 ", %%mm0          \n\t" /* R4     R0      r4      r0 */\
 561         "movq " #src4 ", %%mm1          \n\t" /* R6     R2      r6      r2 */\
 562         "movq " #src1 ", %%mm2          \n\t" /* R3     R1      r3      r1 */\
 563         "movq " #src5 ", %%mm3          \n\t" /* R7     R5      r7      r5 */\
 564         "movq %%mm0, %%mm4              \n\t"\
 565         "por %%mm1, %%mm4               \n\t"\
 566         "por %%mm2, %%mm4               \n\t"\
 567         "por %%mm3, %%mm4               \n\t"\
 568         "packssdw %%mm4,%%mm4           \n\t"\
 569         "movd %%mm4, %%eax              \n\t"\
 570         "orl %%eax, %%eax               \n\t"\
 571         "jz " #bt "                     \n\t"\
 572         "movq 16(%2), %%mm4             \n\t" /* C4     C4      C4      C4 */\
 573         "pmaddwd %%mm0, %%mm4           \n\t" /* C4R4+C4R0      C4r4+C4r0 */\
 574         "movq 24(%2), %%mm5             \n\t" /* -C4    C4      -C4     C4 */\
 575         "pmaddwd %%mm5, %%mm0           \n\t" /* -C4R4+C4R0     -C4r4+C4r0 */\
 576         "movq 32(%2), %%mm5             \n\t" /* C6     C2      C6      C2 */\
 577         "pmaddwd %%mm1, %%mm5           \n\t" /* C6R6+C2R2      C6r6+C2r2 */\
 578         "movq 40(%2), %%mm6             \n\t" /* -C2    C6      -C2     C6 */\
 579         "pmaddwd %%mm6, %%mm1           \n\t" /* -C2R6+C6R2     -C2r6+C6r2 */\
 580         "movq 48(%2), %%mm7             \n\t" /* C3     C1      C3      C1 */\
 581         "pmaddwd %%mm2, %%mm7           \n\t" /* C3R3+C1R1      C3r3+C1r1 */\
 582         #rounder ", %%mm4               \n\t"\
 583         "movq %%mm4, %%mm6              \n\t" /* C4R4+C4R0      C4r4+C4r0 */\
 584         "paddd %%mm5, %%mm4             \n\t" /* A0             a0 */\
 585         "psubd %%mm5, %%mm6             \n\t" /* A3             a3 */\
 586         "movq 56(%2), %%mm5             \n\t" /* C7     C5      C7      C5 */\
 587         "pmaddwd %%mm3, %%mm5           \n\t" /* C7R7+C5R5      C7r7+C5r5 */\
 588         #rounder ", %%mm0               \n\t"\
 589         "paddd %%mm0, %%mm1             \n\t" /* A1             a1 */\
 590         "paddd %%mm0, %%mm0             \n\t" \
 591         "psubd %%mm1, %%mm0             \n\t" /* A2             a2 */\
 592         "pmaddwd 64(%2), %%mm2          \n\t" /* -C7R3+C3R1     -C7r3+C3r1 */\
 593         "paddd %%mm5, %%mm7             \n\t" /* B0             b0 */\
 594         "movq 72(%2), %%mm5             \n\t" /* -C5    -C1     -C5     -C1 */\
 595         "pmaddwd %%mm3, %%mm5           \n\t" /* -C5R7-C1R5     -C5r7-C1r5 */\
 596         "paddd %%mm4, %%mm7             \n\t" /* A0+B0          a0+b0 */\
 597         "paddd %%mm4, %%mm4             \n\t" /* 2A0            2a0 */\
 598         "psubd %%mm7, %%mm4             \n\t" /* A0-B0          a0-b0 */\
 599         "paddd %%mm2, %%mm5             \n\t" /* B1             b1 */\
 600         "psrad $" #shift ", %%mm7       \n\t"\
 601         "psrad $" #shift ", %%mm4       \n\t"\
 602         "movq %%mm1, %%mm2              \n\t" /* A1             a1 */\
 603         "paddd %%mm5, %%mm1             \n\t" /* A1+B1          a1+b1 */\
 604         "psubd %%mm5, %%mm2             \n\t" /* A1-B1          a1-b1 */\
 605         "psrad $" #shift ", %%mm1       \n\t"\
 606         "psrad $" #shift ", %%mm2       \n\t"\
 607         "packssdw %%mm1, %%mm7          \n\t" /* A1+B1  a1+b1   A0+B0   a0+b0 */\
 608         "packssdw %%mm4, %%mm2          \n\t" /* A0-B0  a0-b0   A1-B1   a1-b1 */\
 609         "movq %%mm7, " #dst "           \n\t"\
 610         "movq " #src1 ", %%mm1          \n\t" /* R3     R1      r3      r1 */\
 611         "movq 80(%2), %%mm4             \n\t" /* -C1    C5      -C1     C5 */\
 612         "movq %%mm2, 24+" #dst "        \n\t"\
 613         "pmaddwd %%mm1, %%mm4           \n\t" /* -C1R3+C5R1     -C1r3+C5r1 */\
 614         "movq 88(%2), %%mm7             \n\t" /* C3     C7      C3      C7 */\
 615         "pmaddwd 96(%2), %%mm1          \n\t" /* -C5R3+C7R1     -C5r3+C7r1 */\
 616         "pmaddwd %%mm3, %%mm7           \n\t" /* C3R7+C7R5      C3r7+C7r5 */\
 617         "movq %%mm0, %%mm2              \n\t" /* A2             a2 */\
 618         "pmaddwd 104(%2), %%mm3         \n\t" /* -C1R7+C3R5     -C1r7+C3r5 */\
 619         "paddd %%mm7, %%mm4             \n\t" /* B2             b2 */\
 620         "paddd %%mm4, %%mm2             \n\t" /* A2+B2          a2+b2 */\
 621         "psubd %%mm4, %%mm0             \n\t" /* a2-B2          a2-b2 */\
 622         "psrad $" #shift ", %%mm2       \n\t"\
 623         "psrad $" #shift ", %%mm0       \n\t"\
 624         "movq %%mm6, %%mm4              \n\t" /* A3             a3 */\
 625         "paddd %%mm1, %%mm3             \n\t" /* B3             b3 */\
 626         "paddd %%mm3, %%mm6             \n\t" /* A3+B3          a3+b3 */\
 627         "psubd %%mm3, %%mm4             \n\t" /* a3-B3          a3-b3 */\
 628         "psrad $" #shift ", %%mm6       \n\t"\
 629         "packssdw %%mm6, %%mm2          \n\t" /* A3+B3  a3+b3   A2+B2   a2+b2 */\
 630         "movq %%mm2, 8+" #dst "         \n\t"\
 631         "psrad $" #shift ", %%mm4       \n\t"\
 632         "packssdw %%mm0, %%mm4          \n\t" /* A2-B2  a2-b2   A3-B3   a3-b3 */\
 633         "movq %%mm4, 16+" #dst "        \n\t"\
 634
 635 #define ROW_IDCT(src0, src4, src1, src5, dst, rounder, shift) \
 636         "movq " #src0 ", %%mm0          \n\t" /* R4     R0      r4      r0 */\
 637         "movq " #src4 ", %%mm1          \n\t" /* R6     R2      r6      r2 */\
 638         "movq " #src1 ", %%mm2          \n\t" /* R3     R1      r3      r1 */\
 639         "movq " #src5 ", %%mm3          \n\t" /* R7     R5      r7      r5 */\
 640         "movq 16(%2), %%mm4             \n\t" /* C4     C4      C4      C4 */\
 641         "pmaddwd %%mm0, %%mm4           \n\t" /* C4R4+C4R0      C4r4+C4r0 */\
 642         "movq 24(%2), %%mm5             \n\t" /* -C4    C4      -C4     C4 */\
 643         "pmaddwd %%mm5, %%mm0           \n\t" /* -C4R4+C4R0     -C4r4+C4r0 */\
 644         "movq 32(%2), %%mm5             \n\t" /* C6     C2      C6      C2 */\
 645         "pmaddwd %%mm1, %%mm5           \n\t" /* C6R6+C2R2      C6r6+C2r2 */\
 646         "movq 40(%2), %%mm6             \n\t" /* -C2    C6      -C2     C6 */\
 647         "pmaddwd %%mm6, %%mm1           \n\t" /* -C2R6+C6R2     -C2r6+C6r2 */\
 648         "movq 48(%2), %%mm7             \n\t" /* C3     C1      C3      C1 */\
 649         "pmaddwd %%mm2, %%mm7           \n\t" /* C3R3+C1R1      C3r3+C1r1 */\
 650         #rounder ", %%mm4               \n\t"\
 651         "movq %%mm4, %%mm6              \n\t" /* C4R4+C4R0      C4r4+C4r0 */\
 652         "paddd %%mm5, %%mm4             \n\t" /* A0             a0 */\
 653         "psubd %%mm5, %%mm6             \n\t" /* A3             a3 */\
 654         "movq 56(%2), %%mm5             \n\t" /* C7     C5      C7      C5 */\
 655         "pmaddwd %%mm3, %%mm5           \n\t" /* C7R7+C5R5      C7r7+C5r5 */\
 656         #rounder ", %%mm0               \n\t"\
 657         "paddd %%mm0, %%mm1             \n\t" /* A1             a1 */\
 658         "paddd %%mm0, %%mm0             \n\t" \
 659         "psubd %%mm1, %%mm0             \n\t" /* A2             a2 */\
 660         "pmaddwd 64(%2), %%mm2          \n\t" /* -C7R3+C3R1     -C7r3+C3r1 */\
 661         "paddd %%mm5, %%mm7             \n\t" /* B0             b0 */\
 662         "movq 72(%2), %%mm5             \n\t" /* -C5    -C1     -C5     -C1 */\
 663         "pmaddwd %%mm3, %%mm5           \n\t" /* -C5R7-C1R5     -C5r7-C1r5 */\
 664         "paddd %%mm4, %%mm7             \n\t" /* A0+B0          a0+b0 */\
 665         "paddd %%mm4, %%mm4             \n\t" /* 2A0            2a0 */\
 666         "psubd %%mm7, %%mm4             \n\t" /* A0-B0          a0-b0 */\
 667         "paddd %%mm2, %%mm5             \n\t" /* B1             b1 */\
 668         "psrad $" #shift ", %%mm7       \n\t"\
 669         "psrad $" #shift ", %%mm4       \n\t"\
 670         "movq %%mm1, %%mm2              \n\t" /* A1             a1 */\
 671         "paddd %%mm5, %%mm1             \n\t" /* A1+B1          a1+b1 */\
 672         "psubd %%mm5, %%mm2             \n\t" /* A1-B1          a1-b1 */\
 673         "psrad $" #shift ", %%mm1       \n\t"\
 674         "psrad $" #shift ", %%mm2       \n\t"\
 675         "packssdw %%mm1, %%mm7          \n\t" /* A1+B1  a1+b1   A0+B0   a0+b0 */\
 676         "packssdw %%mm4, %%mm2          \n\t" /* A0-B0  a0-b0   A1-B1   a1-b1 */\
 677         "movq %%mm7, " #dst "           \n\t"\
 678         "movq " #src1 ", %%mm1          \n\t" /* R3     R1      r3      r1 */\
 679         "movq 80(%2), %%mm4             \n\t" /* -C1    C5      -C1     C5 */\
 680         "movq %%mm2, 24+" #dst "        \n\t"\
 681         "pmaddwd %%mm1, %%mm4           \n\t" /* -C1R3+C5R1     -C1r3+C5r1 */\
 682         "movq 88(%2), %%mm7             \n\t" /* C3     C7      C3      C7 */\
 683         "pmaddwd 96(%2), %%mm1          \n\t" /* -C5R3+C7R1     -C5r3+C7r1 */\
 684         "pmaddwd %%mm3, %%mm7           \n\t" /* C3R7+C7R5      C3r7+C7r5 */\
 685         "movq %%mm0, %%mm2              \n\t" /* A2             a2 */\
 686         "pmaddwd 104(%2), %%mm3         \n\t" /* -C1R7+C3R5     -C1r7+C3r5 */\
 687         "paddd %%mm7, %%mm4             \n\t" /* B2             b2 */\
 688         "paddd %%mm4, %%mm2             \n\t" /* A2+B2          a2+b2 */\
 689         "psubd %%mm4, %%mm0             \n\t" /* a2-B2          a2-b2 */\
 690         "psrad $" #shift ", %%mm2       \n\t"\
 691         "psrad $" #shift ", %%mm0       \n\t"\
 692         "movq %%mm6, %%mm4              \n\t" /* A3             a3 */\
 693         "paddd %%mm1, %%mm3             \n\t" /* B3             b3 */\
 694         "paddd %%mm3, %%mm6             \n\t" /* A3+B3          a3+b3 */\
 695         "psubd %%mm3, %%mm4             \n\t" /* a3-B3          a3-b3 */\
 696         "psrad $" #shift ", %%mm6       \n\t"\
 697         "packssdw %%mm6, %%mm2          \n\t" /* A3+B3  a3+b3   A2+B2   a2+b2 */\
 698         "movq %%mm2, 8+" #dst "         \n\t"\
 699         "psrad $" #shift ", %%mm4       \n\t"\
 700         "packssdw %%mm0, %%mm4          \n\t" /* A2-B2  a2-b2   A3-B3   a3-b3 */\
 701         "movq %%mm4, 16+" #dst "        \n\t"\
 702
 703 //IDCT(         src0,   src4,   src1,   src5,    dst,   rounder, shift)
 704 DC_COND_IDCT(  0(%0),  8(%0), 16(%0), 24(%0),  0(%1),paddd 8(%2), 11)
 705 Z_COND_IDCT(  32(%0), 40(%0), 48(%0), 56(%0), 32(%1),paddd (%2), 11, 4f)
 706 Z_COND_IDCT(  64(%0), 72(%0), 80(%0), 88(%0), 64(%1),paddd (%2), 11, 2f)
 707 Z_COND_IDCT(  96(%0),104(%0),112(%0),120(%0), 96(%1),paddd (%2), 11, 1f)
 708
 709 #undef IDCT
 710 #define IDCT(src0, src4, src1, src5, dst, shift) \
 711         "movq " #src0 ", %%mm0          \n\t" /* R4     R0      r4      r0 */\
 712         "movq " #src4 ", %%mm1          \n\t" /* R6     R2      r6      r2 */\
 713         "movq " #src1 ", %%mm2          \n\t" /* R3     R1      r3      r1 */\
 714         "movq " #src5 ", %%mm3          \n\t" /* R7     R5      r7      r5 */\
 715         "movq 16(%2), %%mm4             \n\t" /* C4     C4      C4      C4 */\
 716         "pmaddwd %%mm0, %%mm4           \n\t" /* C4R4+C4R0      C4r4+C4r0 */\
 717         "movq 24(%2), %%mm5             \n\t" /* -C4    C4      -C4     C4 */\
 718         "pmaddwd %%mm5, %%mm0           \n\t" /* -C4R4+C4R0     -C4r4+C4r0 */\
 719         "movq 32(%2), %%mm5             \n\t" /* C6     C2      C6      C2 */\
 720         "pmaddwd %%mm1, %%mm5           \n\t" /* C6R6+C2R2      C6r6+C2r2 */\
 721         "movq 40(%2), %%mm6             \n\t" /* -C2    C6      -C2     C6 */\
 722         "pmaddwd %%mm6, %%mm1           \n\t" /* -C2R6+C6R2     -C2r6+C6r2 */\
 723         "movq %%mm4, %%mm6              \n\t" /* C4R4+C4R0      C4r4+C4r0 */\
 724         "movq 48(%2), %%mm7             \n\t" /* C3     C1      C3      C1 */\
 725         "pmaddwd %%mm2, %%mm7           \n\t" /* C3R3+C1R1      C3r3+C1r1 */\
 726         "paddd %%mm5, %%mm4             \n\t" /* A0             a0 */\
 727         "psubd %%mm5, %%mm6             \n\t" /* A3             a3 */\
 728         "movq %%mm0, %%mm5              \n\t" /* -C4R4+C4R0     -C4r4+C4r0 */\
 729         "paddd %%mm1, %%mm0             \n\t" /* A1             a1 */\
 730         "psubd %%mm1, %%mm5             \n\t" /* A2             a2 */\
 731         "movq 56(%2), %%mm1             \n\t" /* C7     C5      C7      C5 */\
 732         "pmaddwd %%mm3, %%mm1           \n\t" /* C7R7+C5R5      C7r7+C5r5 */\
 733         "pmaddwd 64(%2), %%mm2          \n\t" /* -C7R3+C3R1     -C7r3+C3r1 */\
 734         "paddd %%mm1, %%mm7             \n\t" /* B0             b0 */\
 735         "movq 72(%2), %%mm1             \n\t" /* -C5    -C1     -C5     -C1 */\
 736         "pmaddwd %%mm3, %%mm1           \n\t" /* -C5R7-C1R5     -C5r7-C1r5 */\
 737         "paddd %%mm4, %%mm7             \n\t" /* A0+B0          a0+b0 */\
 738         "paddd %%mm4, %%mm4             \n\t" /* 2A0            2a0 */\
 739         "psubd %%mm7, %%mm4             \n\t" /* A0-B0          a0-b0 */\
 740         "paddd %%mm2, %%mm1             \n\t" /* B1             b1 */\
 741         "psrad $" #shift ", %%mm7       \n\t"\
 742         "psrad $" #shift ", %%mm4       \n\t"\
 743         "movq %%mm0, %%mm2              \n\t" /* A1             a1 */\
 744         "paddd %%mm1, %%mm0             \n\t" /* A1+B1          a1+b1 */\
 745         "psubd %%mm1, %%mm2             \n\t" /* A1-B1          a1-b1 */\
 746         "psrad $" #shift ", %%mm0       \n\t"\
 747         "psrad $" #shift ", %%mm2       \n\t"\
 748         "packssdw %%mm7, %%mm7          \n\t" /* A0+B0  a0+b0 */\
 749         "movd %%mm7, " #dst "           \n\t"\
 750         "packssdw %%mm0, %%mm0          \n\t" /* A1+B1  a1+b1 */\
 751         "movd %%mm0, 16+" #dst "        \n\t"\
 752         "packssdw %%mm2, %%mm2          \n\t" /* A1-B1  a1-b1 */\
 753         "movd %%mm2, 96+" #dst "        \n\t"\
 754         "packssdw %%mm4, %%mm4          \n\t" /* A0-B0  a0-b0 */\
 755         "movd %%mm4, 112+" #dst "       \n\t"\
 756         "movq " #src1 ", %%mm0          \n\t" /* R3     R1      r3      r1 */\
 757         "movq 80(%2), %%mm4             \n\t" /* -C1    C5      -C1     C5 */\
 758         "pmaddwd %%mm0, %%mm4           \n\t" /* -C1R3+C5R1     -C1r3+C5r1 */\
 759         "movq 88(%2), %%mm7             \n\t" /* C3     C7      C3      C7 */\
 760         "pmaddwd 96(%2), %%mm0          \n\t" /* -C5R3+C7R1     -C5r3+C7r1 */\
 761         "pmaddwd %%mm3, %%mm7           \n\t" /* C3R7+C7R5      C3r7+C7r5 */\
 762         "movq %%mm5, %%mm2              \n\t" /* A2             a2 */\
 763         "pmaddwd 104(%2), %%mm3         \n\t" /* -C1R7+C3R5     -C1r7+C3r5 */\
 764         "paddd %%mm7, %%mm4             \n\t" /* B2             b2 */\
 765         "paddd %%mm4, %%mm2             \n\t" /* A2+B2          a2+b2 */\
 766         "psubd %%mm4, %%mm5             \n\t" /* a2-B2          a2-b2 */\
 767         "psrad $" #shift ", %%mm2       \n\t"\
 768         "psrad $" #shift ", %%mm5       \n\t"\
 769         "movq %%mm6, %%mm4              \n\t" /* A3             a3 */\
 770         "paddd %%mm0, %%mm3             \n\t" /* B3             b3 */\
 771         "paddd %%mm3, %%mm6             \n\t" /* A3+B3          a3+b3 */\
 772         "psubd %%mm3, %%mm4             \n\t" /* a3-B3          a3-b3 */\
 773         "psrad $" #shift ", %%mm6       \n\t"\
 774         "psrad $" #shift ", %%mm4       \n\t"\
 775         "packssdw %%mm2, %%mm2          \n\t" /* A2+B2  a2+b2 */\
 776         "packssdw %%mm6, %%mm6          \n\t" /* A3+B3  a3+b3 */\
 777         "movd %%mm2, 32+" #dst "        \n\t"\
 778         "packssdw %%mm4, %%mm4          \n\t" /* A3-B3  a3-b3 */\
 779         "packssdw %%mm5, %%mm5          \n\t" /* A2-B2  a2-b2 */\
 780         "movd %%mm6, 48+" #dst "        \n\t"\
 781         "movd %%mm4, 64+" #dst "        \n\t"\
 782         "movd %%mm5, 80+" #dst "        \n\t"
 783
 784
 785 //IDCT(  src0,   src4,   src1,    src5,    dst, shift)
 786 IDCT(    (%1), 64(%1), 32(%1),  96(%1),  0(%0), 20)
 787 IDCT(   8(%1), 72(%1), 40(%1), 104(%1),  4(%0), 20)
 788 IDCT(  16(%1), 80(%1), 48(%1), 112(%1),  8(%0), 20)
 789 IDCT(  24(%1), 88(%1), 56(%1), 120(%1), 12(%0), 20)
 790         "jmp 9f                         \n\t"
 791
 792         "# .p2align 4                   \n\t"\
 793         "4:                             \n\t"
 794 Z_COND_IDCT(  64(%0), 72(%0), 80(%0), 88(%0), 64(%1),paddd (%2), 11, 6f)
 795 Z_COND_IDCT(  96(%0),104(%0),112(%0),120(%0), 96(%1),paddd (%2), 11, 5f)
 796
 797 #undef IDCT
 798 #define IDCT(src0, src4, src1, src5, dst, shift) \
 799         "movq " #src0 ", %%mm0          \n\t" /* R4     R0      r4      r0 */\
 800         "movq " #src4 ", %%mm1          \n\t" /* R6     R2      r6      r2 */\
 801         "movq " #src5 ", %%mm3          \n\t" /* R7     R5      r7      r5 */\
 802         "movq 16(%2), %%mm4             \n\t" /* C4     C4      C4      C4 */\
 803         "pmaddwd %%mm0, %%mm4           \n\t" /* C4R4+C4R0      C4r4+C4r0 */\
 804         "movq 24(%2), %%mm5             \n\t" /* -C4    C4      -C4     C4 */\
 805         "pmaddwd %%mm5, %%mm0           \n\t" /* -C4R4+C4R0     -C4r4+C4r0 */\
 806         "movq 32(%2), %%mm5             \n\t" /* C6     C2      C6      C2 */\
 807         "pmaddwd %%mm1, %%mm5           \n\t" /* C6R6+C2R2      C6r6+C2r2 */\
 808         "movq 40(%2), %%mm6             \n\t" /* -C2    C6      -C2     C6 */\
 809         "pmaddwd %%mm6, %%mm1           \n\t" /* -C2R6+C6R2     -C2r6+C6r2 */\
 810         "movq %%mm4, %%mm6              \n\t" /* C4R4+C4R0      C4r4+C4r0 */\
 811         "paddd %%mm5, %%mm4             \n\t" /* A0             a0 */\
 812         "psubd %%mm5, %%mm6             \n\t" /* A3             a3 */\
 813         "movq %%mm0, %%mm5              \n\t" /* -C4R4+C4R0     -C4r4+C4r0 */\
 814         "paddd %%mm1, %%mm0             \n\t" /* A1             a1 */\
 815         "psubd %%mm1, %%mm5             \n\t" /* A2             a2 */\
 816         "movq 56(%2), %%mm1             \n\t" /* C7     C5      C7      C5 */\
 817         "pmaddwd %%mm3, %%mm1           \n\t" /* C7R7+C5R5      C7r7+C5r5 */\
 818         "movq 72(%2), %%mm7             \n\t" /* -C5    -C1     -C5     -C1 */\
 819         "pmaddwd %%mm3, %%mm7           \n\t" /* -C5R7-C1R5     -C5r7-C1r5 */\
 820         "paddd %%mm4, %%mm1             \n\t" /* A0+B0          a0+b0 */\
 821         "paddd %%mm4, %%mm4             \n\t" /* 2A0            2a0 */\
 822         "psubd %%mm1, %%mm4             \n\t" /* A0-B0          a0-b0 */\
 823         "psrad $" #shift ", %%mm1       \n\t"\
 824         "psrad $" #shift ", %%mm4       \n\t"\
 825         "movq %%mm0, %%mm2              \n\t" /* A1             a1 */\
 826         "paddd %%mm7, %%mm0             \n\t" /* A1+B1          a1+b1 */\
 827         "psubd %%mm7, %%mm2             \n\t" /* A1-B1          a1-b1 */\
 828         "psrad $" #shift ", %%mm0       \n\t"\
 829         "psrad $" #shift ", %%mm2       \n\t"\
 830         "packssdw %%mm1, %%mm1          \n\t" /* A0+B0  a0+b0 */\
 831         "movd %%mm1, " #dst "           \n\t"\
 832         "packssdw %%mm0, %%mm0          \n\t" /* A1+B1  a1+b1 */\
 833         "movd %%mm0, 16+" #dst "        \n\t"\
 834         "packssdw %%mm2, %%mm2          \n\t" /* A1-B1  a1-b1 */\
 835         "movd %%mm2, 96+" #dst "        \n\t"\
 836         "packssdw %%mm4, %%mm4          \n\t" /* A0-B0  a0-b0 */\
 837         "movd %%mm4, 112+" #dst "       \n\t"\
 838         "movq 88(%2), %%mm1             \n\t" /* C3     C7      C3      C7 */\
 839         "pmaddwd %%mm3, %%mm1           \n\t" /* C3R7+C7R5      C3r7+C7r5 */\
 840         "movq %%mm5, %%mm2              \n\t" /* A2             a2 */\
 841         "pmaddwd 104(%2), %%mm3         \n\t" /* -C1R7+C3R5     -C1r7+C3r5 */\
 842         "paddd %%mm1, %%mm2             \n\t" /* A2+B2          a2+b2 */\
 843         "psubd %%mm1, %%mm5             \n\t" /* a2-B2          a2-b2 */\
 844         "psrad $" #shift ", %%mm2       \n\t"\
 845         "psrad $" #shift ", %%mm5       \n\t"\
 846         "movq %%mm6, %%mm1              \n\t" /* A3             a3 */\
 847         "paddd %%mm3, %%mm6             \n\t" /* A3+B3          a3+b3 */\
 848         "psubd %%mm3, %%mm1             \n\t" /* a3-B3          a3-b3 */\
 849         "psrad $" #shift ", %%mm6       \n\t"\
 850         "psrad $" #shift ", %%mm1       \n\t"\
 851         "packssdw %%mm2, %%mm2          \n\t" /* A2+B2  a2+b2 */\
 852         "packssdw %%mm6, %%mm6          \n\t" /* A3+B3  a3+b3 */\
 853         "movd %%mm2, 32+" #dst "        \n\t"\
 854         "packssdw %%mm1, %%mm1          \n\t" /* A3-B3  a3-b3 */\
 855         "packssdw %%mm5, %%mm5          \n\t" /* A2-B2  a2-b2 */\
 856         "movd %%mm6, 48+" #dst "        \n\t"\
 857         "movd %%mm1, 64+" #dst "        \n\t"\
 858         "movd %%mm5, 80+" #dst "        \n\t"
 859
 860 //IDCT(  src0,   src4,   src1,    src5,    dst, shift)
 861 IDCT(    (%1), 64(%1), 32(%1),  96(%1),  0(%0), 20)
 862 IDCT(   8(%1), 72(%1), 40(%1), 104(%1),  4(%0), 20)
 863 IDCT(  16(%1), 80(%1), 48(%1), 112(%1),  8(%0), 20)
 864 IDCT(  24(%1), 88(%1), 56(%1), 120(%1), 12(%0), 20)
 865         "jmp 9f                         \n\t"
 866
 867         "# .p2align 4                   \n\t"\
 868         "6:                             \n\t"
 869 Z_COND_IDCT(  96(%0),104(%0),112(%0),120(%0), 96(%1),paddd (%2), 11, 7f)
 870
 871 #undef IDCT
 872 #define IDCT(src0, src4, src1, src5, dst, shift) \
 873         "movq " #src0 ", %%mm0          \n\t" /* R4     R0      r4      r0 */\
 874         "movq " #src5 ", %%mm3          \n\t" /* R7     R5      r7      r5 */\
 875         "movq 16(%2), %%mm4             \n\t" /* C4     C4      C4      C4 */\
 876         "pmaddwd %%mm0, %%mm4           \n\t" /* C4R4+C4R0      C4r4+C4r0 */\
 877         "movq 24(%2), %%mm5             \n\t" /* -C4    C4      -C4     C4 */\
 878         "pmaddwd %%mm5, %%mm0           \n\t" /* -C4R4+C4R0     -C4r4+C4r0 */\
 879         "movq %%mm4, %%mm6              \n\t" /* C4R4+C4R0      C4r4+C4r0 */\
 880         "movq %%mm0, %%mm5              \n\t" /* -C4R4+C4R0     -C4r4+C4r0 */\
 881         "movq 56(%2), %%mm1             \n\t" /* C7     C5      C7      C5 */\
 882         "pmaddwd %%mm3, %%mm1           \n\t" /* C7R7+C5R5      C7r7+C5r5 */\
 883         "movq 72(%2), %%mm7             \n\t" /* -C5    -C1     -C5     -C1 */\
 884         "pmaddwd %%mm3, %%mm7           \n\t" /* -C5R7-C1R5     -C5r7-C1r5 */\
 885         "paddd %%mm4, %%mm1             \n\t" /* A0+B0          a0+b0 */\
 886         "paddd %%mm4, %%mm4             \n\t" /* 2A0            2a0 */\
 887         "psubd %%mm1, %%mm4             \n\t" /* A0-B0          a0-b0 */\
 888         "psrad $" #shift ", %%mm1       \n\t"\
 889         "psrad $" #shift ", %%mm4       \n\t"\
 890         "movq %%mm0, %%mm2              \n\t" /* A1             a1 */\
 891         "paddd %%mm7, %%mm0             \n\t" /* A1+B1          a1+b1 */\
 892         "psubd %%mm7, %%mm2             \n\t" /* A1-B1          a1-b1 */\
 893         "psrad $" #shift ", %%mm0       \n\t"\
 894         "psrad $" #shift ", %%mm2       \n\t"\
 895         "packssdw %%mm1, %%mm1          \n\t" /* A0+B0  a0+b0 */\
 896         "movd %%mm1, " #dst "           \n\t"\
 897         "packssdw %%mm0, %%mm0          \n\t" /* A1+B1  a1+b1 */\
 898         "movd %%mm0, 16+" #dst "        \n\t"\
 899         "packssdw %%mm2, %%mm2          \n\t" /* A1-B1  a1-b1 */\
 900         "movd %%mm2, 96+" #dst "        \n\t"\
 901         "packssdw %%mm4, %%mm4          \n\t" /* A0-B0  a0-b0 */\
 902         "movd %%mm4, 112+" #dst "       \n\t"\
 903         "movq 88(%2), %%mm1             \n\t" /* C3     C7      C3      C7 */\
 904         "pmaddwd %%mm3, %%mm1           \n\t" /* C3R7+C7R5      C3r7+C7r5 */\
 905         "movq %%mm5, %%mm2              \n\t" /* A2             a2 */\
 906         "pmaddwd 104(%2), %%mm3         \n\t" /* -C1R7+C3R5     -C1r7+C3r5 */\
 907         "paddd %%mm1, %%mm2             \n\t" /* A2+B2          a2+b2 */\
 908         "psubd %%mm1, %%mm5             \n\t" /* a2-B2          a2-b2 */\
 909         "psrad $" #shift ", %%mm2       \n\t"\
 910         "psrad $" #shift ", %%mm5       \n\t"\
 911         "movq %%mm6, %%mm1              \n\t" /* A3             a3 */\
 912         "paddd %%mm3, %%mm6             \n\t" /* A3+B3          a3+b3 */\
 913         "psubd %%mm3, %%mm1             \n\t" /* a3-B3          a3-b3 */\
 914         "psrad $" #shift ", %%mm6       \n\t"\
 915         "psrad $" #shift ", %%mm1       \n\t"\
 916         "packssdw %%mm2, %%mm2          \n\t" /* A2+B2  a2+b2 */\
 917         "packssdw %%mm6, %%mm6          \n\t" /* A3+B3  a3+b3 */\
 918         "movd %%mm2, 32+" #dst "        \n\t"\
 919         "packssdw %%mm1, %%mm1          \n\t" /* A3-B3  a3-b3 */\
 920         "packssdw %%mm5, %%mm5          \n\t" /* A2-B2  a2-b2 */\
 921         "movd %%mm6, 48+" #dst "        \n\t"\
 922         "movd %%mm1, 64+" #dst "        \n\t"\
 923         "movd %%mm5, 80+" #dst "        \n\t"
 924
 925
 926 //IDCT(  src0,   src4,   src1,    src5,    dst, shift)
 927 IDCT(    (%1), 64(%1), 32(%1),  96(%1),  0(%0), 20)
 928 IDCT(   8(%1), 72(%1), 40(%1), 104(%1),  4(%0), 20)
 929 IDCT(  16(%1), 80(%1), 48(%1), 112(%1),  8(%0), 20)
 930 IDCT(  24(%1), 88(%1), 56(%1), 120(%1), 12(%0), 20)
 931         "jmp 9f                         \n\t"
 932
 933         "# .p2align 4                   \n\t"\
 934         "2:                             \n\t"
 935 Z_COND_IDCT(  96(%0),104(%0),112(%0),120(%0), 96(%1),paddd (%2), 11, 3f)
 936
 937 #undef IDCT
 938 #define IDCT(src0, src4, src1, src5, dst, shift) \
 939         "movq " #src0 ", %%mm0          \n\t" /* R4     R0      r4      r0 */\
 940         "movq " #src1 ", %%mm2          \n\t" /* R3     R1      r3      r1 */\
 941         "movq " #src5 ", %%mm3          \n\t" /* R7     R5      r7      r5 */\
 942         "movq 16(%2), %%mm4             \n\t" /* C4     C4      C4      C4 */\
 943         "pmaddwd %%mm0, %%mm4           \n\t" /* C4R4+C4R0      C4r4+C4r0 */\
 944         "movq 24(%2), %%mm5             \n\t" /* -C4    C4      -C4     C4 */\
 945         "pmaddwd %%mm5, %%mm0           \n\t" /* -C4R4+C4R0     -C4r4+C4r0 */\
 946         "movq %%mm4, %%mm6              \n\t" /* C4R4+C4R0      C4r4+C4r0 */\
 947         "movq 48(%2), %%mm7             \n\t" /* C3     C1      C3      C1 */\
 948         "pmaddwd %%mm2, %%mm7           \n\t" /* C3R3+C1R1      C3r3+C1r1 */\
 949         "movq %%mm0, %%mm5              \n\t" /* -C4R4+C4R0     -C4r4+C4r0 */\
 950         "movq 56(%2), %%mm1             \n\t" /* C7     C5      C7      C5 */\
 951         "pmaddwd %%mm3, %%mm1           \n\t" /* C7R7+C5R5      C7r7+C5r5 */\
 952         "pmaddwd 64(%2), %%mm2          \n\t" /* -C7R3+C3R1     -C7r3+C3r1 */\
 953         "paddd %%mm1, %%mm7             \n\t" /* B0             b0 */\
 954         "movq 72(%2), %%mm1             \n\t" /* -C5    -C1     -C5     -C1 */\
 955         "pmaddwd %%mm3, %%mm1           \n\t" /* -C5R7-C1R5     -C5r7-C1r5 */\
 956         "paddd %%mm4, %%mm7             \n\t" /* A0+B0          a0+b0 */\
 957         "paddd %%mm4, %%mm4             \n\t" /* 2A0            2a0 */\
 958         "psubd %%mm7, %%mm4             \n\t" /* A0-B0          a0-b0 */\
 959         "paddd %%mm2, %%mm1             \n\t" /* B1             b1 */\
 960         "psrad $" #shift ", %%mm7       \n\t"\
 961         "psrad $" #shift ", %%mm4       \n\t"\
 962         "movq %%mm0, %%mm2              \n\t" /* A1             a1 */\
 963         "paddd %%mm1, %%mm0             \n\t" /* A1+B1          a1+b1 */\
 964         "psubd %%mm1, %%mm2             \n\t" /* A1-B1          a1-b1 */\
 965         "psrad $" #shift ", %%mm0       \n\t"\
 966         "psrad $" #shift ", %%mm2       \n\t"\
 967         "packssdw %%mm7, %%mm7          \n\t" /* A0+B0  a0+b0 */\
 968         "movd %%mm7, " #dst "           \n\t"\
 969         "packssdw %%mm0, %%mm0          \n\t" /* A1+B1  a1+b1 */\
 970         "movd %%mm0, 16+" #dst "        \n\t"\
 971         "packssdw %%mm2, %%mm2          \n\t" /* A1-B1  a1-b1 */\
 972         "movd %%mm2, 96+" #dst "        \n\t"\
 973         "packssdw %%mm4, %%mm4          \n\t" /* A0-B0  a0-b0 */\
 974         "movd %%mm4, 112+" #dst "       \n\t"\
 975         "movq " #src1 ", %%mm0          \n\t" /* R3     R1      r3      r1 */\
 976         "movq 80(%2), %%mm4             \n\t" /* -C1    C5      -C1     C5 */\
 977         "pmaddwd %%mm0, %%mm4           \n\t" /* -C1R3+C5R1     -C1r3+C5r1 */\
 978         "movq 88(%2), %%mm7             \n\t" /* C3     C7      C3      C7 */\
 979         "pmaddwd 96(%2), %%mm0          \n\t" /* -C5R3+C7R1     -C5r3+C7r1 */\
 980         "pmaddwd %%mm3, %%mm7           \n\t" /* C3R7+C7R5      C3r7+C7r5 */\
 981         "movq %%mm5, %%mm2              \n\t" /* A2             a2 */\
 982         "pmaddwd 104(%2), %%mm3         \n\t" /* -C1R7+C3R5     -C1r7+C3r5 */\
 983         "paddd %%mm7, %%mm4             \n\t" /* B2             b2 */\
 984         "paddd %%mm4, %%mm2             \n\t" /* A2+B2          a2+b2 */\
 985         "psubd %%mm4, %%mm5             \n\t" /* a2-B2          a2-b2 */\
 986         "psrad $" #shift ", %%mm2       \n\t"\
 987         "psrad $" #shift ", %%mm5       \n\t"\
 988         "movq %%mm6, %%mm4              \n\t" /* A3             a3 */\
 989         "paddd %%mm0, %%mm3             \n\t" /* B3             b3 */\
 990         "paddd %%mm3, %%mm6             \n\t" /* A3+B3          a3+b3 */\
 991         "psubd %%mm3, %%mm4             \n\t" /* a3-B3          a3-b3 */\
 992         "psrad $" #shift ", %%mm6       \n\t"\
 993         "psrad $" #shift ", %%mm4       \n\t"\
 994         "packssdw %%mm2, %%mm2          \n\t" /* A2+B2  a2+b2 */\
 995         "packssdw %%mm6, %%mm6          \n\t" /* A3+B3  a3+b3 */\
 996         "movd %%mm2, 32+" #dst "        \n\t"\
 997         "packssdw %%mm4, %%mm4          \n\t" /* A3-B3  a3-b3 */\
 998         "packssdw %%mm5, %%mm5          \n\t" /* A2-B2  a2-b2 */\
 999         "movd %%mm6, 48+" #dst "        \n\t"\
1000         "movd %%mm4, 64+" #dst "        \n\t"\
1001         "movd %%mm5, 80+" #dst "        \n\t"
1002
1003 //IDCT(  src0,   src4,   src1,    src5,    dst, shift)
1004 IDCT(    (%1), 64(%1), 32(%1),  96(%1),  0(%0), 20)
1005 IDCT(   8(%1), 72(%1), 40(%1), 104(%1),  4(%0), 20)
1006 IDCT(  16(%1), 80(%1), 48(%1), 112(%1),  8(%0), 20)
1007 IDCT(  24(%1), 88(%1), 56(%1), 120(%1), 12(%0), 20)
1008         "jmp 9f                         \n\t"
1009
1010         "# .p2align 4                   \n\t"\
1011         "3:                             \n\t"
1012 #undef IDCT
1013 #define IDCT(src0, src4, src1, src5, dst, shift) \
1014         "movq " #src0 ", %%mm0          \n\t" /* R4     R0      r4      r0 */\
1015         "movq " #src1 ", %%mm2          \n\t" /* R3     R1      r3      r1 */\
1016         "movq 16(%2), %%mm4             \n\t" /* C4     C4      C4      C4 */\
1017         "pmaddwd %%mm0, %%mm4           \n\t" /* C4R4+C4R0      C4r4+C4r0 */\
1018         "movq 24(%2), %%mm5             \n\t" /* -C4    C4      -C4     C4 */\
1019         "pmaddwd %%mm5, %%mm0           \n\t" /* -C4R4+C4R0     -C4r4+C4r0 */\
1020         "movq %%mm4, %%mm6              \n\t" /* C4R4+C4R0      C4r4+C4r0 */\
1021         "movq 48(%2), %%mm7             \n\t" /* C3     C1      C3      C1 */\
1022         "pmaddwd %%mm2, %%mm7           \n\t" /* C3R3+C1R1      C3r3+C1r1 */\
1023         "movq %%mm0, %%mm5              \n\t" /* -C4R4+C4R0     -C4r4+C4r0 */\
1024         "movq 64(%2), %%mm3             \n\t"\
1025         "pmaddwd %%mm2, %%mm3           \n\t" /* -C7R3+C3R1     -C7r3+C3r1 */\
1026         "paddd %%mm4, %%mm7             \n\t" /* A0+B0          a0+b0 */\
1027         "paddd %%mm4, %%mm4             \n\t" /* 2A0            2a0 */\
1028         "psubd %%mm7, %%mm4             \n\t" /* A0-B0          a0-b0 */\
1029         "psrad $" #shift ", %%mm7       \n\t"\
1030         "psrad $" #shift ", %%mm4       \n\t"\
1031         "movq %%mm0, %%mm1              \n\t" /* A1             a1 */\
1032         "paddd %%mm3, %%mm0             \n\t" /* A1+B1          a1+b1 */\
1033         "psubd %%mm3, %%mm1             \n\t" /* A1-B1          a1-b1 */\
1034         "psrad $" #shift ", %%mm0       \n\t"\
1035         "psrad $" #shift ", %%mm1       \n\t"\
1036         "packssdw %%mm7, %%mm7          \n\t" /* A0+B0  a0+b0 */\
1037         "movd %%mm7, " #dst "           \n\t"\
1038         "packssdw %%mm0, %%mm0          \n\t" /* A1+B1  a1+b1 */\
1039         "movd %%mm0, 16+" #dst "        \n\t"\
1040         "packssdw %%mm1, %%mm1          \n\t" /* A1-B1  a1-b1 */\
1041         "movd %%mm1, 96+" #dst "        \n\t"\
1042         "packssdw %%mm4, %%mm4          \n\t" /* A0-B0  a0-b0 */\
1043         "movd %%mm4, 112+" #dst "       \n\t"\
1044         "movq 80(%2), %%mm4             \n\t" /* -C1    C5      -C1     C5 */\
1045         "pmaddwd %%mm2, %%mm4           \n\t" /* -C1R3+C5R1     -C1r3+C5r1 */\
1046         "pmaddwd 96(%2), %%mm2          \n\t" /* -C5R3+C7R1     -C5r3+C7r1 */\
1047         "movq %%mm5, %%mm1              \n\t" /* A2             a2 */\
1048         "paddd %%mm4, %%mm1             \n\t" /* A2+B2          a2+b2 */\
1049         "psubd %%mm4, %%mm5             \n\t" /* a2-B2          a2-b2 */\
1050         "psrad $" #shift ", %%mm1       \n\t"\
1051         "psrad $" #shift ", %%mm5       \n\t"\
1052         "movq %%mm6, %%mm4              \n\t" /* A3             a3 */\
1053         "paddd %%mm2, %%mm6             \n\t" /* A3+B3          a3+b3 */\
1054         "psubd %%mm2, %%mm4             \n\t" /* a3-B3          a3-b3 */\
1055         "psrad $" #shift ", %%mm6       \n\t"\
1056         "psrad $" #shift ", %%mm4       \n\t"\
1057         "packssdw %%mm1, %%mm1          \n\t" /* A2+B2  a2+b2 */\
1058         "packssdw %%mm6, %%mm6          \n\t" /* A3+B3  a3+b3 */\
1059         "movd %%mm1, 32+" #dst "        \n\t"\
1060         "packssdw %%mm4, %%mm4          \n\t" /* A3-B3  a3-b3 */\
1061         "packssdw %%mm5, %%mm5          \n\t" /* A2-B2  a2-b2 */\
1062         "movd %%mm6, 48+" #dst "        \n\t"\
1063         "movd %%mm4, 64+" #dst "        \n\t"\
1064         "movd %%mm5, 80+" #dst "        \n\t"
1065
1066
1067 //IDCT(  src0,   src4,   src1,    src5,    dst, shift)
1068 IDCT(    (%1), 64(%1), 32(%1),  96(%1),  0(%0), 20)
1069 IDCT(   8(%1), 72(%1), 40(%1), 104(%1),  4(%0), 20)
1070 IDCT(  16(%1), 80(%1), 48(%1), 112(%1),  8(%0), 20)
1071 IDCT(  24(%1), 88(%1), 56(%1), 120(%1), 12(%0), 20)
1072         "jmp 9f                         \n\t"
1073
1074         "# .p2align 4                   \n\t"\
1075         "5:                             \n\t"
1076 #undef IDCT
1077 #define IDCT(src0, src4, src1, src5, dst, shift) \
1078         "movq " #src0 ", %%mm0          \n\t" /* R4     R0      r4      r0 */\
1079         "movq " #src4 ", %%mm1          \n\t" /* R6     R2      r6      r2 */\
1080         "movq 16(%2), %%mm4             \n\t" /* C4     C4      C4      C4 */\
1081         "pmaddwd %%mm0, %%mm4           \n\t" /* C4R4+C4R0      C4r4+C4r0 */\
1082         "movq 24(%2), %%mm5             \n\t" /* -C4    C4      -C4     C4 */\
1083         "pmaddwd %%mm5, %%mm0           \n\t" /* -C4R4+C4R0     -C4r4+C4r0 */\
1084         "movq 32(%2), %%mm5             \n\t" /* C6     C2      C6      C2 */\
1085         "pmaddwd %%mm1, %%mm5           \n\t" /* C6R6+C2R2      C6r6+C2r2 */\
1086         "movq 40(%2), %%mm6             \n\t" /* -C2    C6      -C2     C6 */\
1087         "pmaddwd %%mm6, %%mm1           \n\t" /* -C2R6+C6R2     -C2r6+C6r2 */\
1088         "movq %%mm4, %%mm6              \n\t" /* C4R4+C4R0      C4r4+C4r0 */\
1089         "paddd %%mm5, %%mm4             \n\t" /* A0             a0 */\
1090         "psubd %%mm5, %%mm6             \n\t" /* A3             a3 */\
1091         "movq %%mm0, %%mm5              \n\t" /* -C4R4+C4R0     -C4r4+C4r0 */\
1092         "paddd %%mm1, %%mm0             \n\t" /* A1             a1 */\
1093         "psubd %%mm1, %%mm5             \n\t" /* A2             a2 */\
1094         "movq 8+" #src0 ", %%mm2        \n\t" /* R4     R0      r4      r0 */\
1095         "movq 8+" #src4 ", %%mm3        \n\t" /* R6     R2      r6      r2 */\
1096         "movq 16(%2), %%mm1             \n\t" /* C4     C4      C4      C4 */\
1097         "pmaddwd %%mm2, %%mm1           \n\t" /* C4R4+C4R0      C4r4+C4r0 */\
1098         "movq 24(%2), %%mm7             \n\t" /* -C4    C4      -C4     C4 */\
1099         "pmaddwd %%mm7, %%mm2           \n\t" /* -C4R4+C4R0     -C4r4+C4r0 */\
1100         "movq 32(%2), %%mm7             \n\t" /* C6     C2      C6      C2 */\
1101         "pmaddwd %%mm3, %%mm7           \n\t" /* C6R6+C2R2      C6r6+C2r2 */\
1102         "pmaddwd 40(%2), %%mm3          \n\t" /* -C2R6+C6R2     -C2r6+C6r2 */\
1103         "paddd %%mm1, %%mm7             \n\t" /* A0             a0 */\
1104         "paddd %%mm1, %%mm1             \n\t" /* 2C0            2c0 */\
1105         "psubd %%mm7, %%mm1             \n\t" /* A3             a3 */\
1106         "paddd %%mm2, %%mm3             \n\t" /* A1             a1 */\
1107         "paddd %%mm2, %%mm2             \n\t" /* 2C1            2c1 */\
1108         "psubd %%mm3, %%mm2             \n\t" /* A2             a2 */\
1109         "psrad $" #shift ", %%mm4       \n\t"\
1110         "psrad $" #shift ", %%mm7       \n\t"\
1111         "psrad $" #shift ", %%mm3       \n\t"\
1112         "packssdw %%mm7, %%mm4          \n\t" /* A0     a0 */\
1113         "movq %%mm4, " #dst "           \n\t"\
1114         "psrad $" #shift ", %%mm0       \n\t"\
1115         "packssdw %%mm3, %%mm0          \n\t" /* A1     a1 */\
1116         "movq %%mm0, 16+" #dst "        \n\t"\
1117         "movq %%mm0, 96+" #dst "        \n\t"\
1118         "movq %%mm4, 112+" #dst "       \n\t"\
1119         "psrad $" #shift ", %%mm5       \n\t"\
1120         "psrad $" #shift ", %%mm6       \n\t"\
1121         "psrad $" #shift ", %%mm2       \n\t"\
1122         "packssdw %%mm2, %%mm5          \n\t" /* A2-B2  a2-b2 */\
1123         "movq %%mm5, 32+" #dst "        \n\t"\
1124         "psrad $" #shift ", %%mm1       \n\t"\
1125         "packssdw %%mm1, %%mm6          \n\t" /* A3+B3  a3+b3 */\
1126         "movq %%mm6, 48+" #dst "        \n\t"\
1127         "movq %%mm6, 64+" #dst "        \n\t"\
1128         "movq %%mm5, 80+" #dst "        \n\t"
1129
1130
1131 //IDCT(  src0,   src4,   src1,    src5,    dst, shift)
1132 IDCT(    0(%1), 64(%1), 32(%1),  96(%1),  0(%0), 20)
1133 //IDCT(   8(%1), 72(%1), 40(%1), 104(%1),  4(%0), 20)
1134 IDCT(  16(%1), 80(%1), 48(%1), 112(%1),  8(%0), 20)
1135 //IDCT(  24(%1), 88(%1), 56(%1), 120(%1), 12(%0), 20)
1136         "jmp 9f                         \n\t"
1137
1138
1139         "# .p2align 4                   \n\t"\
1140         "1:                             \n\t"
1141 #undef IDCT
1142 #define IDCT(src0, src4, src1, src5, dst, shift) \
1143         "movq " #src0 ", %%mm0          \n\t" /* R4     R0      r4      r0 */\
1144         "movq " #src4 ", %%mm1          \n\t" /* R6     R2      r6      r2 */\
1145         "movq " #src1 ", %%mm2          \n\t" /* R3     R1      r3      r1 */\
1146         "movq 16(%2), %%mm4             \n\t" /* C4     C4      C4      C4 */\
1147         "pmaddwd %%mm0, %%mm4           \n\t" /* C4R4+C4R0      C4r4+C4r0 */\
1148         "movq 24(%2), %%mm5             \n\t" /* -C4    C4      -C4     C4 */\
1149         "pmaddwd %%mm5, %%mm0           \n\t" /* -C4R4+C4R0     -C4r4+C4r0 */\
1150         "movq 32(%2), %%mm5             \n\t" /* C6     C2      C6      C2 */\
1151         "pmaddwd %%mm1, %%mm5           \n\t" /* C6R6+C2R2      C6r6+C2r2 */\
1152         "movq 40(%2), %%mm6             \n\t" /* -C2    C6      -C2     C6 */\
1153         "pmaddwd %%mm6, %%mm1           \n\t" /* -C2R6+C6R2     -C2r6+C6r2 */\
1154         "movq %%mm4, %%mm6              \n\t" /* C4R4+C4R0      C4r4+C4r0 */\
1155         "movq 48(%2), %%mm7             \n\t" /* C3     C1      C3      C1 */\
1156         "pmaddwd %%mm2, %%mm7           \n\t" /* C3R3+C1R1      C3r3+C1r1 */\
1157         "paddd %%mm5, %%mm4             \n\t" /* A0             a0 */\
1158         "psubd %%mm5, %%mm6             \n\t" /* A3             a3 */\
1159         "movq %%mm0, %%mm5              \n\t" /* -C4R4+C4R0     -C4r4+C4r0 */\
1160         "paddd %%mm1, %%mm0             \n\t" /* A1             a1 */\
1161         "psubd %%mm1, %%mm5             \n\t" /* A2             a2 */\
1162         "movq 64(%2), %%mm1             \n\t"\
1163         "pmaddwd %%mm2, %%mm1           \n\t" /* -C7R3+C3R1     -C7r3+C3r1 */\
1164         "paddd %%mm4, %%mm7             \n\t" /* A0+B0          a0+b0 */\
1165         "paddd %%mm4, %%mm4             \n\t" /* 2A0            2a0 */\
1166         "psubd %%mm7, %%mm4             \n\t" /* A0-B0          a0-b0 */\
1167         "psrad $" #shift ", %%mm7       \n\t"\
1168         "psrad $" #shift ", %%mm4       \n\t"\
1169         "movq %%mm0, %%mm3              \n\t" /* A1             a1 */\
1170         "paddd %%mm1, %%mm0             \n\t" /* A1+B1          a1+b1 */\
1171         "psubd %%mm1, %%mm3             \n\t" /* A1-B1          a1-b1 */\
1172         "psrad $" #shift ", %%mm0       \n\t"\
1173         "psrad $" #shift ", %%mm3       \n\t"\
1174         "packssdw %%mm7, %%mm7          \n\t" /* A0+B0  a0+b0 */\
1175         "movd %%mm7, " #dst "           \n\t"\
1176         "packssdw %%mm0, %%mm0          \n\t" /* A1+B1  a1+b1 */\
1177         "movd %%mm0, 16+" #dst "        \n\t"\
1178         "packssdw %%mm3, %%mm3          \n\t" /* A1-B1  a1-b1 */\
1179         "movd %%mm3, 96+" #dst "        \n\t"\
1180         "packssdw %%mm4, %%mm4          \n\t" /* A0-B0  a0-b0 */\
1181         "movd %%mm4, 112+" #dst "       \n\t"\
1182         "movq 80(%2), %%mm4             \n\t" /* -C1    C5      -C1     C5 */\
1183         "pmaddwd %%mm2, %%mm4           \n\t" /* -C1R3+C5R1     -C1r3+C5r1 */\
1184         "pmaddwd 96(%2), %%mm2          \n\t" /* -C5R3+C7R1     -C5r3+C7r1 */\
1185         "movq %%mm5, %%mm3              \n\t" /* A2             a2 */\
1186         "paddd %%mm4, %%mm3             \n\t" /* A2+B2          a2+b2 */\
1187         "psubd %%mm4, %%mm5             \n\t" /* a2-B2          a2-b2 */\
1188         "psrad $" #shift ", %%mm3       \n\t"\
1189         "psrad $" #shift ", %%mm5       \n\t"\
1190         "movq %%mm6, %%mm4              \n\t" /* A3             a3 */\
1191         "paddd %%mm2, %%mm6             \n\t" /* A3+B3          a3+b3 */\
1192         "psubd %%mm2, %%mm4             \n\t" /* a3-B3          a3-b3 */\
1193         "psrad $" #shift ", %%mm6       \n\t"\
1194         "packssdw %%mm3, %%mm3          \n\t" /* A2+B2  a2+b2 */\
1195         "movd %%mm3, 32+" #dst "        \n\t"\
1196         "psrad $" #shift ", %%mm4       \n\t"\
1197         "packssdw %%mm6, %%mm6          \n\t" /* A3+B3  a3+b3 */\
1198         "movd %%mm6, 48+" #dst "        \n\t"\
1199         "packssdw %%mm4, %%mm4          \n\t" /* A3-B3  a3-b3 */\
1200         "packssdw %%mm5, %%mm5          \n\t" /* A2-B2  a2-b2 */\
1201         "movd %%mm4, 64+" #dst "        \n\t"\
1202         "movd %%mm5, 80+" #dst "        \n\t"
1203
1204
1205 //IDCT(  src0,   src4,   src1,    src5,    dst, shift)
1206 IDCT(    (%1), 64(%1), 32(%1),  96(%1),  0(%0), 20)
1207 IDCT(   8(%1), 72(%1), 40(%1), 104(%1),  4(%0), 20)
1208 IDCT(  16(%1), 80(%1), 48(%1), 112(%1),  8(%0), 20)
1209 IDCT(  24(%1), 88(%1), 56(%1), 120(%1), 12(%0), 20)
1210         "jmp 9f                         \n\t"
1211
1212
1213         "# .p2align 4                   \n\t"
1214         "7:                             \n\t"
1215 #undef IDCT
1216 #define IDCT(src0, src4, src1, src5, dst, shift) \
1217         "movq " #src0 ", %%mm0          \n\t" /* R4     R0      r4      r0 */\
1218         "movq 16(%2), %%mm4             \n\t" /* C4     C4      C4      C4 */\
1219         "pmaddwd %%mm0, %%mm4           \n\t" /* C4R4+C4R0      C4r4+C4r0 */\
1220         "movq 24(%2), %%mm5             \n\t" /* -C4    C4      -C4     C4 */\
1221         "pmaddwd %%mm5, %%mm0           \n\t" /* -C4R4+C4R0     -C4r4+C4r0 */\
1222         "psrad $" #shift ", %%mm4       \n\t"\
1223         "psrad $" #shift ", %%mm0       \n\t"\
1224         "movq 8+" #src0 ", %%mm2        \n\t" /* R4     R0      r4      r0 */\
1225         "movq 16(%2), %%mm1             \n\t" /* C4     C4      C4      C4 */\
1226         "pmaddwd %%mm2, %%mm1           \n\t" /* C4R4+C4R0      C4r4+C4r0 */\
1227         "movq 24(%2), %%mm7             \n\t" /* -C4    C4      -C4     C4 */\
1228         "pmaddwd %%mm7, %%mm2           \n\t" /* -C4R4+C4R0     -C4r4+C4r0 */\
1229         "movq 32(%2), %%mm7             \n\t" /* C6     C2      C6      C2 */\
1230         "psrad $" #shift ", %%mm1       \n\t"\
1231         "packssdw %%mm1, %%mm4          \n\t" /* A0     a0 */\
1232         "movq %%mm4, " #dst "           \n\t"\
1233         "psrad $" #shift ", %%mm2       \n\t"\
1234         "packssdw %%mm2, %%mm0          \n\t" /* A1     a1 */\
1235         "movq %%mm0, 16+" #dst "        \n\t"\
1236         "movq %%mm0, 96+" #dst "        \n\t"\
1237         "movq %%mm4, 112+" #dst "       \n\t"\
1238         "movq %%mm0, 32+" #dst "        \n\t"\
1239         "movq %%mm4, 48+" #dst "        \n\t"\
1240         "movq %%mm4, 64+" #dst "        \n\t"\
1241         "movq %%mm0, 80+" #dst "        \n\t"
1242
1243 //IDCT(  src0,   src4,   src1,    src5,    dst, shift)
1244 IDCT(   0(%1), 64(%1), 32(%1),  96(%1),  0(%0), 20)
1245 //IDCT(   8(%1), 72(%1), 40(%1), 104(%1),  4(%0), 20)
1246 IDCT(  16(%1), 80(%1), 48(%1), 112(%1),  8(%0), 20)
1247 //IDCT(  24(%1), 88(%1), 56(%1), 120(%1), 12(%0), 20)
1248
1249
1250 #endif
1251
1252 /*
1253 Input
1254  00 40 04 44 20 60 24 64
1255  10 30 14 34 50 70 54 74
1256  01 41 03 43 21 61 23 63
1257  11 31 13 33 51 71 53 73
1258  02 42 06 46 22 62 26 66
1259  12 32 16 36 52 72 56 76
1260  05 45 07 47 25 65 27 67
1261  15 35 17 37 55 75 57 77
1262
1263 Temp
1264  00 04 10 14 20 24 30 34
1265  40 44 50 54 60 64 70 74
1266  01 03 11 13 21 23 31 33
1267  41 43 51 53 61 63 71 73
1268  02 06 12 16 22 26 32 36
1269  42 46 52 56 62 66 72 76
1270  05 07 15 17 25 27 35 37
1271  45 47 55 57 65 67 75 77
1272 */
1273
1274 "9: \n\t"
1275                 :: "r" (block), "r" (temp), "r" (coeffs)
1276                 : "%eax"
1277         );
1278 }
1279
1280 void ff_simple_idct_mmx(int16_t *block)
1281 {
1282     idct(block);
1283 }
1284
1285 //FIXME merge add/put into the idct
1286
1287 void ff_simple_idct_put_mmx(uint8_t *dest, int line_size, DCTELEM *block)
1288 {
1289     idct(block);
1290     ff_put_pixels_clamped_mmx(block, dest, line_size);
1291 }
1292 void ff_simple_idct_add_mmx(uint8_t *dest, int line_size, DCTELEM *block)
1293 {
1294     idct(block);
1295     ff_add_pixels_clamped_mmx(block, dest, line_size);
1296 }