Libav 0.7.1
|
00001 /* 00002 * Copyright (c) 2004-2005 Michael Niedermayer, Loren Merritt 00003 * 00004 * This file is part of Libav. 00005 * 00006 * Libav is free software; you can redistribute it and/or 00007 * modify it under the terms of the GNU Lesser General Public 00008 * License as published by the Free Software Foundation; either 00009 * version 2.1 of the License, or (at your option) any later version. 00010 * 00011 * Libav is distributed in the hope that it will be useful, 00012 * but WITHOUT ANY WARRANTY; without even the implied warranty of 00013 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 00014 * Lesser General Public License for more details. 00015 * 00016 * You should have received a copy of the GNU Lesser General Public 00017 * License along with Libav; if not, write to the Free Software 00018 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 00019 */ 00020 00021 #include "dsputil_mmx.h" 00022 00023 /***********************************/ 00024 /* motion compensation */ 00025 00026 #define QPEL_H264V_MM(A,B,C,D,E,F,OP,T,Z,d,q)\ 00027 "mov"#q" "#C", "#T" \n\t"\ 00028 "mov"#d" (%0), "#F" \n\t"\ 00029 "paddw "#D", "#T" \n\t"\ 00030 "psllw $2, "#T" \n\t"\ 00031 "psubw "#B", "#T" \n\t"\ 00032 "psubw "#E", "#T" \n\t"\ 00033 "punpcklbw "#Z", "#F" \n\t"\ 00034 "pmullw "MANGLE(ff_pw_5)", "#T"\n\t"\ 00035 "paddw "MANGLE(ff_pw_16)", "#A"\n\t"\ 00036 "add %2, %0 \n\t"\ 00037 "paddw "#F", "#A" \n\t"\ 00038 "paddw "#A", "#T" \n\t"\ 00039 "psraw $5, "#T" \n\t"\ 00040 "packuswb "#T", "#T" \n\t"\ 00041 OP(T, (%1), A, d)\ 00042 "add %3, %1 \n\t" 00043 00044 #define QPEL_H264HV_MM(A,B,C,D,E,F,OF,T,Z,d,q)\ 00045 "mov"#q" "#C", "#T" \n\t"\ 00046 "mov"#d" (%0), "#F" \n\t"\ 00047 "paddw "#D", "#T" \n\t"\ 00048 "psllw $2, "#T" \n\t"\ 00049 "paddw "MANGLE(ff_pw_16)", "#A"\n\t"\ 00050 "psubw "#B", "#T" \n\t"\ 00051 "psubw "#E", "#T" \n\t"\ 00052 "punpcklbw "#Z", "#F" \n\t"\ 00053 "pmullw "MANGLE(ff_pw_5)", "#T"\n\t"\ 00054 "paddw "#F", "#A" \n\t"\ 00055 "add %2, %0 \n\t"\ 00056 "paddw "#A", "#T" \n\t"\ 00057 "mov"#q" "#T", "#OF"(%1) \n\t" 00058 00059 #define QPEL_H264V(A,B,C,D,E,F,OP) QPEL_H264V_MM(A,B,C,D,E,F,OP,%%mm6,%%mm7,d,q) 00060 #define QPEL_H264HV(A,B,C,D,E,F,OF) QPEL_H264HV_MM(A,B,C,D,E,F,OF,%%mm6,%%mm7,d,q) 00061 #define QPEL_H264V_XMM(A,B,C,D,E,F,OP) QPEL_H264V_MM(A,B,C,D,E,F,OP,%%xmm6,%%xmm7,q,dqa) 00062 #define QPEL_H264HV_XMM(A,B,C,D,E,F,OF) QPEL_H264HV_MM(A,B,C,D,E,F,OF,%%xmm6,%%xmm7,q,dqa) 00063 00064 00065 #define QPEL_H264(OPNAME, OP, MMX)\ 00066 static av_noinline void OPNAME ## h264_qpel4_h_lowpass_ ## MMX(uint8_t *dst, uint8_t *src, int dstStride, int srcStride){\ 00067 int h=4;\ 00068 \ 00069 __asm__ volatile(\ 00070 "pxor %%mm7, %%mm7 \n\t"\ 00071 "movq "MANGLE(ff_pw_5) ", %%mm4\n\t"\ 00072 "movq "MANGLE(ff_pw_16)", %%mm5\n\t"\ 00073 "1: \n\t"\ 00074 "movd -1(%0), %%mm1 \n\t"\ 00075 "movd (%0), %%mm2 \n\t"\ 00076 "movd 1(%0), %%mm3 \n\t"\ 00077 "movd 2(%0), %%mm0 \n\t"\ 00078 "punpcklbw %%mm7, %%mm1 \n\t"\ 00079 "punpcklbw %%mm7, %%mm2 \n\t"\ 00080 "punpcklbw %%mm7, %%mm3 \n\t"\ 00081 "punpcklbw %%mm7, %%mm0 \n\t"\ 00082 "paddw %%mm0, %%mm1 \n\t"\ 00083 "paddw %%mm3, %%mm2 \n\t"\ 00084 "movd -2(%0), %%mm0 \n\t"\ 00085 "movd 3(%0), %%mm3 \n\t"\ 00086 "punpcklbw %%mm7, %%mm0 \n\t"\ 00087 "punpcklbw %%mm7, %%mm3 \n\t"\ 00088 "paddw %%mm3, %%mm0 \n\t"\ 00089 "psllw $2, %%mm2 \n\t"\ 00090 "psubw %%mm1, %%mm2 \n\t"\ 00091 "pmullw %%mm4, %%mm2 \n\t"\ 00092 "paddw %%mm5, %%mm0 \n\t"\ 00093 "paddw %%mm2, %%mm0 \n\t"\ 00094 "psraw $5, %%mm0 \n\t"\ 00095 "packuswb %%mm0, %%mm0 \n\t"\ 00096 OP(%%mm0, (%1),%%mm6, d)\ 00097 "add %3, %0 \n\t"\ 00098 "add %4, %1 \n\t"\ 00099 "decl %2 \n\t"\ 00100 " jnz 1b \n\t"\ 00101 : "+a"(src), "+c"(dst), "+g"(h)\ 00102 : "d"((x86_reg)srcStride), "S"((x86_reg)dstStride)\ 00103 : "memory"\ 00104 );\ 00105 }\ 00106 static av_noinline void OPNAME ## h264_qpel4_h_lowpass_l2_ ## MMX(uint8_t *dst, uint8_t *src, uint8_t *src2, int dstStride, int src2Stride){\ 00107 int h=4;\ 00108 __asm__ volatile(\ 00109 "pxor %%mm7, %%mm7 \n\t"\ 00110 "movq %0, %%mm4 \n\t"\ 00111 "movq %1, %%mm5 \n\t"\ 00112 :: "m"(ff_pw_5), "m"(ff_pw_16)\ 00113 );\ 00114 do{\ 00115 __asm__ volatile(\ 00116 "movd -1(%0), %%mm1 \n\t"\ 00117 "movd (%0), %%mm2 \n\t"\ 00118 "movd 1(%0), %%mm3 \n\t"\ 00119 "movd 2(%0), %%mm0 \n\t"\ 00120 "punpcklbw %%mm7, %%mm1 \n\t"\ 00121 "punpcklbw %%mm7, %%mm2 \n\t"\ 00122 "punpcklbw %%mm7, %%mm3 \n\t"\ 00123 "punpcklbw %%mm7, %%mm0 \n\t"\ 00124 "paddw %%mm0, %%mm1 \n\t"\ 00125 "paddw %%mm3, %%mm2 \n\t"\ 00126 "movd -2(%0), %%mm0 \n\t"\ 00127 "movd 3(%0), %%mm3 \n\t"\ 00128 "punpcklbw %%mm7, %%mm0 \n\t"\ 00129 "punpcklbw %%mm7, %%mm3 \n\t"\ 00130 "paddw %%mm3, %%mm0 \n\t"\ 00131 "psllw $2, %%mm2 \n\t"\ 00132 "psubw %%mm1, %%mm2 \n\t"\ 00133 "pmullw %%mm4, %%mm2 \n\t"\ 00134 "paddw %%mm5, %%mm0 \n\t"\ 00135 "paddw %%mm2, %%mm0 \n\t"\ 00136 "movd (%2), %%mm3 \n\t"\ 00137 "psraw $5, %%mm0 \n\t"\ 00138 "packuswb %%mm0, %%mm0 \n\t"\ 00139 PAVGB" %%mm3, %%mm0 \n\t"\ 00140 OP(%%mm0, (%1),%%mm6, d)\ 00141 "add %4, %0 \n\t"\ 00142 "add %4, %1 \n\t"\ 00143 "add %3, %2 \n\t"\ 00144 : "+a"(src), "+c"(dst), "+d"(src2)\ 00145 : "D"((x86_reg)src2Stride), "S"((x86_reg)dstStride)\ 00146 : "memory"\ 00147 );\ 00148 }while(--h);\ 00149 }\ 00150 static av_noinline void OPNAME ## h264_qpel4_v_lowpass_ ## MMX(uint8_t *dst, uint8_t *src, int dstStride, int srcStride){\ 00151 src -= 2*srcStride;\ 00152 __asm__ volatile(\ 00153 "pxor %%mm7, %%mm7 \n\t"\ 00154 "movd (%0), %%mm0 \n\t"\ 00155 "add %2, %0 \n\t"\ 00156 "movd (%0), %%mm1 \n\t"\ 00157 "add %2, %0 \n\t"\ 00158 "movd (%0), %%mm2 \n\t"\ 00159 "add %2, %0 \n\t"\ 00160 "movd (%0), %%mm3 \n\t"\ 00161 "add %2, %0 \n\t"\ 00162 "movd (%0), %%mm4 \n\t"\ 00163 "add %2, %0 \n\t"\ 00164 "punpcklbw %%mm7, %%mm0 \n\t"\ 00165 "punpcklbw %%mm7, %%mm1 \n\t"\ 00166 "punpcklbw %%mm7, %%mm2 \n\t"\ 00167 "punpcklbw %%mm7, %%mm3 \n\t"\ 00168 "punpcklbw %%mm7, %%mm4 \n\t"\ 00169 QPEL_H264V(%%mm0, %%mm1, %%mm2, %%mm3, %%mm4, %%mm5, OP)\ 00170 QPEL_H264V(%%mm1, %%mm2, %%mm3, %%mm4, %%mm5, %%mm0, OP)\ 00171 QPEL_H264V(%%mm2, %%mm3, %%mm4, %%mm5, %%mm0, %%mm1, OP)\ 00172 QPEL_H264V(%%mm3, %%mm4, %%mm5, %%mm0, %%mm1, %%mm2, OP)\ 00173 \ 00174 : "+a"(src), "+c"(dst)\ 00175 : "S"((x86_reg)srcStride), "D"((x86_reg)dstStride), "m"(ff_pw_5), "m"(ff_pw_16)\ 00176 : "memory"\ 00177 );\ 00178 }\ 00179 static av_noinline void OPNAME ## h264_qpel4_hv_lowpass_ ## MMX(uint8_t *dst, int16_t *tmp, uint8_t *src, int dstStride, int tmpStride, int srcStride){\ 00180 int h=4;\ 00181 int w=3;\ 00182 src -= 2*srcStride+2;\ 00183 while(w--){\ 00184 __asm__ volatile(\ 00185 "pxor %%mm7, %%mm7 \n\t"\ 00186 "movd (%0), %%mm0 \n\t"\ 00187 "add %2, %0 \n\t"\ 00188 "movd (%0), %%mm1 \n\t"\ 00189 "add %2, %0 \n\t"\ 00190 "movd (%0), %%mm2 \n\t"\ 00191 "add %2, %0 \n\t"\ 00192 "movd (%0), %%mm3 \n\t"\ 00193 "add %2, %0 \n\t"\ 00194 "movd (%0), %%mm4 \n\t"\ 00195 "add %2, %0 \n\t"\ 00196 "punpcklbw %%mm7, %%mm0 \n\t"\ 00197 "punpcklbw %%mm7, %%mm1 \n\t"\ 00198 "punpcklbw %%mm7, %%mm2 \n\t"\ 00199 "punpcklbw %%mm7, %%mm3 \n\t"\ 00200 "punpcklbw %%mm7, %%mm4 \n\t"\ 00201 QPEL_H264HV(%%mm0, %%mm1, %%mm2, %%mm3, %%mm4, %%mm5, 0*8*3)\ 00202 QPEL_H264HV(%%mm1, %%mm2, %%mm3, %%mm4, %%mm5, %%mm0, 1*8*3)\ 00203 QPEL_H264HV(%%mm2, %%mm3, %%mm4, %%mm5, %%mm0, %%mm1, 2*8*3)\ 00204 QPEL_H264HV(%%mm3, %%mm4, %%mm5, %%mm0, %%mm1, %%mm2, 3*8*3)\ 00205 \ 00206 : "+a"(src)\ 00207 : "c"(tmp), "S"((x86_reg)srcStride)\ 00208 : "memory"\ 00209 );\ 00210 tmp += 4;\ 00211 src += 4 - 9*srcStride;\ 00212 }\ 00213 tmp -= 3*4;\ 00214 __asm__ volatile(\ 00215 "1: \n\t"\ 00216 "movq (%0), %%mm0 \n\t"\ 00217 "paddw 10(%0), %%mm0 \n\t"\ 00218 "movq 2(%0), %%mm1 \n\t"\ 00219 "paddw 8(%0), %%mm1 \n\t"\ 00220 "movq 4(%0), %%mm2 \n\t"\ 00221 "paddw 6(%0), %%mm2 \n\t"\ 00222 "psubw %%mm1, %%mm0 \n\t"/*a-b (abccba)*/\ 00223 "psraw $2, %%mm0 \n\t"/*(a-b)/4 */\ 00224 "psubw %%mm1, %%mm0 \n\t"/*(a-b)/4-b */\ 00225 "paddsw %%mm2, %%mm0 \n\t"\ 00226 "psraw $2, %%mm0 \n\t"/*((a-b)/4-b+c)/4 */\ 00227 "paddw %%mm2, %%mm0 \n\t"/*(a-5*b+20*c)/16 */\ 00228 "psraw $6, %%mm0 \n\t"\ 00229 "packuswb %%mm0, %%mm0 \n\t"\ 00230 OP(%%mm0, (%1),%%mm7, d)\ 00231 "add $24, %0 \n\t"\ 00232 "add %3, %1 \n\t"\ 00233 "decl %2 \n\t"\ 00234 " jnz 1b \n\t"\ 00235 : "+a"(tmp), "+c"(dst), "+g"(h)\ 00236 : "S"((x86_reg)dstStride)\ 00237 : "memory"\ 00238 );\ 00239 }\ 00240 \ 00241 static av_noinline void OPNAME ## h264_qpel8_h_lowpass_ ## MMX(uint8_t *dst, uint8_t *src, int dstStride, int srcStride){\ 00242 int h=8;\ 00243 __asm__ volatile(\ 00244 "pxor %%mm7, %%mm7 \n\t"\ 00245 "movq "MANGLE(ff_pw_5)", %%mm6\n\t"\ 00246 "1: \n\t"\ 00247 "movq (%0), %%mm0 \n\t"\ 00248 "movq 1(%0), %%mm2 \n\t"\ 00249 "movq %%mm0, %%mm1 \n\t"\ 00250 "movq %%mm2, %%mm3 \n\t"\ 00251 "punpcklbw %%mm7, %%mm0 \n\t"\ 00252 "punpckhbw %%mm7, %%mm1 \n\t"\ 00253 "punpcklbw %%mm7, %%mm2 \n\t"\ 00254 "punpckhbw %%mm7, %%mm3 \n\t"\ 00255 "paddw %%mm2, %%mm0 \n\t"\ 00256 "paddw %%mm3, %%mm1 \n\t"\ 00257 "psllw $2, %%mm0 \n\t"\ 00258 "psllw $2, %%mm1 \n\t"\ 00259 "movq -1(%0), %%mm2 \n\t"\ 00260 "movq 2(%0), %%mm4 \n\t"\ 00261 "movq %%mm2, %%mm3 \n\t"\ 00262 "movq %%mm4, %%mm5 \n\t"\ 00263 "punpcklbw %%mm7, %%mm2 \n\t"\ 00264 "punpckhbw %%mm7, %%mm3 \n\t"\ 00265 "punpcklbw %%mm7, %%mm4 \n\t"\ 00266 "punpckhbw %%mm7, %%mm5 \n\t"\ 00267 "paddw %%mm4, %%mm2 \n\t"\ 00268 "paddw %%mm3, %%mm5 \n\t"\ 00269 "psubw %%mm2, %%mm0 \n\t"\ 00270 "psubw %%mm5, %%mm1 \n\t"\ 00271 "pmullw %%mm6, %%mm0 \n\t"\ 00272 "pmullw %%mm6, %%mm1 \n\t"\ 00273 "movd -2(%0), %%mm2 \n\t"\ 00274 "movd 7(%0), %%mm5 \n\t"\ 00275 "punpcklbw %%mm7, %%mm2 \n\t"\ 00276 "punpcklbw %%mm7, %%mm5 \n\t"\ 00277 "paddw %%mm3, %%mm2 \n\t"\ 00278 "paddw %%mm5, %%mm4 \n\t"\ 00279 "movq "MANGLE(ff_pw_16)", %%mm5\n\t"\ 00280 "paddw %%mm5, %%mm2 \n\t"\ 00281 "paddw %%mm5, %%mm4 \n\t"\ 00282 "paddw %%mm2, %%mm0 \n\t"\ 00283 "paddw %%mm4, %%mm1 \n\t"\ 00284 "psraw $5, %%mm0 \n\t"\ 00285 "psraw $5, %%mm1 \n\t"\ 00286 "packuswb %%mm1, %%mm0 \n\t"\ 00287 OP(%%mm0, (%1),%%mm5, q)\ 00288 "add %3, %0 \n\t"\ 00289 "add %4, %1 \n\t"\ 00290 "decl %2 \n\t"\ 00291 " jnz 1b \n\t"\ 00292 : "+a"(src), "+c"(dst), "+g"(h)\ 00293 : "d"((x86_reg)srcStride), "S"((x86_reg)dstStride)\ 00294 : "memory"\ 00295 );\ 00296 }\ 00297 \ 00298 static av_noinline void OPNAME ## h264_qpel8_h_lowpass_l2_ ## MMX(uint8_t *dst, uint8_t *src, uint8_t *src2, int dstStride, int src2Stride){\ 00299 int h=8;\ 00300 __asm__ volatile(\ 00301 "pxor %%mm7, %%mm7 \n\t"\ 00302 "movq "MANGLE(ff_pw_5)", %%mm6\n\t"\ 00303 "1: \n\t"\ 00304 "movq (%0), %%mm0 \n\t"\ 00305 "movq 1(%0), %%mm2 \n\t"\ 00306 "movq %%mm0, %%mm1 \n\t"\ 00307 "movq %%mm2, %%mm3 \n\t"\ 00308 "punpcklbw %%mm7, %%mm0 \n\t"\ 00309 "punpckhbw %%mm7, %%mm1 \n\t"\ 00310 "punpcklbw %%mm7, %%mm2 \n\t"\ 00311 "punpckhbw %%mm7, %%mm3 \n\t"\ 00312 "paddw %%mm2, %%mm0 \n\t"\ 00313 "paddw %%mm3, %%mm1 \n\t"\ 00314 "psllw $2, %%mm0 \n\t"\ 00315 "psllw $2, %%mm1 \n\t"\ 00316 "movq -1(%0), %%mm2 \n\t"\ 00317 "movq 2(%0), %%mm4 \n\t"\ 00318 "movq %%mm2, %%mm3 \n\t"\ 00319 "movq %%mm4, %%mm5 \n\t"\ 00320 "punpcklbw %%mm7, %%mm2 \n\t"\ 00321 "punpckhbw %%mm7, %%mm3 \n\t"\ 00322 "punpcklbw %%mm7, %%mm4 \n\t"\ 00323 "punpckhbw %%mm7, %%mm5 \n\t"\ 00324 "paddw %%mm4, %%mm2 \n\t"\ 00325 "paddw %%mm3, %%mm5 \n\t"\ 00326 "psubw %%mm2, %%mm0 \n\t"\ 00327 "psubw %%mm5, %%mm1 \n\t"\ 00328 "pmullw %%mm6, %%mm0 \n\t"\ 00329 "pmullw %%mm6, %%mm1 \n\t"\ 00330 "movd -2(%0), %%mm2 \n\t"\ 00331 "movd 7(%0), %%mm5 \n\t"\ 00332 "punpcklbw %%mm7, %%mm2 \n\t"\ 00333 "punpcklbw %%mm7, %%mm5 \n\t"\ 00334 "paddw %%mm3, %%mm2 \n\t"\ 00335 "paddw %%mm5, %%mm4 \n\t"\ 00336 "movq "MANGLE(ff_pw_16)", %%mm5\n\t"\ 00337 "paddw %%mm5, %%mm2 \n\t"\ 00338 "paddw %%mm5, %%mm4 \n\t"\ 00339 "paddw %%mm2, %%mm0 \n\t"\ 00340 "paddw %%mm4, %%mm1 \n\t"\ 00341 "psraw $5, %%mm0 \n\t"\ 00342 "psraw $5, %%mm1 \n\t"\ 00343 "movq (%2), %%mm4 \n\t"\ 00344 "packuswb %%mm1, %%mm0 \n\t"\ 00345 PAVGB" %%mm4, %%mm0 \n\t"\ 00346 OP(%%mm0, (%1),%%mm5, q)\ 00347 "add %5, %0 \n\t"\ 00348 "add %5, %1 \n\t"\ 00349 "add %4, %2 \n\t"\ 00350 "decl %3 \n\t"\ 00351 "jg 1b \n\t"\ 00352 : "+a"(src), "+c"(dst), "+d"(src2), "+g"(h)\ 00353 : "D"((x86_reg)src2Stride), "S"((x86_reg)dstStride)\ 00354 : "memory"\ 00355 );\ 00356 }\ 00357 \ 00358 static av_noinline void OPNAME ## h264_qpel8or16_v_lowpass_ ## MMX(uint8_t *dst, uint8_t *src, int dstStride, int srcStride, int h){\ 00359 int w= 2;\ 00360 src -= 2*srcStride;\ 00361 \ 00362 while(w--){\ 00363 __asm__ volatile(\ 00364 "pxor %%mm7, %%mm7 \n\t"\ 00365 "movd (%0), %%mm0 \n\t"\ 00366 "add %2, %0 \n\t"\ 00367 "movd (%0), %%mm1 \n\t"\ 00368 "add %2, %0 \n\t"\ 00369 "movd (%0), %%mm2 \n\t"\ 00370 "add %2, %0 \n\t"\ 00371 "movd (%0), %%mm3 \n\t"\ 00372 "add %2, %0 \n\t"\ 00373 "movd (%0), %%mm4 \n\t"\ 00374 "add %2, %0 \n\t"\ 00375 "punpcklbw %%mm7, %%mm0 \n\t"\ 00376 "punpcklbw %%mm7, %%mm1 \n\t"\ 00377 "punpcklbw %%mm7, %%mm2 \n\t"\ 00378 "punpcklbw %%mm7, %%mm3 \n\t"\ 00379 "punpcklbw %%mm7, %%mm4 \n\t"\ 00380 QPEL_H264V(%%mm0, %%mm1, %%mm2, %%mm3, %%mm4, %%mm5, OP)\ 00381 QPEL_H264V(%%mm1, %%mm2, %%mm3, %%mm4, %%mm5, %%mm0, OP)\ 00382 QPEL_H264V(%%mm2, %%mm3, %%mm4, %%mm5, %%mm0, %%mm1, OP)\ 00383 QPEL_H264V(%%mm3, %%mm4, %%mm5, %%mm0, %%mm1, %%mm2, OP)\ 00384 QPEL_H264V(%%mm4, %%mm5, %%mm0, %%mm1, %%mm2, %%mm3, OP)\ 00385 QPEL_H264V(%%mm5, %%mm0, %%mm1, %%mm2, %%mm3, %%mm4, OP)\ 00386 QPEL_H264V(%%mm0, %%mm1, %%mm2, %%mm3, %%mm4, %%mm5, OP)\ 00387 QPEL_H264V(%%mm1, %%mm2, %%mm3, %%mm4, %%mm5, %%mm0, OP)\ 00388 "cmpl $16, %4 \n\t"\ 00389 "jne 2f \n\t"\ 00390 QPEL_H264V(%%mm2, %%mm3, %%mm4, %%mm5, %%mm0, %%mm1, OP)\ 00391 QPEL_H264V(%%mm3, %%mm4, %%mm5, %%mm0, %%mm1, %%mm2, OP)\ 00392 QPEL_H264V(%%mm4, %%mm5, %%mm0, %%mm1, %%mm2, %%mm3, OP)\ 00393 QPEL_H264V(%%mm5, %%mm0, %%mm1, %%mm2, %%mm3, %%mm4, OP)\ 00394 QPEL_H264V(%%mm0, %%mm1, %%mm2, %%mm3, %%mm4, %%mm5, OP)\ 00395 QPEL_H264V(%%mm1, %%mm2, %%mm3, %%mm4, %%mm5, %%mm0, OP)\ 00396 QPEL_H264V(%%mm2, %%mm3, %%mm4, %%mm5, %%mm0, %%mm1, OP)\ 00397 QPEL_H264V(%%mm3, %%mm4, %%mm5, %%mm0, %%mm1, %%mm2, OP)\ 00398 "2: \n\t"\ 00399 \ 00400 : "+a"(src), "+c"(dst)\ 00401 : "S"((x86_reg)srcStride), "D"((x86_reg)dstStride), "rm"(h)\ 00402 : "memory"\ 00403 );\ 00404 src += 4-(h+5)*srcStride;\ 00405 dst += 4-h*dstStride;\ 00406 }\ 00407 }\ 00408 static av_always_inline void OPNAME ## h264_qpel8or16_hv1_lowpass_ ## MMX(int16_t *tmp, uint8_t *src, int tmpStride, int srcStride, int size){\ 00409 int w = (size+8)>>2;\ 00410 src -= 2*srcStride+2;\ 00411 while(w--){\ 00412 __asm__ volatile(\ 00413 "pxor %%mm7, %%mm7 \n\t"\ 00414 "movd (%0), %%mm0 \n\t"\ 00415 "add %2, %0 \n\t"\ 00416 "movd (%0), %%mm1 \n\t"\ 00417 "add %2, %0 \n\t"\ 00418 "movd (%0), %%mm2 \n\t"\ 00419 "add %2, %0 \n\t"\ 00420 "movd (%0), %%mm3 \n\t"\ 00421 "add %2, %0 \n\t"\ 00422 "movd (%0), %%mm4 \n\t"\ 00423 "add %2, %0 \n\t"\ 00424 "punpcklbw %%mm7, %%mm0 \n\t"\ 00425 "punpcklbw %%mm7, %%mm1 \n\t"\ 00426 "punpcklbw %%mm7, %%mm2 \n\t"\ 00427 "punpcklbw %%mm7, %%mm3 \n\t"\ 00428 "punpcklbw %%mm7, %%mm4 \n\t"\ 00429 QPEL_H264HV(%%mm0, %%mm1, %%mm2, %%mm3, %%mm4, %%mm5, 0*48)\ 00430 QPEL_H264HV(%%mm1, %%mm2, %%mm3, %%mm4, %%mm5, %%mm0, 1*48)\ 00431 QPEL_H264HV(%%mm2, %%mm3, %%mm4, %%mm5, %%mm0, %%mm1, 2*48)\ 00432 QPEL_H264HV(%%mm3, %%mm4, %%mm5, %%mm0, %%mm1, %%mm2, 3*48)\ 00433 QPEL_H264HV(%%mm4, %%mm5, %%mm0, %%mm1, %%mm2, %%mm3, 4*48)\ 00434 QPEL_H264HV(%%mm5, %%mm0, %%mm1, %%mm2, %%mm3, %%mm4, 5*48)\ 00435 QPEL_H264HV(%%mm0, %%mm1, %%mm2, %%mm3, %%mm4, %%mm5, 6*48)\ 00436 QPEL_H264HV(%%mm1, %%mm2, %%mm3, %%mm4, %%mm5, %%mm0, 7*48)\ 00437 "cmpl $16, %3 \n\t"\ 00438 "jne 2f \n\t"\ 00439 QPEL_H264HV(%%mm2, %%mm3, %%mm4, %%mm5, %%mm0, %%mm1, 8*48)\ 00440 QPEL_H264HV(%%mm3, %%mm4, %%mm5, %%mm0, %%mm1, %%mm2, 9*48)\ 00441 QPEL_H264HV(%%mm4, %%mm5, %%mm0, %%mm1, %%mm2, %%mm3, 10*48)\ 00442 QPEL_H264HV(%%mm5, %%mm0, %%mm1, %%mm2, %%mm3, %%mm4, 11*48)\ 00443 QPEL_H264HV(%%mm0, %%mm1, %%mm2, %%mm3, %%mm4, %%mm5, 12*48)\ 00444 QPEL_H264HV(%%mm1, %%mm2, %%mm3, %%mm4, %%mm5, %%mm0, 13*48)\ 00445 QPEL_H264HV(%%mm2, %%mm3, %%mm4, %%mm5, %%mm0, %%mm1, 14*48)\ 00446 QPEL_H264HV(%%mm3, %%mm4, %%mm5, %%mm0, %%mm1, %%mm2, 15*48)\ 00447 "2: \n\t"\ 00448 : "+a"(src)\ 00449 : "c"(tmp), "S"((x86_reg)srcStride), "rm"(size)\ 00450 : "memory"\ 00451 );\ 00452 tmp += 4;\ 00453 src += 4 - (size+5)*srcStride;\ 00454 }\ 00455 }\ 00456 static av_always_inline void OPNAME ## h264_qpel8or16_hv2_lowpass_ ## MMX(uint8_t *dst, int16_t *tmp, int dstStride, int tmpStride, int size){\ 00457 int w = size>>4;\ 00458 do{\ 00459 int h = size;\ 00460 __asm__ volatile(\ 00461 "1: \n\t"\ 00462 "movq (%0), %%mm0 \n\t"\ 00463 "movq 8(%0), %%mm3 \n\t"\ 00464 "movq 2(%0), %%mm1 \n\t"\ 00465 "movq 10(%0), %%mm4 \n\t"\ 00466 "paddw %%mm4, %%mm0 \n\t"\ 00467 "paddw %%mm3, %%mm1 \n\t"\ 00468 "paddw 18(%0), %%mm3 \n\t"\ 00469 "paddw 16(%0), %%mm4 \n\t"\ 00470 "movq 4(%0), %%mm2 \n\t"\ 00471 "movq 12(%0), %%mm5 \n\t"\ 00472 "paddw 6(%0), %%mm2 \n\t"\ 00473 "paddw 14(%0), %%mm5 \n\t"\ 00474 "psubw %%mm1, %%mm0 \n\t"\ 00475 "psubw %%mm4, %%mm3 \n\t"\ 00476 "psraw $2, %%mm0 \n\t"\ 00477 "psraw $2, %%mm3 \n\t"\ 00478 "psubw %%mm1, %%mm0 \n\t"\ 00479 "psubw %%mm4, %%mm3 \n\t"\ 00480 "paddsw %%mm2, %%mm0 \n\t"\ 00481 "paddsw %%mm5, %%mm3 \n\t"\ 00482 "psraw $2, %%mm0 \n\t"\ 00483 "psraw $2, %%mm3 \n\t"\ 00484 "paddw %%mm2, %%mm0 \n\t"\ 00485 "paddw %%mm5, %%mm3 \n\t"\ 00486 "psraw $6, %%mm0 \n\t"\ 00487 "psraw $6, %%mm3 \n\t"\ 00488 "packuswb %%mm3, %%mm0 \n\t"\ 00489 OP(%%mm0, (%1),%%mm7, q)\ 00490 "add $48, %0 \n\t"\ 00491 "add %3, %1 \n\t"\ 00492 "decl %2 \n\t"\ 00493 " jnz 1b \n\t"\ 00494 : "+a"(tmp), "+c"(dst), "+g"(h)\ 00495 : "S"((x86_reg)dstStride)\ 00496 : "memory"\ 00497 );\ 00498 tmp += 8 - size*24;\ 00499 dst += 8 - size*dstStride;\ 00500 }while(w--);\ 00501 }\ 00502 \ 00503 static void OPNAME ## h264_qpel8_v_lowpass_ ## MMX(uint8_t *dst, uint8_t *src, int dstStride, int srcStride){\ 00504 OPNAME ## h264_qpel8or16_v_lowpass_ ## MMX(dst , src , dstStride, srcStride, 8);\ 00505 }\ 00506 static av_noinline void OPNAME ## h264_qpel16_v_lowpass_ ## MMX(uint8_t *dst, uint8_t *src, int dstStride, int srcStride){\ 00507 OPNAME ## h264_qpel8or16_v_lowpass_ ## MMX(dst , src , dstStride, srcStride, 16);\ 00508 OPNAME ## h264_qpel8or16_v_lowpass_ ## MMX(dst+8, src+8, dstStride, srcStride, 16);\ 00509 }\ 00510 \ 00511 static void OPNAME ## h264_qpel16_h_lowpass_ ## MMX(uint8_t *dst, uint8_t *src, int dstStride, int srcStride){\ 00512 OPNAME ## h264_qpel8_h_lowpass_ ## MMX(dst , src , dstStride, srcStride);\ 00513 OPNAME ## h264_qpel8_h_lowpass_ ## MMX(dst+8, src+8, dstStride, srcStride);\ 00514 src += 8*srcStride;\ 00515 dst += 8*dstStride;\ 00516 OPNAME ## h264_qpel8_h_lowpass_ ## MMX(dst , src , dstStride, srcStride);\ 00517 OPNAME ## h264_qpel8_h_lowpass_ ## MMX(dst+8, src+8, dstStride, srcStride);\ 00518 }\ 00519 \ 00520 static av_noinline void OPNAME ## h264_qpel16_h_lowpass_l2_ ## MMX(uint8_t *dst, uint8_t *src, uint8_t *src2, int dstStride, int src2Stride){\ 00521 OPNAME ## h264_qpel8_h_lowpass_l2_ ## MMX(dst , src , src2 , dstStride, src2Stride);\ 00522 OPNAME ## h264_qpel8_h_lowpass_l2_ ## MMX(dst+8, src+8, src2+8, dstStride, src2Stride);\ 00523 src += 8*dstStride;\ 00524 dst += 8*dstStride;\ 00525 src2 += 8*src2Stride;\ 00526 OPNAME ## h264_qpel8_h_lowpass_l2_ ## MMX(dst , src , src2 , dstStride, src2Stride);\ 00527 OPNAME ## h264_qpel8_h_lowpass_l2_ ## MMX(dst+8, src+8, src2+8, dstStride, src2Stride);\ 00528 }\ 00529 \ 00530 static av_noinline void OPNAME ## h264_qpel8or16_hv_lowpass_ ## MMX(uint8_t *dst, int16_t *tmp, uint8_t *src, int dstStride, int tmpStride, int srcStride, int size){\ 00531 put_h264_qpel8or16_hv1_lowpass_ ## MMX(tmp, src, tmpStride, srcStride, size);\ 00532 OPNAME ## h264_qpel8or16_hv2_lowpass_ ## MMX(dst, tmp, dstStride, tmpStride, size);\ 00533 }\ 00534 static void OPNAME ## h264_qpel8_hv_lowpass_ ## MMX(uint8_t *dst, int16_t *tmp, uint8_t *src, int dstStride, int tmpStride, int srcStride){\ 00535 OPNAME ## h264_qpel8or16_hv_lowpass_ ## MMX(dst , tmp , src , dstStride, tmpStride, srcStride, 8);\ 00536 }\ 00537 \ 00538 static void OPNAME ## h264_qpel16_hv_lowpass_ ## MMX(uint8_t *dst, int16_t *tmp, uint8_t *src, int dstStride, int tmpStride, int srcStride){\ 00539 OPNAME ## h264_qpel8or16_hv_lowpass_ ## MMX(dst , tmp , src , dstStride, tmpStride, srcStride, 16);\ 00540 }\ 00541 \ 00542 static av_noinline void OPNAME ## pixels4_l2_shift5_ ## MMX(uint8_t *dst, int16_t *src16, uint8_t *src8, int dstStride, int src8Stride, int h)\ 00543 {\ 00544 __asm__ volatile(\ 00545 "movq (%1), %%mm0 \n\t"\ 00546 "movq 24(%1), %%mm1 \n\t"\ 00547 "psraw $5, %%mm0 \n\t"\ 00548 "psraw $5, %%mm1 \n\t"\ 00549 "packuswb %%mm0, %%mm0 \n\t"\ 00550 "packuswb %%mm1, %%mm1 \n\t"\ 00551 PAVGB" (%0), %%mm0 \n\t"\ 00552 PAVGB" (%0,%3), %%mm1 \n\t"\ 00553 OP(%%mm0, (%2), %%mm4, d)\ 00554 OP(%%mm1, (%2,%4), %%mm5, d)\ 00555 "lea (%0,%3,2), %0 \n\t"\ 00556 "lea (%2,%4,2), %2 \n\t"\ 00557 "movq 48(%1), %%mm0 \n\t"\ 00558 "movq 72(%1), %%mm1 \n\t"\ 00559 "psraw $5, %%mm0 \n\t"\ 00560 "psraw $5, %%mm1 \n\t"\ 00561 "packuswb %%mm0, %%mm0 \n\t"\ 00562 "packuswb %%mm1, %%mm1 \n\t"\ 00563 PAVGB" (%0), %%mm0 \n\t"\ 00564 PAVGB" (%0,%3), %%mm1 \n\t"\ 00565 OP(%%mm0, (%2), %%mm4, d)\ 00566 OP(%%mm1, (%2,%4), %%mm5, d)\ 00567 :"+a"(src8), "+c"(src16), "+d"(dst)\ 00568 :"S"((x86_reg)src8Stride), "D"((x86_reg)dstStride)\ 00569 :"memory");\ 00570 }\ 00571 static av_noinline void OPNAME ## pixels8_l2_shift5_ ## MMX(uint8_t *dst, int16_t *src16, uint8_t *src8, int dstStride, int src8Stride, int h)\ 00572 {\ 00573 do{\ 00574 __asm__ volatile(\ 00575 "movq (%1), %%mm0 \n\t"\ 00576 "movq 8(%1), %%mm1 \n\t"\ 00577 "movq 48(%1), %%mm2 \n\t"\ 00578 "movq 8+48(%1), %%mm3 \n\t"\ 00579 "psraw $5, %%mm0 \n\t"\ 00580 "psraw $5, %%mm1 \n\t"\ 00581 "psraw $5, %%mm2 \n\t"\ 00582 "psraw $5, %%mm3 \n\t"\ 00583 "packuswb %%mm1, %%mm0 \n\t"\ 00584 "packuswb %%mm3, %%mm2 \n\t"\ 00585 PAVGB" (%0), %%mm0 \n\t"\ 00586 PAVGB" (%0,%3), %%mm2 \n\t"\ 00587 OP(%%mm0, (%2), %%mm5, q)\ 00588 OP(%%mm2, (%2,%4), %%mm5, q)\ 00589 ::"a"(src8), "c"(src16), "d"(dst),\ 00590 "r"((x86_reg)src8Stride), "r"((x86_reg)dstStride)\ 00591 :"memory");\ 00592 src8 += 2L*src8Stride;\ 00593 src16 += 48;\ 00594 dst += 2L*dstStride;\ 00595 }while(h-=2);\ 00596 }\ 00597 static void OPNAME ## pixels16_l2_shift5_ ## MMX(uint8_t *dst, int16_t *src16, uint8_t *src8, int dstStride, int src8Stride, int h)\ 00598 {\ 00599 OPNAME ## pixels8_l2_shift5_ ## MMX(dst , src16 , src8 , dstStride, src8Stride, h);\ 00600 OPNAME ## pixels8_l2_shift5_ ## MMX(dst+8, src16+8, src8+8, dstStride, src8Stride, h);\ 00601 }\ 00602 00603 00604 #if ARCH_X86_64 00605 #define QPEL_H264_H16_XMM(OPNAME, OP, MMX)\ 00606 static av_noinline void OPNAME ## h264_qpel16_h_lowpass_l2_ ## MMX(uint8_t *dst, uint8_t *src, uint8_t *src2, int dstStride, int src2Stride){\ 00607 int h=16;\ 00608 __asm__ volatile(\ 00609 "pxor %%xmm15, %%xmm15 \n\t"\ 00610 "movdqa %6, %%xmm14 \n\t"\ 00611 "movdqa %7, %%xmm13 \n\t"\ 00612 "1: \n\t"\ 00613 "lddqu 6(%0), %%xmm1 \n\t"\ 00614 "lddqu -2(%0), %%xmm7 \n\t"\ 00615 "movdqa %%xmm1, %%xmm0 \n\t"\ 00616 "punpckhbw %%xmm15, %%xmm1 \n\t"\ 00617 "punpcklbw %%xmm15, %%xmm0 \n\t"\ 00618 "punpcklbw %%xmm15, %%xmm7 \n\t"\ 00619 "movdqa %%xmm1, %%xmm2 \n\t"\ 00620 "movdqa %%xmm0, %%xmm6 \n\t"\ 00621 "movdqa %%xmm1, %%xmm3 \n\t"\ 00622 "movdqa %%xmm0, %%xmm8 \n\t"\ 00623 "movdqa %%xmm1, %%xmm4 \n\t"\ 00624 "movdqa %%xmm0, %%xmm9 \n\t"\ 00625 "movdqa %%xmm0, %%xmm12 \n\t"\ 00626 "movdqa %%xmm1, %%xmm11 \n\t"\ 00627 "palignr $10,%%xmm0, %%xmm11\n\t"\ 00628 "palignr $10,%%xmm7, %%xmm12\n\t"\ 00629 "palignr $2, %%xmm0, %%xmm4 \n\t"\ 00630 "palignr $2, %%xmm7, %%xmm9 \n\t"\ 00631 "palignr $4, %%xmm0, %%xmm3 \n\t"\ 00632 "palignr $4, %%xmm7, %%xmm8 \n\t"\ 00633 "palignr $6, %%xmm0, %%xmm2 \n\t"\ 00634 "palignr $6, %%xmm7, %%xmm6 \n\t"\ 00635 "paddw %%xmm0 ,%%xmm11 \n\t"\ 00636 "palignr $8, %%xmm0, %%xmm1 \n\t"\ 00637 "palignr $8, %%xmm7, %%xmm0 \n\t"\ 00638 "paddw %%xmm12,%%xmm7 \n\t"\ 00639 "paddw %%xmm3, %%xmm2 \n\t"\ 00640 "paddw %%xmm8, %%xmm6 \n\t"\ 00641 "paddw %%xmm4, %%xmm1 \n\t"\ 00642 "paddw %%xmm9, %%xmm0 \n\t"\ 00643 "psllw $2, %%xmm2 \n\t"\ 00644 "psllw $2, %%xmm6 \n\t"\ 00645 "psubw %%xmm1, %%xmm2 \n\t"\ 00646 "psubw %%xmm0, %%xmm6 \n\t"\ 00647 "paddw %%xmm13,%%xmm11 \n\t"\ 00648 "paddw %%xmm13,%%xmm7 \n\t"\ 00649 "pmullw %%xmm14,%%xmm2 \n\t"\ 00650 "pmullw %%xmm14,%%xmm6 \n\t"\ 00651 "lddqu (%2), %%xmm3 \n\t"\ 00652 "paddw %%xmm11,%%xmm2 \n\t"\ 00653 "paddw %%xmm7, %%xmm6 \n\t"\ 00654 "psraw $5, %%xmm2 \n\t"\ 00655 "psraw $5, %%xmm6 \n\t"\ 00656 "packuswb %%xmm2,%%xmm6 \n\t"\ 00657 "pavgb %%xmm3, %%xmm6 \n\t"\ 00658 OP(%%xmm6, (%1), %%xmm4, dqa)\ 00659 "add %5, %0 \n\t"\ 00660 "add %5, %1 \n\t"\ 00661 "add %4, %2 \n\t"\ 00662 "decl %3 \n\t"\ 00663 "jg 1b \n\t"\ 00664 : "+a"(src), "+c"(dst), "+d"(src2), "+g"(h)\ 00665 : "D"((x86_reg)src2Stride), "S"((x86_reg)dstStride),\ 00666 "m"(ff_pw_5), "m"(ff_pw_16)\ 00667 : XMM_CLOBBERS("%xmm0" , "%xmm1" , "%xmm2" , "%xmm3" , \ 00668 "%xmm4" , "%xmm5" , "%xmm6" , "%xmm7" , \ 00669 "%xmm8" , "%xmm9" , "%xmm10", "%xmm11", \ 00670 "%xmm12", "%xmm13", "%xmm14", "%xmm15",)\ 00671 "memory"\ 00672 );\ 00673 } 00674 #else // ARCH_X86_64 00675 #define QPEL_H264_H16_XMM(OPNAME, OP, MMX)\ 00676 static av_noinline void OPNAME ## h264_qpel16_h_lowpass_l2_ ## MMX(uint8_t *dst, uint8_t *src, uint8_t *src2, int dstStride, int src2Stride){\ 00677 OPNAME ## h264_qpel8_h_lowpass_l2_ ## MMX(dst , src , src2 , dstStride, src2Stride);\ 00678 OPNAME ## h264_qpel8_h_lowpass_l2_ ## MMX(dst+8, src+8, src2+8, dstStride, src2Stride);\ 00679 src += 8*dstStride;\ 00680 dst += 8*dstStride;\ 00681 src2 += 8*src2Stride;\ 00682 OPNAME ## h264_qpel8_h_lowpass_l2_ ## MMX(dst , src , src2 , dstStride, src2Stride);\ 00683 OPNAME ## h264_qpel8_h_lowpass_l2_ ## MMX(dst+8, src+8, src2+8, dstStride, src2Stride);\ 00684 } 00685 #endif // ARCH_X86_64 00686 00687 #define QPEL_H264_H_XMM(OPNAME, OP, MMX)\ 00688 static av_noinline void OPNAME ## h264_qpel8_h_lowpass_l2_ ## MMX(uint8_t *dst, uint8_t *src, uint8_t *src2, int dstStride, int src2Stride){\ 00689 int h=8;\ 00690 __asm__ volatile(\ 00691 "pxor %%xmm7, %%xmm7 \n\t"\ 00692 "movdqa "MANGLE(ff_pw_5)", %%xmm6\n\t"\ 00693 "1: \n\t"\ 00694 "lddqu -2(%0), %%xmm1 \n\t"\ 00695 "movdqa %%xmm1, %%xmm0 \n\t"\ 00696 "punpckhbw %%xmm7, %%xmm1 \n\t"\ 00697 "punpcklbw %%xmm7, %%xmm0 \n\t"\ 00698 "movdqa %%xmm1, %%xmm2 \n\t"\ 00699 "movdqa %%xmm1, %%xmm3 \n\t"\ 00700 "movdqa %%xmm1, %%xmm4 \n\t"\ 00701 "movdqa %%xmm1, %%xmm5 \n\t"\ 00702 "palignr $2, %%xmm0, %%xmm4 \n\t"\ 00703 "palignr $4, %%xmm0, %%xmm3 \n\t"\ 00704 "palignr $6, %%xmm0, %%xmm2 \n\t"\ 00705 "palignr $8, %%xmm0, %%xmm1 \n\t"\ 00706 "palignr $10,%%xmm0, %%xmm5 \n\t"\ 00707 "paddw %%xmm5, %%xmm0 \n\t"\ 00708 "paddw %%xmm3, %%xmm2 \n\t"\ 00709 "paddw %%xmm4, %%xmm1 \n\t"\ 00710 "psllw $2, %%xmm2 \n\t"\ 00711 "movq (%2), %%xmm3 \n\t"\ 00712 "psubw %%xmm1, %%xmm2 \n\t"\ 00713 "paddw "MANGLE(ff_pw_16)", %%xmm0\n\t"\ 00714 "pmullw %%xmm6, %%xmm2 \n\t"\ 00715 "paddw %%xmm0, %%xmm2 \n\t"\ 00716 "psraw $5, %%xmm2 \n\t"\ 00717 "packuswb %%xmm2, %%xmm2 \n\t"\ 00718 "pavgb %%xmm3, %%xmm2 \n\t"\ 00719 OP(%%xmm2, (%1), %%xmm4, q)\ 00720 "add %5, %0 \n\t"\ 00721 "add %5, %1 \n\t"\ 00722 "add %4, %2 \n\t"\ 00723 "decl %3 \n\t"\ 00724 "jg 1b \n\t"\ 00725 : "+a"(src), "+c"(dst), "+d"(src2), "+g"(h)\ 00726 : "D"((x86_reg)src2Stride), "S"((x86_reg)dstStride)\ 00727 : XMM_CLOBBERS("%xmm0", "%xmm1", "%xmm2", "%xmm3", \ 00728 "%xmm4", "%xmm5", "%xmm6", "%xmm7",)\ 00729 "memory"\ 00730 );\ 00731 }\ 00732 QPEL_H264_H16_XMM(OPNAME, OP, MMX)\ 00733 \ 00734 static av_noinline void OPNAME ## h264_qpel8_h_lowpass_ ## MMX(uint8_t *dst, uint8_t *src, int dstStride, int srcStride){\ 00735 int h=8;\ 00736 __asm__ volatile(\ 00737 "pxor %%xmm7, %%xmm7 \n\t"\ 00738 "movdqa "MANGLE(ff_pw_5)", %%xmm6\n\t"\ 00739 "1: \n\t"\ 00740 "lddqu -2(%0), %%xmm1 \n\t"\ 00741 "movdqa %%xmm1, %%xmm0 \n\t"\ 00742 "punpckhbw %%xmm7, %%xmm1 \n\t"\ 00743 "punpcklbw %%xmm7, %%xmm0 \n\t"\ 00744 "movdqa %%xmm1, %%xmm2 \n\t"\ 00745 "movdqa %%xmm1, %%xmm3 \n\t"\ 00746 "movdqa %%xmm1, %%xmm4 \n\t"\ 00747 "movdqa %%xmm1, %%xmm5 \n\t"\ 00748 "palignr $2, %%xmm0, %%xmm4 \n\t"\ 00749 "palignr $4, %%xmm0, %%xmm3 \n\t"\ 00750 "palignr $6, %%xmm0, %%xmm2 \n\t"\ 00751 "palignr $8, %%xmm0, %%xmm1 \n\t"\ 00752 "palignr $10,%%xmm0, %%xmm5 \n\t"\ 00753 "paddw %%xmm5, %%xmm0 \n\t"\ 00754 "paddw %%xmm3, %%xmm2 \n\t"\ 00755 "paddw %%xmm4, %%xmm1 \n\t"\ 00756 "psllw $2, %%xmm2 \n\t"\ 00757 "psubw %%xmm1, %%xmm2 \n\t"\ 00758 "paddw "MANGLE(ff_pw_16)", %%xmm0\n\t"\ 00759 "pmullw %%xmm6, %%xmm2 \n\t"\ 00760 "paddw %%xmm0, %%xmm2 \n\t"\ 00761 "psraw $5, %%xmm2 \n\t"\ 00762 "packuswb %%xmm2, %%xmm2 \n\t"\ 00763 OP(%%xmm2, (%1), %%xmm4, q)\ 00764 "add %3, %0 \n\t"\ 00765 "add %4, %1 \n\t"\ 00766 "decl %2 \n\t"\ 00767 " jnz 1b \n\t"\ 00768 : "+a"(src), "+c"(dst), "+g"(h)\ 00769 : "D"((x86_reg)srcStride), "S"((x86_reg)dstStride)\ 00770 : XMM_CLOBBERS("%xmm0", "%xmm1", "%xmm2", "%xmm3", \ 00771 "%xmm4", "%xmm5", "%xmm6", "%xmm7",)\ 00772 "memory"\ 00773 );\ 00774 }\ 00775 static void OPNAME ## h264_qpel16_h_lowpass_ ## MMX(uint8_t *dst, uint8_t *src, int dstStride, int srcStride){\ 00776 OPNAME ## h264_qpel8_h_lowpass_ ## MMX(dst , src , dstStride, srcStride);\ 00777 OPNAME ## h264_qpel8_h_lowpass_ ## MMX(dst+8, src+8, dstStride, srcStride);\ 00778 src += 8*srcStride;\ 00779 dst += 8*dstStride;\ 00780 OPNAME ## h264_qpel8_h_lowpass_ ## MMX(dst , src , dstStride, srcStride);\ 00781 OPNAME ## h264_qpel8_h_lowpass_ ## MMX(dst+8, src+8, dstStride, srcStride);\ 00782 }\ 00783 00784 #define QPEL_H264_V_XMM(OPNAME, OP, MMX)\ 00785 static av_noinline void OPNAME ## h264_qpel8or16_v_lowpass_ ## MMX(uint8_t *dst, uint8_t *src, int dstStride, int srcStride, int h){\ 00786 src -= 2*srcStride;\ 00787 \ 00788 __asm__ volatile(\ 00789 "pxor %%xmm7, %%xmm7 \n\t"\ 00790 "movq (%0), %%xmm0 \n\t"\ 00791 "add %2, %0 \n\t"\ 00792 "movq (%0), %%xmm1 \n\t"\ 00793 "add %2, %0 \n\t"\ 00794 "movq (%0), %%xmm2 \n\t"\ 00795 "add %2, %0 \n\t"\ 00796 "movq (%0), %%xmm3 \n\t"\ 00797 "add %2, %0 \n\t"\ 00798 "movq (%0), %%xmm4 \n\t"\ 00799 "add %2, %0 \n\t"\ 00800 "punpcklbw %%xmm7, %%xmm0 \n\t"\ 00801 "punpcklbw %%xmm7, %%xmm1 \n\t"\ 00802 "punpcklbw %%xmm7, %%xmm2 \n\t"\ 00803 "punpcklbw %%xmm7, %%xmm3 \n\t"\ 00804 "punpcklbw %%xmm7, %%xmm4 \n\t"\ 00805 QPEL_H264V_XMM(%%xmm0, %%xmm1, %%xmm2, %%xmm3, %%xmm4, %%xmm5, OP)\ 00806 QPEL_H264V_XMM(%%xmm1, %%xmm2, %%xmm3, %%xmm4, %%xmm5, %%xmm0, OP)\ 00807 QPEL_H264V_XMM(%%xmm2, %%xmm3, %%xmm4, %%xmm5, %%xmm0, %%xmm1, OP)\ 00808 QPEL_H264V_XMM(%%xmm3, %%xmm4, %%xmm5, %%xmm0, %%xmm1, %%xmm2, OP)\ 00809 QPEL_H264V_XMM(%%xmm4, %%xmm5, %%xmm0, %%xmm1, %%xmm2, %%xmm3, OP)\ 00810 QPEL_H264V_XMM(%%xmm5, %%xmm0, %%xmm1, %%xmm2, %%xmm3, %%xmm4, OP)\ 00811 QPEL_H264V_XMM(%%xmm0, %%xmm1, %%xmm2, %%xmm3, %%xmm4, %%xmm5, OP)\ 00812 QPEL_H264V_XMM(%%xmm1, %%xmm2, %%xmm3, %%xmm4, %%xmm5, %%xmm0, OP)\ 00813 "cmpl $16, %4 \n\t"\ 00814 "jne 2f \n\t"\ 00815 QPEL_H264V_XMM(%%xmm2, %%xmm3, %%xmm4, %%xmm5, %%xmm0, %%xmm1, OP)\ 00816 QPEL_H264V_XMM(%%xmm3, %%xmm4, %%xmm5, %%xmm0, %%xmm1, %%xmm2, OP)\ 00817 QPEL_H264V_XMM(%%xmm4, %%xmm5, %%xmm0, %%xmm1, %%xmm2, %%xmm3, OP)\ 00818 QPEL_H264V_XMM(%%xmm5, %%xmm0, %%xmm1, %%xmm2, %%xmm3, %%xmm4, OP)\ 00819 QPEL_H264V_XMM(%%xmm0, %%xmm1, %%xmm2, %%xmm3, %%xmm4, %%xmm5, OP)\ 00820 QPEL_H264V_XMM(%%xmm1, %%xmm2, %%xmm3, %%xmm4, %%xmm5, %%xmm0, OP)\ 00821 QPEL_H264V_XMM(%%xmm2, %%xmm3, %%xmm4, %%xmm5, %%xmm0, %%xmm1, OP)\ 00822 QPEL_H264V_XMM(%%xmm3, %%xmm4, %%xmm5, %%xmm0, %%xmm1, %%xmm2, OP)\ 00823 "2: \n\t"\ 00824 \ 00825 : "+a"(src), "+c"(dst)\ 00826 : "S"((x86_reg)srcStride), "D"((x86_reg)dstStride), "rm"(h)\ 00827 : XMM_CLOBBERS("%xmm0", "%xmm1", "%xmm2", "%xmm3", \ 00828 "%xmm4", "%xmm5", "%xmm6", "%xmm7",)\ 00829 "memory"\ 00830 );\ 00831 }\ 00832 static void OPNAME ## h264_qpel8_v_lowpass_ ## MMX(uint8_t *dst, uint8_t *src, int dstStride, int srcStride){\ 00833 OPNAME ## h264_qpel8or16_v_lowpass_ ## MMX(dst , src , dstStride, srcStride, 8);\ 00834 }\ 00835 static av_noinline void OPNAME ## h264_qpel16_v_lowpass_ ## MMX(uint8_t *dst, uint8_t *src, int dstStride, int srcStride){\ 00836 OPNAME ## h264_qpel8or16_v_lowpass_ ## MMX(dst , src , dstStride, srcStride, 16);\ 00837 OPNAME ## h264_qpel8or16_v_lowpass_ ## MMX(dst+8, src+8, dstStride, srcStride, 16);\ 00838 } 00839 00840 static av_always_inline void put_h264_qpel8or16_hv1_lowpass_sse2(int16_t *tmp, uint8_t *src, int tmpStride, int srcStride, int size){ 00841 int w = (size+8)>>3; 00842 src -= 2*srcStride+2; 00843 while(w--){ 00844 __asm__ volatile( 00845 "pxor %%xmm7, %%xmm7 \n\t" 00846 "movq (%0), %%xmm0 \n\t" 00847 "add %2, %0 \n\t" 00848 "movq (%0), %%xmm1 \n\t" 00849 "add %2, %0 \n\t" 00850 "movq (%0), %%xmm2 \n\t" 00851 "add %2, %0 \n\t" 00852 "movq (%0), %%xmm3 \n\t" 00853 "add %2, %0 \n\t" 00854 "movq (%0), %%xmm4 \n\t" 00855 "add %2, %0 \n\t" 00856 "punpcklbw %%xmm7, %%xmm0 \n\t" 00857 "punpcklbw %%xmm7, %%xmm1 \n\t" 00858 "punpcklbw %%xmm7, %%xmm2 \n\t" 00859 "punpcklbw %%xmm7, %%xmm3 \n\t" 00860 "punpcklbw %%xmm7, %%xmm4 \n\t" 00861 QPEL_H264HV_XMM(%%xmm0, %%xmm1, %%xmm2, %%xmm3, %%xmm4, %%xmm5, 0*48) 00862 QPEL_H264HV_XMM(%%xmm1, %%xmm2, %%xmm3, %%xmm4, %%xmm5, %%xmm0, 1*48) 00863 QPEL_H264HV_XMM(%%xmm2, %%xmm3, %%xmm4, %%xmm5, %%xmm0, %%xmm1, 2*48) 00864 QPEL_H264HV_XMM(%%xmm3, %%xmm4, %%xmm5, %%xmm0, %%xmm1, %%xmm2, 3*48) 00865 QPEL_H264HV_XMM(%%xmm4, %%xmm5, %%xmm0, %%xmm1, %%xmm2, %%xmm3, 4*48) 00866 QPEL_H264HV_XMM(%%xmm5, %%xmm0, %%xmm1, %%xmm2, %%xmm3, %%xmm4, 5*48) 00867 QPEL_H264HV_XMM(%%xmm0, %%xmm1, %%xmm2, %%xmm3, %%xmm4, %%xmm5, 6*48) 00868 QPEL_H264HV_XMM(%%xmm1, %%xmm2, %%xmm3, %%xmm4, %%xmm5, %%xmm0, 7*48) 00869 "cmpl $16, %3 \n\t" 00870 "jne 2f \n\t" 00871 QPEL_H264HV_XMM(%%xmm2, %%xmm3, %%xmm4, %%xmm5, %%xmm0, %%xmm1, 8*48) 00872 QPEL_H264HV_XMM(%%xmm3, %%xmm4, %%xmm5, %%xmm0, %%xmm1, %%xmm2, 9*48) 00873 QPEL_H264HV_XMM(%%xmm4, %%xmm5, %%xmm0, %%xmm1, %%xmm2, %%xmm3, 10*48) 00874 QPEL_H264HV_XMM(%%xmm5, %%xmm0, %%xmm1, %%xmm2, %%xmm3, %%xmm4, 11*48) 00875 QPEL_H264HV_XMM(%%xmm0, %%xmm1, %%xmm2, %%xmm3, %%xmm4, %%xmm5, 12*48) 00876 QPEL_H264HV_XMM(%%xmm1, %%xmm2, %%xmm3, %%xmm4, %%xmm5, %%xmm0, 13*48) 00877 QPEL_H264HV_XMM(%%xmm2, %%xmm3, %%xmm4, %%xmm5, %%xmm0, %%xmm1, 14*48) 00878 QPEL_H264HV_XMM(%%xmm3, %%xmm4, %%xmm5, %%xmm0, %%xmm1, %%xmm2, 15*48) 00879 "2: \n\t" 00880 : "+a"(src) 00881 : "c"(tmp), "S"((x86_reg)srcStride), "rm"(size) 00882 : XMM_CLOBBERS("%xmm0", "%xmm1", "%xmm2", "%xmm3", 00883 "%xmm4", "%xmm5", "%xmm6", "%xmm7",) 00884 "memory" 00885 ); 00886 tmp += 8; 00887 src += 8 - (size+5)*srcStride; 00888 } 00889 } 00890 00891 #define QPEL_H264_HV2_XMM(OPNAME, OP, MMX)\ 00892 static av_always_inline void OPNAME ## h264_qpel8or16_hv2_lowpass_ ## MMX(uint8_t *dst, int16_t *tmp, int dstStride, int tmpStride, int size){\ 00893 int h = size;\ 00894 if(size == 16){\ 00895 __asm__ volatile(\ 00896 "1: \n\t"\ 00897 "movdqa 32(%0), %%xmm4 \n\t"\ 00898 "movdqa 16(%0), %%xmm5 \n\t"\ 00899 "movdqa (%0), %%xmm7 \n\t"\ 00900 "movdqa %%xmm4, %%xmm3 \n\t"\ 00901 "movdqa %%xmm4, %%xmm2 \n\t"\ 00902 "movdqa %%xmm4, %%xmm1 \n\t"\ 00903 "movdqa %%xmm4, %%xmm0 \n\t"\ 00904 "palignr $10, %%xmm5, %%xmm0 \n\t"\ 00905 "palignr $8, %%xmm5, %%xmm1 \n\t"\ 00906 "palignr $6, %%xmm5, %%xmm2 \n\t"\ 00907 "palignr $4, %%xmm5, %%xmm3 \n\t"\ 00908 "palignr $2, %%xmm5, %%xmm4 \n\t"\ 00909 "paddw %%xmm5, %%xmm0 \n\t"\ 00910 "paddw %%xmm4, %%xmm1 \n\t"\ 00911 "paddw %%xmm3, %%xmm2 \n\t"\ 00912 "movdqa %%xmm5, %%xmm6 \n\t"\ 00913 "movdqa %%xmm5, %%xmm4 \n\t"\ 00914 "movdqa %%xmm5, %%xmm3 \n\t"\ 00915 "palignr $8, %%xmm7, %%xmm4 \n\t"\ 00916 "palignr $2, %%xmm7, %%xmm6 \n\t"\ 00917 "palignr $10, %%xmm7, %%xmm3 \n\t"\ 00918 "paddw %%xmm6, %%xmm4 \n\t"\ 00919 "movdqa %%xmm5, %%xmm6 \n\t"\ 00920 "palignr $6, %%xmm7, %%xmm5 \n\t"\ 00921 "palignr $4, %%xmm7, %%xmm6 \n\t"\ 00922 "paddw %%xmm7, %%xmm3 \n\t"\ 00923 "paddw %%xmm6, %%xmm5 \n\t"\ 00924 \ 00925 "psubw %%xmm1, %%xmm0 \n\t"\ 00926 "psubw %%xmm4, %%xmm3 \n\t"\ 00927 "psraw $2, %%xmm0 \n\t"\ 00928 "psraw $2, %%xmm3 \n\t"\ 00929 "psubw %%xmm1, %%xmm0 \n\t"\ 00930 "psubw %%xmm4, %%xmm3 \n\t"\ 00931 "paddw %%xmm2, %%xmm0 \n\t"\ 00932 "paddw %%xmm5, %%xmm3 \n\t"\ 00933 "psraw $2, %%xmm0 \n\t"\ 00934 "psraw $2, %%xmm3 \n\t"\ 00935 "paddw %%xmm2, %%xmm0 \n\t"\ 00936 "paddw %%xmm5, %%xmm3 \n\t"\ 00937 "psraw $6, %%xmm0 \n\t"\ 00938 "psraw $6, %%xmm3 \n\t"\ 00939 "packuswb %%xmm0, %%xmm3 \n\t"\ 00940 OP(%%xmm3, (%1), %%xmm7, dqa)\ 00941 "add $48, %0 \n\t"\ 00942 "add %3, %1 \n\t"\ 00943 "decl %2 \n\t"\ 00944 " jnz 1b \n\t"\ 00945 : "+a"(tmp), "+c"(dst), "+g"(h)\ 00946 : "S"((x86_reg)dstStride)\ 00947 : XMM_CLOBBERS("%xmm0", "%xmm1", "%xmm2", "%xmm3", \ 00948 "%xmm4", "%xmm5", "%xmm6", "%xmm7",)\ 00949 "memory"\ 00950 );\ 00951 }else{\ 00952 __asm__ volatile(\ 00953 "1: \n\t"\ 00954 "movdqa 16(%0), %%xmm1 \n\t"\ 00955 "movdqa (%0), %%xmm0 \n\t"\ 00956 "movdqa %%xmm1, %%xmm2 \n\t"\ 00957 "movdqa %%xmm1, %%xmm3 \n\t"\ 00958 "movdqa %%xmm1, %%xmm4 \n\t"\ 00959 "movdqa %%xmm1, %%xmm5 \n\t"\ 00960 "palignr $10, %%xmm0, %%xmm5 \n\t"\ 00961 "palignr $8, %%xmm0, %%xmm4 \n\t"\ 00962 "palignr $6, %%xmm0, %%xmm3 \n\t"\ 00963 "palignr $4, %%xmm0, %%xmm2 \n\t"\ 00964 "palignr $2, %%xmm0, %%xmm1 \n\t"\ 00965 "paddw %%xmm5, %%xmm0 \n\t"\ 00966 "paddw %%xmm4, %%xmm1 \n\t"\ 00967 "paddw %%xmm3, %%xmm2 \n\t"\ 00968 "psubw %%xmm1, %%xmm0 \n\t"\ 00969 "psraw $2, %%xmm0 \n\t"\ 00970 "psubw %%xmm1, %%xmm0 \n\t"\ 00971 "paddw %%xmm2, %%xmm0 \n\t"\ 00972 "psraw $2, %%xmm0 \n\t"\ 00973 "paddw %%xmm2, %%xmm0 \n\t"\ 00974 "psraw $6, %%xmm0 \n\t"\ 00975 "packuswb %%xmm0, %%xmm0 \n\t"\ 00976 OP(%%xmm0, (%1), %%xmm7, q)\ 00977 "add $48, %0 \n\t"\ 00978 "add %3, %1 \n\t"\ 00979 "decl %2 \n\t"\ 00980 " jnz 1b \n\t"\ 00981 : "+a"(tmp), "+c"(dst), "+g"(h)\ 00982 : "S"((x86_reg)dstStride)\ 00983 : XMM_CLOBBERS("%xmm0", "%xmm1", "%xmm2", "%xmm3", \ 00984 "%xmm4", "%xmm5", "%xmm6", "%xmm7",)\ 00985 "memory"\ 00986 );\ 00987 }\ 00988 } 00989 00990 #define QPEL_H264_HV_XMM(OPNAME, OP, MMX)\ 00991 static av_noinline void OPNAME ## h264_qpel8or16_hv_lowpass_ ## MMX(uint8_t *dst, int16_t *tmp, uint8_t *src, int dstStride, int tmpStride, int srcStride, int size){\ 00992 put_h264_qpel8or16_hv1_lowpass_sse2(tmp, src, tmpStride, srcStride, size);\ 00993 OPNAME ## h264_qpel8or16_hv2_lowpass_ ## MMX(dst, tmp, dstStride, tmpStride, size);\ 00994 }\ 00995 static void OPNAME ## h264_qpel8_hv_lowpass_ ## MMX(uint8_t *dst, int16_t *tmp, uint8_t *src, int dstStride, int tmpStride, int srcStride){\ 00996 OPNAME ## h264_qpel8or16_hv_lowpass_ ## MMX(dst, tmp, src, dstStride, tmpStride, srcStride, 8);\ 00997 }\ 00998 static void OPNAME ## h264_qpel16_hv_lowpass_ ## MMX(uint8_t *dst, int16_t *tmp, uint8_t *src, int dstStride, int tmpStride, int srcStride){\ 00999 OPNAME ## h264_qpel8or16_hv_lowpass_ ## MMX(dst, tmp, src, dstStride, tmpStride, srcStride, 16);\ 01000 }\ 01001 01002 #define put_pixels8_l2_sse2 put_pixels8_l2_mmx2 01003 #define avg_pixels8_l2_sse2 avg_pixels8_l2_mmx2 01004 #define put_pixels16_l2_sse2 put_pixels16_l2_mmx2 01005 #define avg_pixels16_l2_sse2 avg_pixels16_l2_mmx2 01006 #define put_pixels8_l2_ssse3 put_pixels8_l2_mmx2 01007 #define avg_pixels8_l2_ssse3 avg_pixels8_l2_mmx2 01008 #define put_pixels16_l2_ssse3 put_pixels16_l2_mmx2 01009 #define avg_pixels16_l2_ssse3 avg_pixels16_l2_mmx2 01010 01011 #define put_pixels8_l2_shift5_sse2 put_pixels8_l2_shift5_mmx2 01012 #define avg_pixels8_l2_shift5_sse2 avg_pixels8_l2_shift5_mmx2 01013 #define put_pixels16_l2_shift5_sse2 put_pixels16_l2_shift5_mmx2 01014 #define avg_pixels16_l2_shift5_sse2 avg_pixels16_l2_shift5_mmx2 01015 #define put_pixels8_l2_shift5_ssse3 put_pixels8_l2_shift5_mmx2 01016 #define avg_pixels8_l2_shift5_ssse3 avg_pixels8_l2_shift5_mmx2 01017 #define put_pixels16_l2_shift5_ssse3 put_pixels16_l2_shift5_mmx2 01018 #define avg_pixels16_l2_shift5_ssse3 avg_pixels16_l2_shift5_mmx2 01019 01020 #define put_h264_qpel8_h_lowpass_l2_sse2 put_h264_qpel8_h_lowpass_l2_mmx2 01021 #define avg_h264_qpel8_h_lowpass_l2_sse2 avg_h264_qpel8_h_lowpass_l2_mmx2 01022 #define put_h264_qpel16_h_lowpass_l2_sse2 put_h264_qpel16_h_lowpass_l2_mmx2 01023 #define avg_h264_qpel16_h_lowpass_l2_sse2 avg_h264_qpel16_h_lowpass_l2_mmx2 01024 01025 #define put_h264_qpel8_v_lowpass_ssse3 put_h264_qpel8_v_lowpass_sse2 01026 #define avg_h264_qpel8_v_lowpass_ssse3 avg_h264_qpel8_v_lowpass_sse2 01027 #define put_h264_qpel16_v_lowpass_ssse3 put_h264_qpel16_v_lowpass_sse2 01028 #define avg_h264_qpel16_v_lowpass_ssse3 avg_h264_qpel16_v_lowpass_sse2 01029 01030 #define put_h264_qpel8or16_hv2_lowpass_sse2 put_h264_qpel8or16_hv2_lowpass_mmx2 01031 #define avg_h264_qpel8or16_hv2_lowpass_sse2 avg_h264_qpel8or16_hv2_lowpass_mmx2 01032 01033 #define H264_MC(OPNAME, SIZE, MMX, ALIGN) \ 01034 H264_MC_C(OPNAME, SIZE, MMX, ALIGN)\ 01035 H264_MC_V(OPNAME, SIZE, MMX, ALIGN)\ 01036 H264_MC_H(OPNAME, SIZE, MMX, ALIGN)\ 01037 H264_MC_HV(OPNAME, SIZE, MMX, ALIGN)\ 01038 01039 static void put_h264_qpel16_mc00_sse2 (uint8_t *dst, uint8_t *src, int stride){ 01040 put_pixels16_sse2(dst, src, stride, 16); 01041 } 01042 static void avg_h264_qpel16_mc00_sse2 (uint8_t *dst, uint8_t *src, int stride){ 01043 avg_pixels16_sse2(dst, src, stride, 16); 01044 } 01045 #define put_h264_qpel8_mc00_sse2 put_h264_qpel8_mc00_mmx2 01046 #define avg_h264_qpel8_mc00_sse2 avg_h264_qpel8_mc00_mmx2 01047 01048 #define H264_MC_C(OPNAME, SIZE, MMX, ALIGN) \ 01049 static void OPNAME ## h264_qpel ## SIZE ## _mc00_ ## MMX (uint8_t *dst, uint8_t *src, int stride){\ 01050 OPNAME ## pixels ## SIZE ## _ ## MMX(dst, src, stride, SIZE);\ 01051 }\ 01052 01053 #define H264_MC_H(OPNAME, SIZE, MMX, ALIGN) \ 01054 static void OPNAME ## h264_qpel ## SIZE ## _mc10_ ## MMX(uint8_t *dst, uint8_t *src, int stride){\ 01055 OPNAME ## h264_qpel ## SIZE ## _h_lowpass_l2_ ## MMX(dst, src, src, stride, stride);\ 01056 }\ 01057 \ 01058 static void OPNAME ## h264_qpel ## SIZE ## _mc20_ ## MMX(uint8_t *dst, uint8_t *src, int stride){\ 01059 OPNAME ## h264_qpel ## SIZE ## _h_lowpass_ ## MMX(dst, src, stride, stride);\ 01060 }\ 01061 \ 01062 static void OPNAME ## h264_qpel ## SIZE ## _mc30_ ## MMX(uint8_t *dst, uint8_t *src, int stride){\ 01063 OPNAME ## h264_qpel ## SIZE ## _h_lowpass_l2_ ## MMX(dst, src, src+1, stride, stride);\ 01064 }\ 01065 01066 #define H264_MC_V(OPNAME, SIZE, MMX, ALIGN) \ 01067 static void OPNAME ## h264_qpel ## SIZE ## _mc01_ ## MMX(uint8_t *dst, uint8_t *src, int stride){\ 01068 DECLARE_ALIGNED(ALIGN, uint8_t, temp)[SIZE*SIZE];\ 01069 put_h264_qpel ## SIZE ## _v_lowpass_ ## MMX(temp, src, SIZE, stride);\ 01070 OPNAME ## pixels ## SIZE ## _l2_ ## MMX(dst, src, temp, stride, stride, SIZE);\ 01071 }\ 01072 \ 01073 static void OPNAME ## h264_qpel ## SIZE ## _mc02_ ## MMX(uint8_t *dst, uint8_t *src, int stride){\ 01074 OPNAME ## h264_qpel ## SIZE ## _v_lowpass_ ## MMX(dst, src, stride, stride);\ 01075 }\ 01076 \ 01077 static void OPNAME ## h264_qpel ## SIZE ## _mc03_ ## MMX(uint8_t *dst, uint8_t *src, int stride){\ 01078 DECLARE_ALIGNED(ALIGN, uint8_t, temp)[SIZE*SIZE];\ 01079 put_h264_qpel ## SIZE ## _v_lowpass_ ## MMX(temp, src, SIZE, stride);\ 01080 OPNAME ## pixels ## SIZE ## _l2_ ## MMX(dst, src+stride, temp, stride, stride, SIZE);\ 01081 }\ 01082 01083 #define H264_MC_HV(OPNAME, SIZE, MMX, ALIGN) \ 01084 static void OPNAME ## h264_qpel ## SIZE ## _mc11_ ## MMX(uint8_t *dst, uint8_t *src, int stride){\ 01085 DECLARE_ALIGNED(ALIGN, uint8_t, temp)[SIZE*SIZE];\ 01086 put_h264_qpel ## SIZE ## _v_lowpass_ ## MMX(temp, src, SIZE, stride);\ 01087 OPNAME ## h264_qpel ## SIZE ## _h_lowpass_l2_ ## MMX(dst, src, temp, stride, SIZE);\ 01088 }\ 01089 \ 01090 static void OPNAME ## h264_qpel ## SIZE ## _mc31_ ## MMX(uint8_t *dst, uint8_t *src, int stride){\ 01091 DECLARE_ALIGNED(ALIGN, uint8_t, temp)[SIZE*SIZE];\ 01092 put_h264_qpel ## SIZE ## _v_lowpass_ ## MMX(temp, src+1, SIZE, stride);\ 01093 OPNAME ## h264_qpel ## SIZE ## _h_lowpass_l2_ ## MMX(dst, src, temp, stride, SIZE);\ 01094 }\ 01095 \ 01096 static void OPNAME ## h264_qpel ## SIZE ## _mc13_ ## MMX(uint8_t *dst, uint8_t *src, int stride){\ 01097 DECLARE_ALIGNED(ALIGN, uint8_t, temp)[SIZE*SIZE];\ 01098 put_h264_qpel ## SIZE ## _v_lowpass_ ## MMX(temp, src, SIZE, stride);\ 01099 OPNAME ## h264_qpel ## SIZE ## _h_lowpass_l2_ ## MMX(dst, src+stride, temp, stride, SIZE);\ 01100 }\ 01101 \ 01102 static void OPNAME ## h264_qpel ## SIZE ## _mc33_ ## MMX(uint8_t *dst, uint8_t *src, int stride){\ 01103 DECLARE_ALIGNED(ALIGN, uint8_t, temp)[SIZE*SIZE];\ 01104 put_h264_qpel ## SIZE ## _v_lowpass_ ## MMX(temp, src+1, SIZE, stride);\ 01105 OPNAME ## h264_qpel ## SIZE ## _h_lowpass_l2_ ## MMX(dst, src+stride, temp, stride, SIZE);\ 01106 }\ 01107 \ 01108 static void OPNAME ## h264_qpel ## SIZE ## _mc22_ ## MMX(uint8_t *dst, uint8_t *src, int stride){\ 01109 DECLARE_ALIGNED(ALIGN, uint16_t, temp)[SIZE*(SIZE<8?12:24)];\ 01110 OPNAME ## h264_qpel ## SIZE ## _hv_lowpass_ ## MMX(dst, temp, src, stride, SIZE, stride);\ 01111 }\ 01112 \ 01113 static void OPNAME ## h264_qpel ## SIZE ## _mc21_ ## MMX(uint8_t *dst, uint8_t *src, int stride){\ 01114 DECLARE_ALIGNED(ALIGN, uint8_t, temp)[SIZE*(SIZE<8?12:24)*2 + SIZE*SIZE];\ 01115 uint8_t * const halfHV= temp;\ 01116 int16_t * const halfV= (int16_t*)(temp + SIZE*SIZE);\ 01117 assert(((int)temp & 7) == 0);\ 01118 put_h264_qpel ## SIZE ## _hv_lowpass_ ## MMX(halfHV, halfV, src, SIZE, SIZE, stride);\ 01119 OPNAME ## h264_qpel ## SIZE ## _h_lowpass_l2_ ## MMX(dst, src, halfHV, stride, SIZE);\ 01120 }\ 01121 \ 01122 static void OPNAME ## h264_qpel ## SIZE ## _mc23_ ## MMX(uint8_t *dst, uint8_t *src, int stride){\ 01123 DECLARE_ALIGNED(ALIGN, uint8_t, temp)[SIZE*(SIZE<8?12:24)*2 + SIZE*SIZE];\ 01124 uint8_t * const halfHV= temp;\ 01125 int16_t * const halfV= (int16_t*)(temp + SIZE*SIZE);\ 01126 assert(((int)temp & 7) == 0);\ 01127 put_h264_qpel ## SIZE ## _hv_lowpass_ ## MMX(halfHV, halfV, src, SIZE, SIZE, stride);\ 01128 OPNAME ## h264_qpel ## SIZE ## _h_lowpass_l2_ ## MMX(dst, src+stride, halfHV, stride, SIZE);\ 01129 }\ 01130 \ 01131 static void OPNAME ## h264_qpel ## SIZE ## _mc12_ ## MMX(uint8_t *dst, uint8_t *src, int stride){\ 01132 DECLARE_ALIGNED(ALIGN, uint8_t, temp)[SIZE*(SIZE<8?12:24)*2 + SIZE*SIZE];\ 01133 uint8_t * const halfHV= temp;\ 01134 int16_t * const halfV= (int16_t*)(temp + SIZE*SIZE);\ 01135 assert(((int)temp & 7) == 0);\ 01136 put_h264_qpel ## SIZE ## _hv_lowpass_ ## MMX(halfHV, halfV, src, SIZE, SIZE, stride);\ 01137 OPNAME ## pixels ## SIZE ## _l2_shift5_ ## MMX(dst, halfV+2, halfHV, stride, SIZE, SIZE);\ 01138 }\ 01139 \ 01140 static void OPNAME ## h264_qpel ## SIZE ## _mc32_ ## MMX(uint8_t *dst, uint8_t *src, int stride){\ 01141 DECLARE_ALIGNED(ALIGN, uint8_t, temp)[SIZE*(SIZE<8?12:24)*2 + SIZE*SIZE];\ 01142 uint8_t * const halfHV= temp;\ 01143 int16_t * const halfV= (int16_t*)(temp + SIZE*SIZE);\ 01144 assert(((int)temp & 7) == 0);\ 01145 put_h264_qpel ## SIZE ## _hv_lowpass_ ## MMX(halfHV, halfV, src, SIZE, SIZE, stride);\ 01146 OPNAME ## pixels ## SIZE ## _l2_shift5_ ## MMX(dst, halfV+3, halfHV, stride, SIZE, SIZE);\ 01147 }\ 01148 01149 #define H264_MC_4816(MMX)\ 01150 H264_MC(put_, 4, MMX, 8)\ 01151 H264_MC(put_, 8, MMX, 8)\ 01152 H264_MC(put_, 16,MMX, 8)\ 01153 H264_MC(avg_, 4, MMX, 8)\ 01154 H264_MC(avg_, 8, MMX, 8)\ 01155 H264_MC(avg_, 16,MMX, 8)\ 01156 01157 #define H264_MC_816(QPEL, XMM)\ 01158 QPEL(put_, 8, XMM, 16)\ 01159 QPEL(put_, 16,XMM, 16)\ 01160 QPEL(avg_, 8, XMM, 16)\ 01161 QPEL(avg_, 16,XMM, 16)\ 01162 01163 01164 #define AVG_3DNOW_OP(a,b,temp, size) \ 01165 "mov" #size " " #b ", " #temp " \n\t"\ 01166 "pavgusb " #temp ", " #a " \n\t"\ 01167 "mov" #size " " #a ", " #b " \n\t" 01168 #define AVG_MMX2_OP(a,b,temp, size) \ 01169 "mov" #size " " #b ", " #temp " \n\t"\ 01170 "pavgb " #temp ", " #a " \n\t"\ 01171 "mov" #size " " #a ", " #b " \n\t" 01172 01173 #define PAVGB "pavgusb" 01174 QPEL_H264(put_, PUT_OP, 3dnow) 01175 QPEL_H264(avg_, AVG_3DNOW_OP, 3dnow) 01176 #undef PAVGB 01177 #define PAVGB "pavgb" 01178 QPEL_H264(put_, PUT_OP, mmx2) 01179 QPEL_H264(avg_, AVG_MMX2_OP, mmx2) 01180 QPEL_H264_V_XMM(put_, PUT_OP, sse2) 01181 QPEL_H264_V_XMM(avg_, AVG_MMX2_OP, sse2) 01182 QPEL_H264_HV_XMM(put_, PUT_OP, sse2) 01183 QPEL_H264_HV_XMM(avg_, AVG_MMX2_OP, sse2) 01184 #if HAVE_SSSE3 01185 QPEL_H264_H_XMM(put_, PUT_OP, ssse3) 01186 QPEL_H264_H_XMM(avg_, AVG_MMX2_OP, ssse3) 01187 QPEL_H264_HV2_XMM(put_, PUT_OP, ssse3) 01188 QPEL_H264_HV2_XMM(avg_, AVG_MMX2_OP, ssse3) 01189 QPEL_H264_HV_XMM(put_, PUT_OP, ssse3) 01190 QPEL_H264_HV_XMM(avg_, AVG_MMX2_OP, ssse3) 01191 #endif 01192 #undef PAVGB 01193 01194 H264_MC_4816(3dnow) 01195 H264_MC_4816(mmx2) 01196 H264_MC_816(H264_MC_V, sse2) 01197 H264_MC_816(H264_MC_HV, sse2) 01198 #if HAVE_SSSE3 01199 H264_MC_816(H264_MC_H, ssse3) 01200 H264_MC_816(H264_MC_HV, ssse3) 01201 #endif