/* * FFT transform with Altivec optimizations * Copyright (c) 2009 Loren Merritt * * This algorithm (though not any of the implementation details) is * based on libdjbfft by D. J. Bernstein. * * This file is part of FFmpeg. * * FFmpeg is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * FFmpeg is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with FFmpeg; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA */ /* * These functions are not individually interchangeable with the C versions. * While C takes arrays of FFTComplex, Altivec leaves intermediate results * in blocks as convenient to the vector size. * i.e. {4x real, 4x imaginary, 4x real, ...} * * I ignore standard calling convention. * Instead, the following registers are treated as global constants: * v14: zero * v15..v18: cosines * v19..v29: permutations * r9: 16 * r12: ff_cos_tabs * and the rest are free for local use. */ #include "config.h" #include "asm.S" .text .macro addi2 ra, imm // add 32-bit immediate .if \imm & 0xffff addi \ra, \ra, \imm@l .endif .if (\imm+0x8000)>>16 addis \ra, \ra, \imm@ha .endif .endm .macro FFT4 a0, a1, a2, a3 // in:0-1 out:2-3 vperm \a2,\a0,\a1,v20 // vcprm(0,1,s2,s1) // {r0,i0,r3,i2} vperm \a3,\a0,\a1,v21 // vcprm(2,3,s0,s3) // {r1,i1,r2,i3} vaddfp \a0,\a2,\a3 // {t1,t2,t6,t5} vsubfp \a1,\a2,\a3 // {t3,t4,t8,t7} vmrghw \a2,\a0,\a1 // vcprm(0,s0,1,s1) // {t1,t3,t2,t4} vperm \a3,\a0,\a1,v22 // vcprm(2,s3,3,s2) // {t6,t7,t5,t8} vaddfp \a0,\a2,\a3 // {r0,r1,i0,i1} vsubfp \a1,\a2,\a3 // {r2,r3,i2,i3} vperm \a2,\a0,\a1,v23 // vcprm(0,1,s0,s1) // {r0,r1,r2,r3} vperm \a3,\a0,\a1,v24 // vcprm(2,3,s2,s3) // {i0,i1,i2,i3} .endm .macro FFT4x2 a0, a1, b0, b1, a2, a3, b2, b3 vperm \a2,\a0,\a1,v20 // vcprm(0,1,s2,s1) // {r0,i0,r3,i2} vperm \a3,\a0,\a1,v21 // vcprm(2,3,s0,s3) // {r1,i1,r2,i3} vperm \b2,\b0,\b1,v20 vperm \b3,\b0,\b1,v21 vaddfp \a0,\a2,\a3 // {t1,t2,t6,t5} vsubfp \a1,\a2,\a3 // {t3,t4,t8,t7} vaddfp \b0,\b2,\b3 vsubfp \b1,\b2,\b3 vmrghw \a2,\a0,\a1 // vcprm(0,s0,1,s1) // {t1,t3,t2,t4} vperm \a3,\a0,\a1,v22 // vcprm(2,s3,3,s2) // {t6,t7,t5,t8} vmrghw \b2,\b0,\b1 vperm \b3,\b0,\b1,v22 vaddfp \a0,\a2,\a3 // {r0,r1,i0,i1} vsubfp \a1,\a2,\a3 // {r2,r3,i2,i3} vaddfp \b0,\b2,\b3 vsubfp \b1,\b2,\b3 vperm \a2,\a0,\a1,v23 // vcprm(0,1,s0,s1) // {r0,r1,r2,r3} vperm \a3,\a0,\a1,v24 // vcprm(2,3,s2,s3) // {i0,i1,i2,i3} vperm \b2,\b0,\b1,v23 vperm \b3,\b0,\b1,v24 .endm .macro FFT8 a0, a1, b0, b1, a2, a3, b2, b3, b4 // in,out:a0-b1 vmrghw \b2,\b0,\b1 // vcprm(0,s0,1,s1) // {r4,r6,i4,i6} vmrglw \b3,\b0,\b1 // vcprm(2,s2,3,s3) // {r5,r7,i5,i7} vperm \a2,\a0,\a1,v20 // FFT4 ... vperm \a3,\a0,\a1,v21 vaddfp \b0,\b2,\b3 // {t1,t3,t2,t4} vsubfp \b1,\b2,\b3 // {r5,r7,i5,i7} vperm \b4,\b1,\b1,v25 // vcprm(2,3,0,1) // {i5,i7,r5,r7} vaddfp \a0,\a2,\a3 vsubfp \a1,\a2,\a3 vmaddfp \b1,\b1,v17,v14 // * {-1,1,1,-1}/sqrt(2) vmaddfp \b1,\b4,v18,\b1 // * { 1,1,1,1 }/sqrt(2) // {t8,ta,t7,t9} vmrghw \a2,\a0,\a1 vperm \a3,\a0,\a1,v22 vperm \b2,\b0,\b1,v26 // vcprm(1,2,s3,s0) // {t3,t2,t9,t8} vperm \b3,\b0,\b1,v27 // vcprm(0,3,s2,s1) // {t1,t4,t7,ta} vaddfp \a0,\a2,\a3 vsubfp \a1,\a2,\a3 vaddfp \b0,\b2,\b3 // {t1,t2,t9,ta} vsubfp \b1,\b2,\b3 // {t6,t5,tc,tb} vperm \a2,\a0,\a1,v23 vperm \a3,\a0,\a1,v24 vperm \b2,\b0,\b1,v28 // vcprm(0,2,s1,s3) // {t1,t9,t5,tb} vperm \b3,\b0,\b1,v29 // vcprm(1,3,s0,s2) // {t2,ta,t6,tc} vsubfp \b0,\a2,\b2 // {r4,r5,r6,r7} vsubfp \b1,\a3,\b3 // {i4,i5,i6,i7} vaddfp \a0,\a2,\b2 // {r0,r1,r2,r3} vaddfp \a1,\a3,\b3 // {i0,i1,i2,i3} .endm .macro BF d0,d1,s0,s1 vsubfp \d1,\s0,\s1 vaddfp \d0,\s0,\s1 .endm .macro zip d0,d1,s0,s1 vmrghw \d0,\s0,\s1 vmrglw \d1,\s0,\s1 .endm .macro def_fft4 interleave fft4\interleave\()_altivec: lvx v0, 0,r3 lvx v1,r9,r3 FFT4 v0,v1,v2,v3 .ifnb \interleave zip v0,v1,v2,v3 stvx v0, 0,r3 stvx v1,r9,r3 .else stvx v2, 0,r3 stvx v3,r9,r3 .endif blr .endm .macro def_fft8 interleave fft8\interleave\()_altivec: addi r4,r3,32 lvx v0, 0,r3 lvx v1,r9,r3 lvx v2, 0,r4 lvx v3,r9,r4 FFT8 v0,v1,v2,v3,v4,v5,v6,v7,v8 .ifnb \interleave zip v4,v5,v0,v1 zip v6,v7,v2,v3 stvx v4, 0,r3 stvx v5,r9,r3 stvx v6, 0,r4 stvx v7,r9,r4 .else stvx v0, 0,r3 stvx v1,r9,r3 stvx v2, 0,r4 stvx v3,r9,r4 .endif blr .endm .macro def_fft16 interleave fft16\interleave\()_altivec: addi r5,r3,64 addi r6,r3,96 addi r4,r3,32 lvx v0, 0,r5 lvx v1,r9,r5 lvx v2, 0,r6 lvx v3,r9,r6 FFT4x2 v0,v1,v2,v3,v4,v5,v6,v7 lvx v0, 0,r3 lvx v1,r9,r3 lvx v2, 0,r4 lvx v3,r9,r4 FFT8 v0,v1,v2,v3,v8,v9,v10,v11,v12 vmaddfp v8,v4,v15,v14 // r2*wre vmaddfp v9,v5,v15,v14 // i2*wre vmaddfp v10,v6,v15,v14 // r3*wre vmaddfp v11,v7,v15,v14 // i3*wre vmaddfp v8,v5,v16,v8 // i2*wim vnmsubfp v9,v4,v16,v9 // r2*wim vnmsubfp v10,v7,v16,v10 // i3*wim vmaddfp v11,v6,v16,v11 // r3*wim BF v10,v12,v10,v8 BF v11,v13,v9,v11 BF v0,v4,v0,v10 BF v3,v7,v3,v12 BF v1,v5,v1,v11 BF v2,v6,v2,v13 .ifnb \interleave zip v8, v9,v0,v1 zip v10,v11,v2,v3 zip v12,v13,v4,v5 zip v14,v15,v6,v7 stvx v8, 0,r3 stvx v9,r9,r3 stvx v10, 0,r4 stvx v11,r9,r4 stvx v12, 0,r5 stvx v13,r9,r5 stvx v14, 0,r6 stvx v15,r9,r6 .else stvx v0, 0,r3 stvx v4, 0,r5 stvx v3,r9,r4 stvx v7,r9,r6 stvx v1,r9,r3 stvx v5,r9,r5 stvx v2, 0,r4 stvx v6, 0,r6 .endif blr .endm // void pass(float *z, float *wre, int n) .macro PASS interleave, suffix fft_pass\suffix\()_altivec: mtctr r5 slwi r0,r5,4 slwi r7,r5,6 // o2 slwi r5,r5,5 // o1 add r10,r5,r7 // o3 add r0,r4,r0 // wim addi r6,r5,16 // o1+16 addi r8,r7,16 // o2+16 addi r11,r10,16 // o3+16 1: lvx v8, 0,r4 // wre lvx v10, 0,r0 // wim sub r0,r0,r9 lvx v9, 0,r0 vperm v9,v9,v10,v19 // vcprm(s0,3,2,1) => wim[0 .. -3] lvx v4,r3,r7 // r2 = z[o2] lvx v5,r3,r8 // i2 = z[o2+16] lvx v6,r3,r10 // r3 = z[o3] lvx v7,r3,r11 // i3 = z[o3+16] vmaddfp v10,v4,v8,v14 // r2*wre vmaddfp v11,v5,v8,v14 // i2*wre vmaddfp v12,v6,v8,v14 // r3*wre vmaddfp v13,v7,v8,v14 // i3*wre lvx v0, 0,r3 // r0 = z[0] lvx v3,r3,r6 // i1 = z[o1+16] vmaddfp v10,v5,v9,v10 // i2*wim vnmsubfp v11,v4,v9,v11 // r2*wim vnmsubfp v12,v7,v9,v12 // i3*wim vmaddfp v13,v6,v9,v13 // r3*wim lvx v1,r3,r9 // i0 = z[16] lvx v2,r3,r5 // r1 = z[o1] BF v12,v8,v12,v10 BF v13,v9,v11,v13 BF v0,v4,v0,v12 BF v3,v7,v3,v8 .if !\interleave stvx v0, 0,r3 stvx v4,r3,r7 stvx v3,r3,r6 stvx v7,r3,r11 .endif BF v1,v5,v1,v13 BF v2,v6,v2,v9 .if !\interleave stvx v1,r3,r9 stvx v2,r3,r5 stvx v5,r3,r8 stvx v6,r3,r10 .else vmrghw v8,v0,v1 vmrglw v9,v0,v1 stvx v8, 0,r3 stvx v9,r3,r9 vmrghw v8,v2,v3 vmrglw v9,v2,v3 stvx v8,r3,r5 stvx v9,r3,r6 vmrghw v8,v4,v5 vmrglw v9,v4,v5 stvx v8,r3,r7 stvx v9,r3,r8 vmrghw v8,v6,v7 vmrglw v9,v6,v7 stvx v8,r3,r10 stvx v9,r3,r11 .endif addi r3,r3,32 addi r4,r4,16 bdnz 1b sub r3,r3,r5 blr .endm #define M_SQRT1_2 0.70710678118654752440 /* 1/sqrt(2) */ #define WORD_0 0x00,0x01,0x02,0x03 #define WORD_1 0x04,0x05,0x06,0x07 #define WORD_2 0x08,0x09,0x0a,0x0b #define WORD_3 0x0c,0x0d,0x0e,0x0f #define WORD_s0 0x10,0x11,0x12,0x13 #define WORD_s1 0x14,0x15,0x16,0x17 #define WORD_s2 0x18,0x19,0x1a,0x1b #define WORD_s3 0x1c,0x1d,0x1e,0x1f #define vcprm(a, b, c, d) .byte WORD_##a, WORD_##b, WORD_##c, WORD_##d .rodata .align 4 fft_data: .float 0, 0, 0, 0 .float 1, 0.92387953, M_SQRT1_2, 0.38268343 .float 0, 0.38268343, M_SQRT1_2, 0.92387953 .float -M_SQRT1_2, M_SQRT1_2, M_SQRT1_2,-M_SQRT1_2 .float M_SQRT1_2, M_SQRT1_2, M_SQRT1_2, M_SQRT1_2 vcprm(s0,3,2,1) vcprm(0,1,s2,s1) vcprm(2,3,s0,s3) vcprm(2,s3,3,s2) vcprm(0,1,s0,s1) vcprm(2,3,s2,s3) vcprm(2,3,0,1) vcprm(1,2,s3,s0) vcprm(0,3,s2,s1) vcprm(0,2,s1,s3) vcprm(1,3,s0,s2) .macro lvm b, r, regs:vararg lvx \r, 0, \b addi \b, \b, 16 .ifnb \regs lvm \b, \regs .endif .endm .macro stvm b, r, regs:vararg stvx \r, 0, \b addi \b, \b, 16 .ifnb \regs stvm \b, \regs .endif .endm .macro fft_calc interleave extfunc ff_fft_calc\interleave\()_altivec mflr r0 stp r0, 2*PS(r1) stpu r1, -(160+16*PS)(r1) addi r6, r1, 16*PS stvm r6, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29 mfvrsave r0 stw r0, 15*PS(r1) li r6, 0xfffffffc mtvrsave r6 movrel r6, fft_data lvm r6, v14, v15, v16, v17, v18, v19, v20, v21 lvm r6, v22, v23, v24, v25, v26, v27, v28, v29 li r9, 16 movrel r12, X(ff_cos_tabs) movrel r6, fft_dispatch_tab\interleave\()_altivec lwz r3, 0(r3) subi r3, r3, 2 slwi r3, r3, 2+ARCH_PPC64 lpx r3, r3, r6 mtctr r3 mr r3, r4 bctrl addi r6, r1, 16*PS lvm r6, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29 lwz r6, 15*PS(r1) mtvrsave r6 lp r1, 0(r1) lp r0, 2*PS(r1) mtlr r0 blr .endm .macro DECL_FFT suffix, bits, n, n2, n4 fft\n\suffix\()_altivec: mflr r0 stp r0,PS*(\bits-3)(r1) bl fft\n2\()_altivec addi2 r3,\n*4 bl fft\n4\()_altivec addi2 r3,\n*2 bl fft\n4\()_altivec addi2 r3,\n*-6 lp r0,PS*(\bits-3)(r1) lp r4,\bits*PS(r12) mtlr r0 li r5,\n/16 b fft_pass\suffix\()_altivec .endm .macro DECL_FFTS interleave, suffix .text def_fft4 \suffix def_fft8 \suffix def_fft16 \suffix PASS \interleave, \suffix DECL_FFT \suffix, 5, 32, 16, 8 DECL_FFT \suffix, 6, 64, 32, 16 DECL_FFT \suffix, 7, 128, 64, 32 DECL_FFT \suffix, 8, 256, 128, 64 DECL_FFT \suffix, 9, 512, 256, 128 DECL_FFT \suffix,10, 1024, 512, 256 DECL_FFT \suffix,11, 2048, 1024, 512 DECL_FFT \suffix,12, 4096, 2048, 1024 DECL_FFT \suffix,13, 8192, 4096, 2048 DECL_FFT \suffix,14,16384, 8192, 4096 DECL_FFT \suffix,15,32768,16384, 8192 DECL_FFT \suffix,16,65536,32768,16384 fft_calc \suffix .rodata .align 3 fft_dispatch_tab\suffix\()_altivec: PTR fft4\suffix\()_altivec PTR fft8\suffix\()_altivec PTR fft16\suffix\()_altivec PTR fft32\suffix\()_altivec PTR fft64\suffix\()_altivec PTR fft128\suffix\()_altivec PTR fft256\suffix\()_altivec PTR fft512\suffix\()_altivec PTR fft1024\suffix\()_altivec PTR fft2048\suffix\()_altivec PTR fft4096\suffix\()_altivec PTR fft8192\suffix\()_altivec PTR fft16384\suffix\()_altivec PTR fft32768\suffix\()_altivec PTR fft65536\suffix\()_altivec .endm DECL_FFTS 0 DECL_FFTS 1, _interleave