Define gather and scatter SIMD macro

diff --git a/timidity/optcode.h b/timidity/optcode.h
index dbb8202..b17bc36 100644 (file)
--- a/timidity/optcode.h
+++ b/timidity/optcode.h
@@ -619,28 +619,7 @@ static inline int32 signlong(int32 a)
 
 #if (USE_X86_EXT_INTRIN || USE_X86_AMD_EXT_INTRIN)
 #ifdef __GNUC__
-//#if defined(USE_X86_EXT_INTRIN) && (USE_X86_EXT_INTRIN >= 8)
-//#include <avxintrin.h>
-//#elif defined(USE_X86_EXT_INTRIN) && (USE_X86_EXT_INTRIN >= 7)
-//#include <nmmintrin.h>
-//#elif defined(USE_X86_EXT_INTRIN) && (USE_X86_EXT_INTRIN >= 6)
-//#include <smmintrin.h>
-//#elif defined(USE_X86_EXT_INTRIN) && (USE_X86_EXT_INTRIN >= 5)
-//#include <tmmintrin.h>
-//#elif defined(USE_X86_EXT_INTRIN) && (USE_X86_EXT_INTRIN >= 4)
-//#include <pmmintrin.h>
-//#elif defined(USE_X86_EXT_INTRIN) && (USE_X86_EXT_INTRIN >= 3)
-//#include <emmintrin.h>
-//#elif defined(USE_X86_EXT_INTRIN) && (USE_X86_EXT_INTRIN >= 2)
-//#include <xmmintrin.h>
-//#else
-//#include <mmintrin.h>
-//#endif
-//#if defined(USE_X86_AMD_EXT_INTRIN) && (USE_X86_AMD_EXT_INTRIN >= 2)
-//#include <mm3dnow.h>
-//#endif
-#include <immintrin.h>
-
+#include <x86intrin.h>
 #elif (_MSC_VER >= 1600) // VC2010(VC10)
 #include <intrin.h>
 #else // VC2003(VC7) VC2005(VC8) VC2008(VC9)
@@ -862,6 +841,113 @@ LSU : Unalignment (use loadu/storeu
 #endif
 #endif // (USE_X86_EXT_INTRIN >= 1)
 
+/*
+       gather and scatter
+*/
+
+#if (USE_X86_EXT_INTRIN >= 8)
+#if (USE_X86_EXT_INTRIN >= 9)
+#define MM256_I32GATHER_I32(base, offset, scale) _mm256_i32gather_epi32(base, offset, scale)
+#else
+
+static FORCEINLINE __m256i mm256_i32gather_i32_impl(const int *base, __m256i offset, int scale)
+{
+       __m256i byte_offset = _mm256_mullo_epi32(offset, _mm256_set1_epi32(scale));
+       return _mm256_set_epi32(
+               *(const int *)((const char *)base + MM256_EXTRACT_I32(byte_offset, 7)),
+               *(const int *)((const char *)base + MM256_EXTRACT_I32(byte_offset, 6)),
+               *(const int *)((const char *)base + MM256_EXTRACT_I32(byte_offset, 5)),
+               *(const int *)((const char *)base + MM256_EXTRACT_I32(byte_offset, 4)),
+               *(const int *)((const char *)base + MM256_EXTRACT_I32(byte_offset, 3)),
+               *(const int *)((const char *)base + MM256_EXTRACT_I32(byte_offset, 2)),
+               *(const int *)((const char *)base + MM256_EXTRACT_I32(byte_offset, 1)),
+               *(const int *)((const char *)base + MM256_EXTRACT_I32(byte_offset, 0))
+       );
+}
+
+#define MM256_I32GATHER_I32(base, offset, scale) mm256_i32gather_i32_impl(base, offset, scale)
+#endif // (USE_X86_EXT_INTRIN >= 9)
+
+static FORCEINLINE void mm256_i32scatter_i32_impl(void *base, __m256i offset, __m256i val, int scale)
+{
+       __m256i byte_offset = _mm256_mullo_epi32(offset, _mm256_set1_epi32(scale));
+       *(int *)((char *)base + MM256_EXTRACT_I32(byte_offset, 7)) = MM256_EXTRACT_I32(val, 7);
+       *(int *)((char *)base + MM256_EXTRACT_I32(byte_offset, 6)) = MM256_EXTRACT_I32(val, 6);
+       *(int *)((char *)base + MM256_EXTRACT_I32(byte_offset, 5)) = MM256_EXTRACT_I32(val, 5);
+       *(int *)((char *)base + MM256_EXTRACT_I32(byte_offset, 4)) = MM256_EXTRACT_I32(val, 4);
+       *(int *)((char *)base + MM256_EXTRACT_I32(byte_offset, 3)) = MM256_EXTRACT_I32(val, 3);
+       *(int *)((char *)base + MM256_EXTRACT_I32(byte_offset, 2)) = MM256_EXTRACT_I32(val, 2);
+       *(int *)((char *)base + MM256_EXTRACT_I32(byte_offset, 1)) = MM256_EXTRACT_I32(val, 1);
+       *(int *)((char *)base + MM256_EXTRACT_I32(byte_offset, 0)) = MM256_EXTRACT_I32(val, 0);
+}
+
+#define MM256_I32SCATTER_I32(base, offset, val, scale) mm256_i32scatter_i32_impl(base, offset, val, scale)
+
+#endif // (USE_X86_EXT_INTRIN >= 8)
+
+#if (USE_X86_EXT_INTRIN >= 1)
+#if (USE_X86_EXT_INTRIN >= 9)
+#define MM_I32GATHER_I32(base, offset, scale) _mm_i32gather_epi32(base, offset, scale)
+#else
+
+#if (USE_X86_EXT_INTRIN >= 6)
+
+static FORCEINLINE __m128i mm_i32gather_i32_impl(const int *base, __m128i offset, int scale)
+{
+       __m128i byte_offset = _mm_mullo_epi32(offset, _mm_set1_epi32(scale));
+       return _mm_set_epi32(
+               *(const int *)((const char *)base + MM_EXTRACT_I32(byte_offset, 3)),
+               *(const int *)((const char *)base + MM_EXTRACT_I32(byte_offset, 2)),
+               *(const int *)((const char *)base + MM_EXTRACT_I32(byte_offset, 1)),
+               *(const int *)((const char *)base + MM_EXTRACT_I32(byte_offset, 0))
+       );
+}
+
+#else
+
+static FORCEINLINE __m128i mm_i32gather_i32_impl(const int *base, __m128i offset, int scale)
+{
+       return _mm_set_epi32(
+               *(const int *)((const char *)base + MM_EXTRACT_I32(offset, 3) * scale),
+               *(const int *)((const char *)base + MM_EXTRACT_I32(offset, 2) * scale),
+               *(const int *)((const char *)base + MM_EXTRACT_I32(offset, 1) * scale),
+               *(const int *)((const char *)base + MM_EXTRACT_I32(offset, 0) * scale)
+       );
+}
+
+#endif // (USE_X86_EXT_INTRIN >= 6)
+
+#define MM_I32GATHER_I32(base, offset, scale) mm_i32gather_i32_impl(base, offset, scale)
+
+#endif // (USE_X86_EXT_INTRIN >= 9)
+
+#if (USE_X86_EXT_INTRIN >= 6)
+
+static FORCEINLINE void mm_i32scatter_i32_impl(void *base, __m128i offset, __m128i val, int scale)
+{
+       __m128i byte_offset = _mm_mullo_epi32(offset, _mm_set1_epi32(scale));
+       *(int *)((char *)base + MM_EXTRACT_I32(byte_offset, 3)) = MM_EXTRACT_I32(val, 3);
+       *(int *)((char *)base + MM_EXTRACT_I32(byte_offset, 2)) = MM_EXTRACT_I32(val, 2);
+       *(int *)((char *)base + MM_EXTRACT_I32(byte_offset, 1)) = MM_EXTRACT_I32(val, 1);
+       *(int *)((char *)base + MM_EXTRACT_I32(byte_offset, 0)) = MM_EXTRACT_I32(val, 0);
+}
+
+#else
+
+static FORCEINLINE void mm_i32scatter_i32_impl(void *base, __m128i offset, __m128i val, int scale)
+{
+       *(int *)((char *)base + MM_EXTRACT_I32(offset, 3) * scale) = MM_EXTRACT_I32(val, 3);
+       *(int *)((char *)base + MM_EXTRACT_I32(offset, 2) * scale) = MM_EXTRACT_I32(val, 2);
+       *(int *)((char *)base + MM_EXTRACT_I32(offset, 1) * scale) = MM_EXTRACT_I32(val, 1);
+       *(int *)((char *)base + MM_EXTRACT_I32(offset, 0) * scale) = MM_EXTRACT_I32(val, 0);
+}
+
+#endif // (USE_X86_EXT_INTRIN >= 6)
+
+#define MM_I32SCATTER_I32(base, offset, val, scale) mm_i32scatter_i32_impl(base, offset, val, scale)
+
+#endif // (USE_X86_EXT_INTRIN >= 1)
+
 #define IS_ALIGN(ptr) (!((int32)ptr & (ALIGN_SIZE - 1)))
 extern int is_x86ext_available(void);
 

diff --git a/timidity/resample.c b/timidity/resample.c
index 025c783..cb567ee 100644 (file)
--- a/timidity/resample.c
+++ b/timidity/resample.c
@@ -4277,9 +4277,9 @@ static inline DATA_T *resample_linear_multi(Voice *vp, DATA_T *dest, int32 req_c
        for(; i < count; i += 8) {
        __m256i vofsi = _mm256_srli_epi32(vofs, FRACTION_BITS);
 #if 1
-       __m256i vsrc01 = _mm256_i32gather_epi32((const int*)src, vofsi, 2);
+       __m256i vsrc01 = MM256_I32GATHER_I32((const int*)src, vofsi, 2);
        __m256i vsrc0 = _mm256_srai_epi32(_mm256_slli_epi32(vsrc01, 16), 16);
-       __m256i vsrc1 = _mm256_srli_epi32(vsrc01, 16);
+       __m256i vsrc1 = _mm256_srai_epi32(vsrc01, 16);
        __m256 vv1 = _mm256_cvtepi32_ps(vsrc0);
        __m256 vv2 = _mm256_cvtepi32_ps(vsrc1);
 #else

diff --git a/timidity/sysdep.h b/timidity/sysdep.h
index 64b9466..525cabb 100644 (file)
--- a/timidity/sysdep.h
+++ b/timidity/sysdep.h
@@ -97,6 +97,15 @@
 #define restrict /* not C99 */
 #endif /* !restrict */
 
+#ifndef FORCEINLINE
+#ifdef __GNUC__
+#define FORCEINLINE __attribute__((__always_inline__))
+#elif defined(_MSC_VER)
+#define FORCEINLINE __forceinline
+#else
+#define FORCEINLINE inline
+#endif
+#endif /* FORCEINLINE */
 
 /* The size of the internal buffer is 2^AUDIO_BUFFER_BITS samples.
    This determines maximum number of samples ever computed in a row.