actions/itgmania212121
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Update for floating point. No need to bother with the integer read because OS X doesn't use it.

Browse Source
This commit is contained in:
Steve Checkoway
2007-01-27 17:09:15 +00:00
parent f3a489aa01
commit f4d616e7a2
3 changed files with 108 additions and 550 deletions
Download Patch File Download Diff File Expand all files Collapse all files
stepmania/src/archutils/Darwin/VectorHelper.cpp
+104 -531
View File
@@ -29,131 +29,100 @@ bool Vector::CheckForVector()
return !sysctl( selectors, 2, &result, &length, NULL, 0 ) && result;
}
/* for( size_t pos = 0; pos < size; ++pos )
* dest[pos] += src[pos] * volume;
/* for( unsigned pos = 0; pos < size; ++pos )
* dest[pos] += src[pos];
* Idea from: http://developer.apple.com/hardwaredrivers/ve/downloads/add.c */
void Vector::FastSoundWrite( int32_t *dest, const int16_t *src, unsigned size, short volume )
void Vector::FastSoundWrite( float *dest, const float *src, unsigned size )
{
if( size > 7 )
if( size > 4 )
{
int index = 0;
vUInt8 one = (vUInt8)(1);
vUInt8 volMask = vec_lvsl( 0, &volume );
vSInt16 vol = vec_lde( 0, &volume );
vol = vec_splat( vec_perm(vol, vol, volMask), 0 );
// Setup the masks.
vUInt8 srcMask = vec_add( vec_lvsl(15, src), one );
vUInt8 loadMask = vec_add( vec_lvsl(15, dest), one );
vUInt8 storeMask = vec_lvsr( 0, dest ); // I have no idea why shift right for stores.
vSInt16 load1Src = vec_ld( 0, src );
vSInt32 load1Dest = vec_ld( 0, dest );
vSInt32 store = (vSInt32)(0);
vUInt8 destMask = vec_add(vec_lvsl(15, dest), one );
vUInt8 storeMask = vec_lvsr( 0, dest );
vFloat load1Src = vec_ld( 0, src );
vFloat load1Dest = vec_ld( 0, dest );
vFloat store = (vFloat)(0.0f);
// If dest is misaligned, pull first loop iteration out.
// If dest is misaligned, pull the first loop iteration out.
if( intptr_t(dest) & 0xF )
{
vSInt16 load2Src = vec_ld( 15, src );
vSInt32 load2Dest = vec_ld( 15, dest );
vSInt32 load3Dest = vec_ld( 31, dest );
load1Src = vec_perm( load1Src, load2Src, srcMask );
load1Dest = vec_perm( load1Dest, load2Dest, loadMask );
load2Dest = vec_perm( load2Dest, load3Dest, loadMask );
vFloat load2Src = vec_ld( 15, src );
vFloat load2Dest = vec_ld( 15, dest );
/* Multiply the even 2-byte elements in data with those in vol to get
* 4-byte elements. Do the same with the odd elements then merge both
* high and low halves of the vectors into two new 4-element, 4-byte
* vectors. In this way the combined vector <first,second> contains
* the 8 products in the correct order. */
vSInt32 even = vec_mule( load1Src, vol );
vSInt32 odd = vec_mulo( load1Src, vol );
vSInt32 first = vec_mergeh( even, odd );
vSInt32 second = vec_mergel( even, odd );
load1Src = vec_perm( load1Src, load2Src, srcMask );
load1Dest = vec_perm( load1Dest, load2Dest, destMask );
load1Dest = vec_add( load1Dest, first );
load2Dest = vec_add( load2Dest, second );
load1Dest = vec_add( load1Dest, load1Src );
store = vec_perm( load1Dest, load1Dest, storeMask );
load1Dest = vec_perm( load1Dest, load2Dest, storeMask );
while( (intptr_t(dest) + index) & 0xC )
{
vec_ste( store, index, dest );
index += 4;
}
vec_st( load1Dest, index, dest );
load1Src = load2Src;
load1Dest = load3Dest;
store = load2Dest;
src += 8;
dest += 8;
size -= 8;
store = load1Dest;
load1Dest = load2Dest;
src += 4;
dest += 4;
size -= 4;
/* Incrementing the index is supposed to have the same effect
* as incrementing dest bust since we read from dest as well
* as incrementing dest but since we read from dest as well
* we don't want to increment twice so decrement the index. */
index -= 16;
// XXX: What in the world did I mean here?
index -= 16;
}
while( size >= 32 )
{
vSInt16 load2Src = vec_ld( 15, src );
vSInt16 load3Src = vec_ld( 31, src );
vSInt16 load4Src = vec_ld( 47, src );
vSInt16 load5Src = vec_ld( 63, src );
vSInt32 load2Dest = vec_ld( 15, dest );
vSInt32 load3Dest = vec_ld( 31, dest );
vSInt32 load4Dest = vec_ld( 47, dest );
vSInt32 load5Dest = vec_ld( 63, dest );
vSInt32 load6Dest = vec_ld( 79, dest );
vSInt32 load7Dest = vec_ld( 95, dest );
vSInt32 load8Dest = vec_ld( 111, dest );
vSInt32 load9Dest = vec_ld( 127, dest );
// Align the data
vFloat load2Src = vec_ld( 15, src );
vFloat load3Src = vec_ld( 31, src );
vFloat load4Src = vec_ld( 47, src );
vFloat load5Src = vec_ld( 63, src );
vFloat load6Src = vec_ld( 79, src );
vFloat load7Src = vec_ld( 95, src );
vFloat load8Src = vec_ld( 111, src );
vFloat load9Src = vec_ld( 127, src );
vFloat load2Dest = vec_ld( 15, dest );
vFloat load3Dest = vec_ld( 31, dest );
vFloat load4Dest = vec_ld( 47, dest );
vFloat load5Dest = vec_ld( 63, dest );
vFloat load6Dest = vec_ld( 79, dest );
vFloat load7Dest = vec_ld( 95, dest );
vFloat load8Dest = vec_ld( 111, dest );
vFloat load9Dest = vec_ld( 127, dest );
// Align the data.
load1Src = vec_perm( load1Src, load2Src, srcMask );
load2Src = vec_perm( load2Src, load3Src, srcMask );
load3Src = vec_perm( load3Src, load4Src, srcMask );
load4Src = vec_perm( load4Src, load5Src, srcMask );
load5Src = vec_perm( load5Src, load6Src, srcMask );
load6Src = vec_perm( load6Src, load7Src, srcMask );
load7Src = vec_perm( load7Src, load8Src, srcMask );
load8Src = vec_perm( load8Src, load9Src, srcMask );
// Not load5Src, it's untouched and used later.
load1Dest = vec_perm( load1Dest, load2Dest, loadMask );
load2Dest = vec_perm( load2Dest, load3Dest, loadMask );
load3Dest = vec_perm( load3Dest, load4Dest, loadMask );
load4Dest = vec_perm( load4Dest, load5Dest, loadMask );
load5Dest = vec_perm( load5Dest, load6Dest, loadMask );
load6Dest = vec_perm( load6Dest, load7Dest, loadMask );
load7Dest = vec_perm( load7Dest, load8Dest, loadMask );
load8Dest = vec_perm( load8Dest, load9Dest, loadMask );
load1Dest = vec_perm( load1Dest, load2Dest, destMask );
load2Dest = vec_perm( load2Dest, load3Dest, destMask );
load3Dest = vec_perm( load3Dest, load4Dest, destMask );
load4Dest = vec_perm( load4Dest, load5Dest, destMask );
load5Dest = vec_perm( load5Dest, load6Dest, destMask );
load6Dest = vec_perm( load6Dest, load7Dest, destMask );
load7Dest = vec_perm( load7Dest, load8Dest, destMask );
load8Dest = vec_perm( load8Dest, load9Dest, destMask );
// Not load9Dest.
vSInt32 even1 = vec_mule( load1Src, vol );
vSInt32 odd1 = vec_mulo( load1Src, vol );
vSInt32 even2 = vec_mule( load2Src, vol );
vSInt32 odd2 = vec_mulo( load2Src, vol );
vSInt32 even3 = vec_mule( load3Src, vol );
vSInt32 odd3 = vec_mulo( load3Src, vol );
vSInt32 even4 = vec_mule( load4Src, vol );
vSInt32 odd4 = vec_mulo( load4Src, vol );
vSInt32 first = vec_mergeh( even1, odd1 );
vSInt32 second = vec_mergel( even1, odd1 );
vSInt32 third = vec_mergeh( even2, odd2 );
vSInt32 fourth = vec_mergel( even2, odd2 );
vSInt32 fifth = vec_mergeh( even3, odd3 );
vSInt32 sixth = vec_mergel( even3, odd3 );
vSInt32 seventh = vec_mergeh( even4, odd4 );
vSInt32 eighth = vec_mergel( even4, odd4 );
load1Dest = vec_add( load1Dest, first );
load2Dest = vec_add( load2Dest, second );
load3Dest = vec_add( load3Dest, third );
load4Dest = vec_add( load4Dest, fourth );
load5Dest = vec_add( load5Dest, fifth );
load6Dest = vec_add( load6Dest, sixth );
load7Dest = vec_add( load7Dest, seventh );
load8Dest = vec_add( load8Dest, eighth );
// Unalign results.
load1Dest = vec_add( load1Dest, load1Src );
load2Dest = vec_add( load2Dest, load2Src );
load3Dest = vec_add( load3Dest, load3Src );
load4Dest = vec_add( load4Dest, load4Src );
load5Dest = vec_add( load5Dest, load5Src );
load6Dest = vec_add( load6Dest, load6Src );
load7Dest = vec_add( load7Dest, load7Src );
load8Dest = vec_add( load8Dest, load8Src );
// Unalign the results.
store = vec_perm( store, load1Dest, storeMask );
load1Dest = vec_perm( load1Dest, load2Dest, storeMask );
load2Dest = vec_perm( load2Dest, load3Dest, storeMask );
@@ -172,51 +141,41 @@ void Vector::FastSoundWrite( int32_t *dest, const int16_t *src, unsigned size, s
vec_st( load5Dest, index + 80, dest );
vec_st( load6Dest, index + 96, dest );
vec_st( load7Dest, index + 112, dest );
load1Src = load5Src;
load1Src = load9Src;
load1Dest = load9Dest;
store = load8Dest;
dest += 32;
src += 32;
size -= 32;
}
/* This completely baffles gcc's loop unrolling. If I make it > 7 instead,
/* This completely baffles gcc's loop unrolling. If I make it > 3 instead,
* then gcc produces 4 identical copies of the loop without scheduling them
* in a sane manner (hence the manual unrolling above) but this loop will
* never be executed more than 3 times so that code will never be used.
* This produces code the way gcc _should_ do it by unrolling and scheduling
* and then producing the rolled version. */
while( size & ~0x7 )
while( size & ~0x3 )
{
vSInt16 load2Src = vec_ld( 15, src );
vSInt32 load2Dest = vec_ld( 15, dest );
vSInt32 load3Dest = vec_ld( 31, dest );
vFloat load2Src = vec_ld( 15, src );
vFloat load2Dest = vec_ld( 15, dest );
load1Src = vec_perm( load1Src, load2Src, srcMask );
load1Dest = vec_perm( load1Dest, load2Dest, loadMask );
load2Dest = vec_perm( load2Dest, load3Dest, loadMask );
vSInt32 even = vec_mule( load1Src, vol );
vSInt32 odd = vec_mulo( load1Src, vol );
vSInt32 first = vec_mergeh( even, odd );
vSInt32 second = vec_mergel( even, odd );
load1Dest = vec_add( load1Dest, first );
load2Dest = vec_add( load2Dest, second );
load1Dest = vec_perm( load1Dest, load2Dest, destMask );
load1Dest = vec_add( load1Dest, load1Src );
store = vec_perm( store, load1Dest, storeMask );
load1Dest = vec_perm( load1Dest, load2Dest, storeMask );
vec_st( store, index + 0, dest );
vec_st( load1Dest, index + 16, dest );
vec_st( store, index, dest );
load1Src = load2Src;
load1Dest = load3Dest;
store = load2Dest;
src += 8;
dest += 8;
size -= 8;
store = load1Dest;
load1Dest = load2Dest;
src += 4;
dest += 4;
size -= 4;
}
// Store the remainder of the vector, if it was misaligned.
if( index < 0 )
{
@@ -229,259 +188,11 @@ void Vector::FastSoundWrite( int32_t *dest, const int16_t *src, unsigned size, s
}
}
/* If we account for both misaligned dest and src, there is really no way to
* do this in vector code so do the last at most 7 elements in scalar code. */
* do this in vector code so do the last at most 3 elements in scalar code. */
while( size-- )
*(dest++) += *(src++) * volume;
*(dest++) += *(src++);
}
/* for( size_t pos = 0; pos < size; ++pos )
* dest[pos] = clamp( src[pos]/256, -32768, 32767 );
*/
void Vector::FastSoundRead( int16_t *dest, const int32_t *src, unsigned size )
{
int index = 0;
vSInt32 zero = (vSInt32)( 0 );
vUInt32 shift = (vUInt32)( 8 );
vSInt16 store = (vSInt16)( 0 );
vUInt8 storeMask = vec_lvsr( 0, dest ); // Setup the store mask.
if( intptr_t(dest) & 0xF )
{
index -= intptr_t(dest) & 0xF;
store = vec_ld( 0, dest );
}
/* This is tricky. We need to divide signed 4-byte integers by 256 and stuff
* them into 2-byte integers. First, find the elements which are negative
* by comparing to zero (those less than zero will have each bit in the
* 32-bit element set to 1 and those at least zero will have them all set
* to 0). Take the absolute value (it actually subtracts the vector from zero
* and computes the max to do that), shift right by 8 bits, use the masks
* to get vectors containing only those elements which were negative and
* subtract twice. Use saturated arithmatic to deal with overflow. Lastly,
* pack the two vectors into signed 2-byte integers (again saturated). */
while( size >= 32 )
{
// Use LRU load which marks the address as LRU. Does nothing on the G5.
vSInt32 first = vec_ldl( 0, src );
vSInt32 second = vec_ldl( 16, src );
vSInt32 third = vec_ldl( 32, src );
vSInt32 fourth = vec_ldl( 48, src );
vSInt32 fifth = vec_ldl( 64, src );
vSInt32 sixth = vec_ldl( 80, src );
vSInt32 seventh = vec_ldl( 96, src );
vSInt32 eighth = vec_ldl( 112, src );
vSInt32 temp1 = (vSInt32)vec_cmplt( first, zero );
vSInt32 temp2 = (vSInt32)vec_cmplt( second, zero );
vSInt32 temp3 = (vSInt32)vec_cmplt( third, zero );
vSInt32 temp4 = (vSInt32)vec_cmplt( fourth, zero );
vSInt32 temp5 = (vSInt32)vec_cmplt( fifth, zero );
vSInt32 temp6 = (vSInt32)vec_cmplt( sixth, zero );
vSInt32 temp7 = (vSInt32)vec_cmplt( seventh, zero );
vSInt32 temp8 = (vSInt32)vec_cmplt( eighth, zero );
first = vec_sr( vec_abss(first), shift );
second = vec_sr( vec_abss(second), shift );
third = vec_sr( vec_abss(third), shift );
fourth = vec_sr( vec_abss(fourth), shift );
fifth = vec_sr( vec_abss(fifth), shift );
sixth = vec_sr( vec_abss(sixth), shift );
seventh = vec_sr( vec_abss(seventh), shift );
eighth = vec_sr( vec_abss(eighth), shift );
temp1 = vec_and( first, temp1 );
temp2 = vec_and( second, temp2 );
temp3 = vec_and( third, temp3 );
temp4 = vec_and( fourth, temp4 );
temp5 = vec_and( fifth, temp5 );
temp6 = vec_and( sixth, temp6 );
temp7 = vec_and( seventh, temp7 );
temp8 = vec_and( eighth, temp8 );
first = vec_subs( vec_sub(first, temp1), temp1 );
second = vec_subs( vec_sub(second, temp2), temp2 );
third = vec_subs( vec_sub(third, temp3), temp3 );
fourth = vec_subs( vec_sub(fourth, temp4), temp4 );
fifth = vec_subs( vec_sub(fifth, temp5), temp5 );
sixth = vec_subs( vec_sub(sixth, temp6), temp6 );
seventh = vec_subs( vec_sub(seventh, temp7), temp7 );
eighth = vec_subs( vec_sub(eighth, temp8), temp8 );
vSInt16 result1 = vec_packs( first, second );
vSInt16 result2 = vec_packs( third, fourth );
vSInt16 result3 = vec_packs( fifth, sixth );
vSInt16 result4 = vec_packs( seventh, eighth );
store = vec_perm( store, result1, storeMask );
result1 = vec_perm( result1, result2, storeMask );
result2 = vec_perm( result2, result3, storeMask );
result3 = vec_perm( result3, result4, storeMask );
vec_st( store, 0, dest );
vec_st( result1, 16, dest );
vec_st( result2, 32, dest );
vec_st( result3, 48, dest );
store = result4;
dest += 32;
src += 32;
size -= 32;
}
// Befuddle optimizer as above.
while( size & ~0x7 )
{
vSInt32 first = vec_ldl( 0, src );
vSInt32 second = vec_ldl( 16, src );
vSInt32 temp1 = (vSInt32)vec_cmplt( first, zero );
vSInt32 temp2 = (vSInt32)vec_cmplt( second, zero );
first = vec_sr( vec_abss(first), shift );
second = vec_sr( vec_abss(second), shift );
temp1 = vec_and( first, temp1 );
temp2 = vec_and( second, temp2 );
first = vec_subs( vec_sub(first, temp1), temp1 );
second = vec_subs( vec_sub(second, temp2), temp2 );
vSInt16 result = vec_packs( first, second );
vec_st( vec_perm(store, result, storeMask), 0, dest );
store = result;
dest += 8;
src += 8;
size -= 8;
}
store = vec_perm( store, store, storeMask );
int temp = index;
while( index < 0 )
{
vec_ste( store, index, dest );
index += 2;
}
temp >>=1;
dest += temp;
src += temp;
size -= temp;
while( size-- )
*dest++ = max( -32768, min(*src++>>8, 32767) );
}
/* for( size_t pos = 0; pos < size; ++pos )
* dest[pos] = SCALE( float(src[pos]), -32768*256, 32767*256, -1.0f, 1.0f );
*/
void Vector::FastSoundRead( float *dest, const int32_t *src, unsigned size )
{
/* m = -32768; M = 32767
* (x-2^8*m)(1-(-1))/(2^8*M-2^8*m)+(-1)
* = ((x-2^8*m)/(2^8))*(2/(M-m))+(-1)
* = ((x-2^8*m)/(2^16))*((2*2^8)/(M-m))+(-1)
* l1 = 2^8*m = -8388608
* scale = 2*2^8/(M-m) = 0.00781261921110856794
* l2 = -1 */
int index = 0;
vFloat scale = (vFloat) ( 0.00781261921110856794f );
vSInt32 l1 = (vSInt32) ( -8388608 );
vFloat l2 = (vFloat) ( -1.0f );
vUInt8 storeMask = vec_lvsr( 0, dest ); // Setup the store mask.
vFloat st = (vFloat)( 0.0f );
if( intptr_t(dest) & 0xF )
{
index -= intptr_t(dest) & 0xF;
st = vec_ld( 0, dest );
}
while( size >= 32 )
{
/* By far the simplest of these, we need only perform the scale
* operation which amounts to subtracting l1, converting to a float,
* multiplying by a constant, and adding l2. We can multiply and add
* in one instruction. The 16 in vec_ctf(X,16) divides by 2^16. */
vSInt32 x1 = vec_ldl( 0, src );
vSInt32 x2 = vec_ldl( 16, src );
vSInt32 x3 = vec_ldl( 32, src );
vSInt32 x4 = vec_ldl( 48, src );
vSInt32 x5 = vec_ldl( 64, src );
vSInt32 x6 = vec_ldl( 80, src );
vSInt32 x7 = vec_ldl( 96, src );
vSInt32 x8 = vec_ldl( 112, src );
x1 = vec_subs( x1, l1 );
x2 = vec_subs( x2, l1 );
x3 = vec_subs( x3, l1 );
x4 = vec_subs( x4, l1 );
x5 = vec_subs( x5, l1 );
x6 = vec_subs( x6, l1 );
x7 = vec_subs( x7, l1 );
x8 = vec_subs( x8, l1 );
vFloat f1 = vec_ctf( x1, 16 );
vFloat f2 = vec_ctf( x2, 16 );
vFloat f3 = vec_ctf( x3, 16 );
vFloat f4 = vec_ctf( x4, 16 );
vFloat f5 = vec_ctf( x5, 16 );
vFloat f6 = vec_ctf( x6, 16 );
vFloat f7 = vec_ctf( x7, 16 );
vFloat f8 = vec_ctf( x8, 16 );
f1 = vec_madd( f1, scale, l2 );
f2 = vec_madd( f2, scale, l2 );
f3 = vec_madd( f3, scale, l2 );
f4 = vec_madd( f4, scale, l2 );
f5 = vec_madd( f5, scale, l2 );
f6 = vec_madd( f6, scale, l2 );
f7 = vec_madd( f7, scale, l2 );
f8 = vec_madd( f8, scale, l2 );
st = vec_perm( st, f1, storeMask );
f1 = vec_perm( f1, f2, storeMask );
f2 = vec_perm( f2, f3, storeMask );
f3 = vec_perm( f3, f4, storeMask );
f4 = vec_perm( f4, f5, storeMask );
f5 = vec_perm( f5, f6, storeMask );
f6 = vec_perm( f6, f7, storeMask );
f7 = vec_perm( f7, f8, storeMask );
vec_st( st, 0, dest );
vec_st( f1, 16, dest );
vec_st( f2, 32, dest );
vec_st( f3, 48, dest );
vec_st( f4, 64, dest );
vec_st( f5, 80, dest );
vec_st( f6, 96, dest );
vec_st( f7, 112, dest );
st = f8;
dest += 32;
src += 32;
size -= 32;
}
while( size & ~0x3 )
{
vFloat result = vec_ctf( vec_subs(vec_ldl(0, src), l1), 16 );
result = vec_madd( result, scale, l2 );
st = vec_perm( st, result, storeMask );
vec_st( st, 0, dest );
st = result;
dest += 4;
src += 4;
size -= 4;
}
st = vec_perm( st, st, storeMask );
int temp = index;
while( index < 0 )
{
vec_ste( st, index, dest );
index += 2;
}
temp >>= 2;
dest += temp;
src += temp;
size -= temp;
while( size-- )
*dest++ = float( *src++ + 32768*256 ) * 1.1921110856794079500e-7f - 1.0f;
}
#elif defined(__SSE2__)
#include <xmmintrin.h>
// This is portable to other sysems since it uses Intel's intrinsics.
@@ -493,180 +204,42 @@ bool Vector::CheckForVector()
}
template<typename T>
static inline void Write( T load, int32_t *&dest, const int16_t *&src,
unsigned &size, __m128i vol ) __attribute__((always_inline));
template<typename T>
inline void Write( T load, int32_t *&dest, const int16_t *&src, unsigned &size, __m128i vol )
static inline void __attribute__((always_inline)) Write( T load, float *&dest, const float *&src, unsigned &size )
{
// There are only 8 XMM registers so no 4x unrolling
// There are only 8 XMM registers so no 8x unrolling. Let's do 2 though.
while( size >= 8 )
{
__m128i data = load( (__m128i *)src );
__m128i hi = _mm_mulhi_epi16( data, vol );
__m128i low = _mm_mullo_epi16( data, vol );
__m128i result1 = _mm_unpacklo_epi16( low, hi );
__m128i result2 = _mm_unpackhi_epi16( low, hi );
result1 = _mm_add_epi32( result1, *(__m128i *)(dest + 0) );
result2 = _mm_add_epi32( result2, *(__m128i *)(dest + 4) );
_mm_store_si128( (__m128i *)(dest + 0), result1 );
_mm_store_si128( (__m128i *)(dest + 4), result2 );
src += 8;
dest += 8;
size -= 8;
}
}
{
__m128 data1 = load( src + 0 );
__m128 data2 = load( src + 4 );
void Vector::FastSoundWrite( int32_t *dest, const int16_t *src, unsigned size, short volume )
data1 = _mm_add_ps( data1, *(__m128 *)(dest + 0) );
data2 = _mm_add_ps( data2, *(__m128 *)(dest + 4) );
_mm_store_ps( dest + 0, data1 );
_mm_store_ps( dest + 4, data2 );
src += 8;
dest += 8;
size -= 8;
}
}
void Vector::FastSoundWrite( float *dest, const float *src, unsigned size )
{
while( (intptr_t(dest) & 0xF) && size )
{
// Misaligned stores are slow.
*(dest++) += *(src++) * volume;
*(dest++) += *(src++);
--size;
}
__m128i vol = _mm_set1_epi16( volume );
// Misaligned loads are slower so specialize to aligned loads when possible.
if( intptr_t(src) & 0xF )
Write( _mm_loadu_si128, dest, src, size, vol );
Write( _mm_loadu_ps, dest, src, size );
else
Write( _mm_load_si128, dest, src, size, vol );
Write( _mm_load_ps, dest, src, size );
while( size-- )
*(dest++) += *(src++) * volume;
*(dest++) += *(src++);
}
template<typename T>
static inline void Read( T load, int16_t *&dest, const int32_t *&src, unsigned &size ) __attribute__((always_inline));
template<typename T>
static inline void Read( T load, int16_t *&dest, const int32_t *&src, unsigned &size )
{
__m128i zero = _mm_setzero_si128();
while( size >= 8 )
{
__m128i data1 = load( (__m128i *)(src + 0) );
__m128i data2 = load( (__m128i *)(src + 4) );
__m128i mask1 = _mm_cmplt_epi32( data1, zero );
__m128i mask2 = _mm_cmplt_epi32( data2, zero );
__m128i t1 = _mm_srai_epi32( data1, 31 );
__m128i t2 = _mm_srai_epi32( data2, 31 );
// We can't do 32 bit saturating arithmetic but that's unlikely to be a problem
data1 = _mm_sub_epi32( _mm_xor_si128(data1, t1), t1 );
data2 = _mm_sub_epi32( _mm_xor_si128(data2, t2), t2 );
data1 = _mm_srai_epi32( data1, 8 );
data2 = _mm_srai_epi32( data2, 8 );
mask1 = _mm_and_si128( mask1, data1 ); // destructive logic, we want data1 still
mask2 = _mm_and_si128( mask2, data2 ); // destructive logic
data1 = _mm_sub_epi32( _mm_sub_epi32(data1, mask1), mask1 );
data2 = _mm_sub_epi32( _mm_sub_epi32(data2, mask2), mask2 );
data1 = _mm_packs_epi32( data1, data2 );
_mm_store_si128( (__m128i *)dest, data1 );
src += 8;
dest += 8;
size -= 8;
}
if( size )
{
__m128i data1 = load( (__m128i *)(src + 0) );
__m128i data2 = size > 4 ? load( (__m128i *)(src + 4) ) : zero;
__m128i mask1 = _mm_cmplt_epi32( data1, zero );
__m128i mask2 = _mm_cmplt_epi32( data2, zero );
__m128i t1 = _mm_srai_epi32( data1, 31 );
__m128i t2 = _mm_srai_epi32( data2, 31 );
// We can't do 32 bit saturating arithmetic but that's unlikely to be a problem
data1 = _mm_sub_epi32( _mm_xor_si128(data1, t1), t1 );
data2 = _mm_sub_epi32( _mm_xor_si128(data2, t2), t2 );
data1 = _mm_srai_epi32( data1, 8 );
data2 = _mm_srai_epi32( data2, 8 );
mask1 = _mm_and_si128( mask1, data1 ); // destructive logic, we want data1 still
mask2 = _mm_and_si128( mask2, data2 ); // destructive logic
data1 = _mm_sub_epi32( _mm_sub_epi32(data1, mask1), mask1 );
data2 = _mm_sub_epi32( _mm_sub_epi32(data2, mask2), mask2 );
data1 = _mm_packs_epi32( data1, data2 );
#define X(x) (-(size >= (x)))
data2 = _mm_set_epi8( 0, 0, X(7), X(7), X(6), X(6), X(5), X(5),
X(4), X(4), X(3), X(3), X(2), X(2), -1, -1 );
#undef X
_mm_maskmoveu_si128( data1, data2, (char *)dest );
}
}
void Vector::FastSoundRead( int16_t *dest, const int32_t *src, unsigned size )
{
while( (intptr_t(dest) & 0xF) && size )
{
// Misaligned stores are very slow.
*dest++ = max( -32768, min(*src++>>8, 32767) );
--size;
}
// Specialize loads.
if( intptr_t(src) & 0xF )
Read( _mm_loadu_si128, dest, src, size );
else
Read( _mm_load_si128, dest, src, size );
while( size-- )
*dest++ = max( -32768, min(*src++>>8, 32767) );
}
template<typename T>
static inline void Read( T load, float *&dest, const int32_t *&src, unsigned &size ) __attribute__((always_inline));
template<typename T>
static inline void Read( T load, float *&dest, const int32_t *&src, unsigned &size )
{
/* m = -32768; M = 32767
* (x-2^8*m)(1-(-1))/(2^8*M-2^8*m)+(-1)
* (x-2^8*m)/(2^7*(M-m))+(-1)
* l1 = 2^8*m = -8388608
* scale = 1/(2^7*(M-m)) = 0.00000011921110856794
* l2 = -1 */
__m128 scale = _mm_set1_ps( 0.00000011921110856794f );
__m128i l1 = _mm_set1_epi32( -8388608 );
__m128 l2 = _mm_set1_ps( -1.0f );
while( size >= 4 )
{
__m128i data = _mm_sub_epi32( load((__m128i *)src), l1 );
__m128 result = _mm_cvtepi32_ps( data );
result = _mm_add_ps( _mm_mul_ps(result, scale), l2 );
_mm_store_ps( dest, result );
src += 4;
dest += 4;
size -= 4;
}
if( size )
{
#define X(x) (-(size >= (x)))
__m128i storeMask = _mm_set_epi8( 0, 0, 0, 0, X(3), X(3), X(3), X(3),
X(2), X(2), X(2), X(2), -1, -1, -1, -1 );
#undef X
__m128i data = _mm_sub_epi32( load((__m128i *)src), l1 );
__m128 result = _mm_cvtepi32_ps( data );
result = _mm_add_ps( _mm_mul_ps(result, scale), l2 );
// This might not be valid.
_mm_maskmoveu_si128( (__m128i)result, storeMask, (char *)dest );
}
}
void Vector::FastSoundRead( float *dest, const int32_t *src, unsigned size )
{
while( (intptr_t(dest) & 0xF) && size )
{
// Misaligned stores are very slow.
*dest++ = float( *src++ + 32768*256 ) * 1.1921110856794079500e-7f - 1.0f;
--size;
}
// Specialize loads.
if( intptr_t(src) & 0xF )
Read( _mm_loadu_si128, dest, src, size );
else
Read( _mm_load_si128, dest, src, size );
while( size-- )
*dest++ = float( *src++ + 32768*256 ) * 1.1921110856794079500e-7f - 1.0f;;
}
#else
#error huh?
#endif