Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 8 additions & 8 deletions internal/ailego/math/fht_amd64.go
Original file line number Diff line number Diff line change
Expand Up @@ -49,15 +49,15 @@ func fhtFlipSignsAVX2(signs []byte, data []float32) {
fhtFlipSignsScalar(signs, data)
return
}
xvec_avx2_fht_flip_signs(unsafe.Pointer(&signs[0]), unsafe.Pointer(&data[0]), int64(len(data)))
fht_flip_sign_avx2(unsafe.Pointer(&signs[0]), unsafe.Pointer(&data[0]), int64(len(data)))
}

func fhtKacWalkAVX2(data []float32) {
if len(data)/2 < 8 {
fhtKacWalkScalar(data)
return
}
xvec_avx2_fht_kac_walk(unsafe.Pointer(&data[0]), int64(len(data)))
fht_kacs_walk_avx2(unsafe.Pointer(&data[0]), int64(len(data)))
if len(data)%2 != 0 {
data[len(data)/2] *= float32(math.Sqrt2)
}
Expand All @@ -71,31 +71,31 @@ func fhtInverseKacWalkAVX2(data []float32) {
if len(data)%2 != 0 {
data[len(data)/2] *= float32(math.Sqrt(.5))
}
xvec_avx2_fht_inverse_kac_walk(unsafe.Pointer(&data[0]), int64(len(data)))
fht_inv_kacs_walk_avx2(unsafe.Pointer(&data[0]), int64(len(data)))
}

func fhtInPlaceAVX2(data []float32) {
if len(data) < 16 {
fhtInPlaceScalar(data)
return
}
xvec_avx2_fht_in_place(unsafe.Pointer(&data[0]), int64(len(data)))
fht_inplace_avx2(unsafe.Pointer(&data[0]), int64(len(data)))
}

func fhtFlipSignsAVX512(signs []byte, data []float32) {
if len(data) < 16 {
fhtFlipSignsScalar(signs, data)
return
}
xvec_avx512_fht_flip_signs(unsafe.Pointer(&signs[0]), unsafe.Pointer(&data[0]), int64(len(data)))
fht_flip_sign_avx512(unsafe.Pointer(&signs[0]), unsafe.Pointer(&data[0]), int64(len(data)))
}

func fhtKacWalkAVX512(data []float32) {
if len(data)/2 < 16 {
fhtKacWalkScalar(data)
return
}
xvec_avx512_fht_kac_walk(unsafe.Pointer(&data[0]), int64(len(data)))
fht_kacs_walk_avx512(unsafe.Pointer(&data[0]), int64(len(data)))
if len(data)%2 != 0 {
data[len(data)/2] *= float32(math.Sqrt2)
}
Expand All @@ -109,13 +109,13 @@ func fhtInverseKacWalkAVX512(data []float32) {
if len(data)%2 != 0 {
data[len(data)/2] *= float32(math.Sqrt(.5))
}
xvec_avx512_fht_inverse_kac_walk(unsafe.Pointer(&data[0]), int64(len(data)))
fht_inv_kacs_walk_avx512(unsafe.Pointer(&data[0]), int64(len(data)))
}

func fhtInPlaceAVX512(data []float32) {
if len(data) < 32 {
fhtInPlaceScalar(data)
return
}
xvec_avx512_fht_in_place(unsafe.Pointer(&data[0]), int64(len(data)))
fht_inplace_avx512(unsafe.Pointer(&data[0]), int64(len(data)))
}
8 changes: 4 additions & 4 deletions internal/ailego/math/fht_arm64.go
Original file line number Diff line number Diff line change
Expand Up @@ -41,15 +41,15 @@ func fhtFlipSignsNEON(signs []byte, data []float32) {
fhtFlipSignsScalar(signs, data)
return
}
xvec_neon_fht_flip_signs(unsafe.Pointer(&signs[0]), unsafe.Pointer(&data[0]), int64(len(data)))
fht_flip_sign_neon(unsafe.Pointer(&signs[0]), unsafe.Pointer(&data[0]), int64(len(data)))
}

func fhtKacWalkNEON(data []float32) {
if len(data)/2 < 4 {
fhtKacWalkScalar(data)
return
}
xvec_neon_fht_kac_walk(unsafe.Pointer(&data[0]), int64(len(data)))
fht_kacs_walk_neon(unsafe.Pointer(&data[0]), int64(len(data)))
if len(data)%2 != 0 {
data[len(data)/2] *= float32(math.Sqrt2)
}
Expand All @@ -63,13 +63,13 @@ func fhtInverseKacWalkNEON(data []float32) {
if len(data)%2 != 0 {
data[len(data)/2] *= float32(math.Sqrt(.5))
}
xvec_neon_fht_inverse_kac_walk(unsafe.Pointer(&data[0]), int64(len(data)))
fht_inv_kacs_walk_neon(unsafe.Pointer(&data[0]), int64(len(data)))
}

func fhtInPlaceNEON(data []float32) {
if len(data) < 8 {
fhtInPlaceScalar(data)
return
}
xvec_neon_fht_in_place(unsafe.Pointer(&data[0]), int64(len(data)))
fht_inplace_neon(unsafe.Pointer(&data[0]), int64(len(data)))
}
8 changes: 4 additions & 4 deletions internal/ailego/math/fht_avx2.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

8 changes: 4 additions & 4 deletions internal/ailego/math/fht_avx2.s
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
// flags: -mavx2 -O3
// source: src/fht_avx2.c

TEXT ·xvec_avx2_fht_flip_signs(SB), $0-24
TEXT ·fht_flip_sign_avx2(SB), $0-24
MOVQ signs+0(FP), DI
MOVQ data+8(FP), SI
MOVQ size+16(FP), DX
Expand Down Expand Up @@ -252,7 +252,7 @@ LBB0_5:
LONG $0x8f748042; WORD $0x8003 // xorb $-128, 3(%rdi,%r9,4)
JMP LBB0_7

TEXT ·xvec_avx2_fht_kac_walk(SB), $0-16
TEXT ·fht_kacs_walk_avx2(SB), $0-16
MOVQ data+0(FP), DI
MOVQ size+8(FP), SI
WORD $0x8948; BYTE $0xf0 // movq %rsi, %rax
Expand Down Expand Up @@ -411,7 +411,7 @@ LBB1_21:
WORD $0x014c; BYTE $0xca // addq %r9, %rdx
JMP LBB1_5

TEXT ·xvec_avx2_fht_inverse_kac_walk(SB), $0-16
TEXT ·fht_inv_kacs_walk_avx2(SB), $0-16
MOVQ data+0(FP), DI
MOVQ size+8(FP), SI
BYTE $0x55 // pushq %rbp
Expand Down Expand Up @@ -472,7 +472,7 @@ LBB2_6:
WORD $0xf8c5; BYTE $0x77 // vzeroupper
RET

TEXT ·xvec_avx2_fht_in_place(SB), $0-16
TEXT ·fht_inplace_avx2(SB), $0-16
MOVQ data+0(FP), DI
MOVQ size+8(FP), SI
BYTE $0x55 // pushq %rbp
Expand Down
8 changes: 4 additions & 4 deletions internal/ailego/math/fht_avx512.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

8 changes: 4 additions & 4 deletions internal/ailego/math/fht_avx512.s
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
// flags: -mavx512f -mavx512dq -O3
// source: src/fht_avx512.c

TEXT ·xvec_avx512_fht_flip_signs(SB), $0-24
TEXT ·fht_flip_sign_avx512(SB), $0-24
MOVQ signs+0(FP), DI
MOVQ data+8(FP), SI
MOVQ size+16(FP), DX
Expand Down Expand Up @@ -81,7 +81,7 @@ LBB0_4:
LONG $0x03867480; BYTE $0x80 // xorb $-128, 3(%rsi,%rax,4)
JMP LBB0_6

TEXT ·xvec_avx512_fht_kac_walk(SB), $0-16
TEXT ·fht_kacs_walk_avx512(SB), $0-16
MOVQ data+0(FP), DI
MOVQ size+8(FP), SI
WORD $0x8948; BYTE $0xf0 // movq %rsi, %rax
Expand Down Expand Up @@ -240,7 +240,7 @@ LBB1_21:
WORD $0x014c; BYTE $0xca // addq %r9, %rdx
JMP LBB1_5

TEXT ·xvec_avx512_fht_inverse_kac_walk(SB), $0-16
TEXT ·fht_inv_kacs_walk_avx512(SB), $0-16
MOVQ data+0(FP), DI
MOVQ size+8(FP), SI
BYTE $0x55 // pushq %rbp
Expand Down Expand Up @@ -301,7 +301,7 @@ LBB2_6:
WORD $0xf8c5; BYTE $0x77 // vzeroupper
RET

TEXT ·xvec_avx512_fht_in_place(SB), $0-16
TEXT ·fht_inplace_avx512(SB), $0-16
MOVQ data+0(FP), DI
MOVQ size+8(FP), SI
BYTE $0x55 // pushq %rbp
Expand Down
8 changes: 4 additions & 4 deletions internal/ailego/math/fht_neon.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

8 changes: 4 additions & 4 deletions internal/ailego/math/fht_neon.s
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
// flags: -O3
// source: src/fht_neon.c

TEXT ·xvec_neon_fht_flip_signs(SB), $0-24
TEXT ·fht_flip_sign_neon(SB), $0-24
MOVD signs+0(FP), R0
MOVD data+8(FP), R1
MOVD size+16(FP), R2
Expand Down Expand Up @@ -82,7 +82,7 @@ LBB0_8:
WORD $0xbc287820 // str s0, [x1, x8, lsl #2]
B LBB0_7

TEXT ·xvec_neon_fht_kac_walk(SB), $0-16
TEXT ·fht_kacs_walk_neon(SB), $0-16
MOVD data+0(FP), R0
MOVD size+8(FP), R1
WORD $0x8b41fc28 // add x8, x1, x1, lsr #63
Expand Down Expand Up @@ -175,7 +175,7 @@ LBB1_10:
LBB1_11:
RET

TEXT ·xvec_neon_fht_inverse_kac_walk(SB), $0-16
TEXT ·fht_inv_kacs_walk_neon(SB), $0-16
MOVD data+0(FP), R0
MOVD size+8(FP), R1
WORD $0x8b41fc28 // add x8, x1, x1, lsr #63
Expand Down Expand Up @@ -269,7 +269,7 @@ LBB2_10:
LBB2_11:
RET

TEXT ·xvec_neon_fht_in_place(SB), $0-16
TEXT ·fht_inplace_neon(SB), $0-16
MOVD data+0(FP), R0
MOVD size+8(FP), R1
WORD $0xf100083f // cmp x1, #2
Expand Down
8 changes: 4 additions & 4 deletions internal/ailego/math/src/fht_avx2.c
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
#include <immintrin.h>
#include <stdint.h>

void xvec_avx2_fht_flip_signs(uint8_t *signs, float *data, int64_t size) {
void fht_flip_sign_avx2(uint8_t *signs, float *data, int64_t size) {
int64_t simd_end = size & ~31LL;
for (int64_t index = 0; index < simd_end; index += 32) {
uint32_t bits;
Expand All @@ -39,7 +39,7 @@ void xvec_avx2_fht_flip_signs(uint8_t *signs, float *data, int64_t size) {
}
}

void xvec_avx2_fht_kac_walk(float *data, int64_t size) {
void fht_kacs_walk_avx2(float *data, int64_t size) {
int64_t half = size / 2;
int64_t base = size % 2;
int64_t offset = base + half;
Expand All @@ -59,7 +59,7 @@ void xvec_avx2_fht_kac_walk(float *data, int64_t size) {

}

void xvec_avx2_fht_inverse_kac_walk(float *data, int64_t size) {
void fht_inv_kacs_walk_avx2(float *data, int64_t size) {
int64_t half = size / 2;
int64_t base = size % 2;
int64_t offset = base + half;
Expand All @@ -80,7 +80,7 @@ void xvec_avx2_fht_inverse_kac_walk(float *data, int64_t size) {
}
}

void xvec_avx2_fht_in_place(float *data, int64_t size) {
void fht_inplace_avx2(float *data, int64_t size) {
for (int64_t width = 1; width < size; width <<= 1) {
int64_t step = width << 1;
int64_t simd_end = width & ~7LL;
Expand Down
8 changes: 4 additions & 4 deletions internal/ailego/math/src/fht_avx512.c
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
#include <immintrin.h>
#include <stdint.h>

void xvec_avx512_fht_flip_signs(uint8_t *signs, float *data, int64_t size) {
void fht_flip_sign_avx512(uint8_t *signs, float *data, int64_t size) {
int64_t simd_end = size & ~63LL;
volatile int32_t sign = (int32_t)0x80000000u;
const __m512 sign_bit = _mm512_castsi512_ps(_mm512_set1_epi32(sign));
Expand All @@ -36,7 +36,7 @@ void xvec_avx512_fht_flip_signs(uint8_t *signs, float *data, int64_t size) {
}
}

void xvec_avx512_fht_kac_walk(float *data, int64_t size) {
void fht_kacs_walk_avx512(float *data, int64_t size) {
int64_t half = size / 2;
int64_t base = size % 2;
int64_t offset = base + half;
Expand All @@ -56,7 +56,7 @@ void xvec_avx512_fht_kac_walk(float *data, int64_t size) {

}

void xvec_avx512_fht_inverse_kac_walk(float *data, int64_t size) {
void fht_inv_kacs_walk_avx512(float *data, int64_t size) {
int64_t half = size / 2;
int64_t base = size % 2;
int64_t offset = base + half;
Expand All @@ -77,7 +77,7 @@ void xvec_avx512_fht_inverse_kac_walk(float *data, int64_t size) {
}
}

void xvec_avx512_fht_in_place(float *data, int64_t size) {
void fht_inplace_avx512(float *data, int64_t size) {
for (int64_t width = 1; width < size; width <<= 1) {
int64_t step = width << 1;
int64_t simd_end = width & ~15LL;
Expand Down
8 changes: 4 additions & 4 deletions internal/ailego/math/src/fht_neon.c
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
#include <arm_neon.h>
#include <stdint.h>

void xvec_neon_fht_flip_signs(uint8_t *signs, float *data, int64_t size) {
void fht_flip_sign_neon(uint8_t *signs, float *data, int64_t size) {
int64_t simd_end = size & ~3LL;
for (int64_t index = 0; index < simd_end; index += 4) {
uint16_t bits = signs[index / 8];
Expand All @@ -38,7 +38,7 @@ void xvec_neon_fht_flip_signs(uint8_t *signs, float *data, int64_t size) {
}
}

void xvec_neon_fht_kac_walk(float *data, int64_t size) {
void fht_kacs_walk_neon(float *data, int64_t size) {
int64_t half = size / 2;
int64_t base = size % 2;
int64_t offset = base + half;
Expand All @@ -58,7 +58,7 @@ void xvec_neon_fht_kac_walk(float *data, int64_t size) {

}

void xvec_neon_fht_inverse_kac_walk(float *data, int64_t size) {
void fht_inv_kacs_walk_neon(float *data, int64_t size) {
int64_t half = size / 2;
int64_t base = size % 2;
int64_t offset = base + half;
Expand All @@ -79,7 +79,7 @@ void xvec_neon_fht_inverse_kac_walk(float *data, int64_t size) {
}
}

void xvec_neon_fht_in_place(float *data, int64_t size) {
void fht_inplace_neon(float *data, int64_t size) {
for (int64_t width = 1; width < size; width <<= 1) {
int64_t step = width << 1;
int64_t simd_end = width & ~3LL;
Expand Down
Loading