diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 86aae39..60e8074 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -98,6 +98,6 @@ jobs: sudo apt-get update sudo apt-get install -y qemu-user - name: Test RISC-V RVV - run: make -C internal/floats test-rvv + run: make -C internal/ailego/math test-rvv - name: Test LoongArch LASX - run: make -C internal/floats test-lasx + run: make -C internal/ailego/math test-lasx diff --git a/go.mod b/go.mod index fd8b271..1e9facf 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/gorse-io/xvec -go 1.27 +go 1.27.0 require ( github.com/RoaringBitmap/roaring/v2 v2.24.0 @@ -36,7 +36,7 @@ require ( github.com/golang/protobuf v1.5.3 // indirect github.com/golang/snappy v0.0.5-0.20231225225746-43d5d4cd4e0e // indirect github.com/google/uuid v1.6.0 // indirect - github.com/gorse-io/goat v0.2.1-0.20260618151728-201cbcf325ad // indirect + github.com/gorse-io/goat v0.2.2-0.20260901023055-3025b7527c27 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/klauspost/asmfmt v1.3.2 // indirect github.com/klauspost/compress v1.17.11 // indirect diff --git a/go.sum b/go.sum index 95b8f57..feada0c 100644 --- a/go.sum +++ b/go.sum @@ -72,8 +72,8 @@ github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/gorse-io/goat v0.2.1-0.20260618151728-201cbcf325ad h1:HESEie1ivHg8yI9Qnej4Cu+AP8w8gQ9gqpYcnV5zC7Q= -github.com/gorse-io/goat v0.2.1-0.20260618151728-201cbcf325ad/go.mod h1:NjwKvGIyhFveMjnUdYrngJYvYLqK0aK7Qdp7QJWkBLo= +github.com/gorse-io/goat v0.2.2-0.20260901023055-3025b7527c27 h1:7Ui/Xhc9+zMN6cR8A5GVFffssiCdDCVGB0WBrlLeMBw= +github.com/gorse-io/goat v0.2.2-0.20260901023055-3025b7527c27/go.mod h1:gRNfHg2VIBM1rLZXr6LtS2BX+qVI8DKckQC9LBESUMk= github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM= github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= diff --git a/internal/ailego/math/Makefile b/internal/ailego/math/Makefile index 887dde3..54ad180 100644 --- a/internal/ailego/math/Makefile +++ b/internal/ailego/math/Makefile @@ -2,28 +2,97 @@ GO ?= go GOAT ?= $(GO) tool goat GOAT_TARGET_AMD64 ?= amd64 GOAT_TARGET_ARM64 ?= arm64 +GOAT_TARGET_RVV ?= riscv64 +GOAT_TARGET_LASX ?= loong64 GOAT_OUTPUT_DIR ?= .goat/mathutil MV ?= mv +CLANG_RESOURCE_DIR ?= $(shell clang -print-resource-dir) +GOAT_RVV_INCLUDE ?= $(CLANG_RESOURCE_DIR)/include +LASX_OBJDUMP ?= loongarch64-linux-gnu-objdump +RISCV64_QEMU ?= qemu-riscv64 +RISCV64_QEMU_FLAGS ?= -cpu max,v=true +LOONG64_QEMU ?= qemu-loongarch64 +LOONG64_QEMU_FLAGS ?= -cpu max +TEST_RVV_BINARY ?= .test-rvv.riscv64.test +TEST_LASX_BINARY ?= .test-lasx.loong64.test -.PHONY: generate avx2 avx512 neon clean +.PHONY: generate avx avx2 avx512 neon rvv lasx fht-avx2 fht-avx512 fht-neon distance-avx distance-avx512 distance-neon distance-rvv distance-lasx clean test-rvv test-lasx -generate: avx2 avx512 neon clean +generate: fht-avx2 fht-avx512 fht-neon distance-avx distance-avx512 distance-neon distance-rvv distance-lasx clean -avx2: +avx: distance-avx + +avx2: fht-avx2 + +avx512: fht-avx512 distance-avx512 + +neon: fht-neon distance-neon + +rvv: distance-rvv + +lasx: distance-lasx + +fht-avx2: mkdir -p $(GOAT_OUTPUT_DIR) $(GOAT) src/fht_avx2.c --target $(GOAT_TARGET_AMD64) -O3 -mavx2 -o $(GOAT_OUTPUT_DIR) $(MV) $(GOAT_OUTPUT_DIR)/fht_avx2.go $(GOAT_OUTPUT_DIR)/fht_avx2.s . -avx512: +fht-avx512: mkdir -p $(GOAT_OUTPUT_DIR) $(GOAT) src/fht_avx512.c --target $(GOAT_TARGET_AMD64) -O3 -mavx512f -mavx512dq -o $(GOAT_OUTPUT_DIR) $(MV) $(GOAT_OUTPUT_DIR)/fht_avx512.go $(GOAT_OUTPUT_DIR)/fht_avx512.s . -neon: +fht-neon: mkdir -p $(GOAT_OUTPUT_DIR) $(GOAT) src/fht_neon.c --target $(GOAT_TARGET_ARM64) -O3 -o $(GOAT_OUTPUT_DIR) $(MV) $(GOAT_OUTPUT_DIR)/fht_neon.go $(GOAT_OUTPUT_DIR)/fht_neon.s . +distance-avx: + $(RM) -r $(GOAT_OUTPUT_DIR) + mkdir -p $(GOAT_OUTPUT_DIR) + $(GOAT) src/euclidean_distance_matrix_fp32_sse.c --target $(GOAT_TARGET_AMD64) -O3 -msse2 -o $(GOAT_OUTPUT_DIR) + $(GOAT) src/inner_product_matrix_fp32_sse.c --target $(GOAT_TARGET_AMD64) -O3 -msse2 -o $(GOAT_OUTPUT_DIR) + $(GOAT) src/mips_euclidean_distance_matrix_fp32_sse.c --target $(GOAT_TARGET_AMD64) -O3 -msse2 -o $(GOAT_OUTPUT_DIR) + $(GOAT) src/euclidean_distance_matrix_fp32_avx.c --target $(GOAT_TARGET_AMD64) -O3 -mavx -o $(GOAT_OUTPUT_DIR) + $(GOAT) src/inner_product_matrix_fp32_avx.c --target $(GOAT_TARGET_AMD64) -O3 -mavx -o $(GOAT_OUTPUT_DIR) + $(GOAT) src/mips_euclidean_distance_matrix_fp32_avx.c --target $(GOAT_TARGET_AMD64) -O3 -mavx -o $(GOAT_OUTPUT_DIR) + $(GOAT) src/distance_utility_batch_avx.c --target $(GOAT_TARGET_AMD64) -O3 -mavx -o $(GOAT_OUTPUT_DIR) + $(MV) $(GOAT_OUTPUT_DIR)/*.go $(GOAT_OUTPUT_DIR)/*.s . + +distance-avx512: + $(RM) -r $(GOAT_OUTPUT_DIR) + mkdir -p $(GOAT_OUTPUT_DIR) + $(GOAT) src/euclidean_distance_matrix_fp32_avx512.c --target $(GOAT_TARGET_AMD64) -O3 -mavx -mfma -mavx512f -o $(GOAT_OUTPUT_DIR) + $(GOAT) src/inner_product_matrix_fp32_avx512.c --target $(GOAT_TARGET_AMD64) -O3 -mavx -mfma -mavx512f -o $(GOAT_OUTPUT_DIR) + $(GOAT) src/mips_euclidean_distance_matrix_fp32_avx512.c --target $(GOAT_TARGET_AMD64) -O3 -mavx -mfma -mavx512f -o $(GOAT_OUTPUT_DIR) + $(MV) $(GOAT_OUTPUT_DIR)/*.go $(GOAT_OUTPUT_DIR)/*.s . + +distance-neon: + $(RM) -r $(GOAT_OUTPUT_DIR) + mkdir -p $(GOAT_OUTPUT_DIR) + $(GOAT) src/euclidean_distance_matrix_fp32_neon.c --target $(GOAT_TARGET_ARM64) -O3 -o $(GOAT_OUTPUT_DIR) + $(GOAT) src/inner_product_matrix_fp32_neon.c --target $(GOAT_TARGET_ARM64) -O3 -o $(GOAT_OUTPUT_DIR) + $(GOAT) src/mips_euclidean_distance_matrix_fp32_neon.c --target $(GOAT_TARGET_ARM64) -O3 -o $(GOAT_OUTPUT_DIR) + $(MV) $(GOAT_OUTPUT_DIR)/*.go $(GOAT_OUTPUT_DIR)/*.s . + +distance-rvv: + $(GOAT) src/distance_utility_rvv.c --target $(GOAT_TARGET_RVV) -O3 -march=rv64imafdv -e=-isystem -e=$(GOAT_RVV_INCLUDE) + +distance-lasx: + OBJDUMP=$(LASX_OBJDUMP) $(GOAT) src/distance_utility_lasx.c --target $(GOAT_TARGET_LASX) -O3 -mlasx + clean: $(RM) src/*.o src/*.s $(RM) -r .goat + +test-rvv: + @set -e; \ + trap '$(RM) ./$(TEST_RVV_BINARY)' EXIT; \ + GOOS=linux GOARCH=riscv64 CGO_ENABLED=0 $(GO) test -c -o $(TEST_RVV_BINARY) .; \ + $(RISCV64_QEMU) $(RISCV64_QEMU_FLAGS) ./$(TEST_RVV_BINARY) + +test-lasx: + @set -e; \ + trap '$(RM) ./$(TEST_LASX_BINARY)' EXIT; \ + GOOS=linux GOARCH=loong64 CGO_ENABLED=0 $(GO) test -c -o $(TEST_LASX_BINARY) .; \ + $(LOONG64_QEMU) $(LOONG64_QEMU_FLAGS) ./$(TEST_LASX_BINARY) diff --git a/internal/ailego/math/distance.go b/internal/ailego/math/distance.go index 870061c..371bcc2 100644 --- a/internal/ailego/math/distance.go +++ b/internal/ailego/math/distance.go @@ -17,8 +17,6 @@ package mathutil import ( "errors" "math" - - "github.com/gorse-io/xvec/internal/floats" ) var ( @@ -36,7 +34,7 @@ func L2Squared(left, right []float32) (float32, error) { if err := validateDenseFinite(left, right); err != nil { return 0, err } - return finiteScore(float64(floats.L2Squared(left, right))) + return finiteScore(float64(squaredEuclidean(left, right))) } // InnerProduct computes the dot-product similarity. Higher scores are better. @@ -47,7 +45,7 @@ func InnerProduct(left, right []float32) (float32, error) { if err := validateDenseFinite(left, right); err != nil { return 0, err } - return finiteScore(float64(floats.InnerProduct(left, right))) + return finiteScore(float64(innerProduct(left, right))) } // CosineDistance computes 1-cos(left,right). Lower scores are better. Two zero @@ -63,7 +61,7 @@ func CosineDistance(left, right []float32) (float32, error) { } func cosineDistance(left, right []float32) float32 { - inner, leftNorm, rightNorm := floats.DotNorms(left, right) + inner, leftNorm, rightNorm := dotNorms(left, right) if leftNorm == 0 && rightNorm == 0 { return 0 } @@ -91,7 +89,7 @@ func MIPSL2Squared(left, right []float32) (float32, error) { } func mipsL2Squared(left, right []float32) float32 { - inner, leftNorm, rightNorm := floats.DotNorms(left, right) + inner, leftNorm, rightNorm := dotNorms(left, right) denominator := max(leftNorm, rightNorm) if denominator == 0 { return 0 diff --git a/internal/ailego/math/distance_prevalidated.go b/internal/ailego/math/distance_prevalidated.go index 7b1f1d8..0b7f695 100644 --- a/internal/ailego/math/distance_prevalidated.go +++ b/internal/ailego/math/distance_prevalidated.go @@ -16,8 +16,6 @@ package mathutil import ( "math" - - "github.com/gorse-io/xvec/internal/floats" ) // DenseDistance computes a score for two already validated dense vectors. @@ -28,12 +26,12 @@ type DenseDistance func(left, right []float32) (float32, error) // its inputs. It is intended for index hot paths whose storage boundary has // already validated every vector. func L2SquaredPrevalidated(left, right []float32) (float32, error) { - return finiteScore(float64(floats.L2Squared(left, right))) + return finiteScore(float64(squaredEuclidean(left, right))) } // InnerProductPrevalidated computes inner product without validating inputs. func InnerProductPrevalidated(left, right []float32) (float32, error) { - return finiteScore(float64(floats.InnerProduct(left, right))) + return finiteScore(float64(innerProduct(left, right))) } // CosineDistancePrevalidated computes cosine distance without validating inputs. @@ -44,7 +42,7 @@ func CosineDistancePrevalidated(left, right []float32) (float32, error) { // L2MagnitudePrevalidated computes a vector magnitude without validating its // components. It is intended for indexes that cache norms at ingestion time. func L2MagnitudePrevalidated(vector []float32) (float32, error) { - norm := floats.InnerProduct(vector, vector) + norm := innerProduct(vector, vector) if norm < 0 { norm = 0 } @@ -61,7 +59,7 @@ func CosineDistanceWithMagnitudesPrevalidated(left, right []float32, leftMagnitu if leftMagnitude == 0 || rightMagnitude == 0 { return 1, nil } - cosine := floats.InnerProduct(left, right) / (leftMagnitude * rightMagnitude) + cosine := innerProduct(left, right) / (leftMagnitude * rightMagnitude) cosine = min(1, max(-1, cosine)) return finiteScore(float64(1 - cosine)) } @@ -73,7 +71,7 @@ func CosineDistances2WithMagnitudesPrevalidated( query, first, second []float32, queryMagnitude, firstMagnitude, secondMagnitude float32, ) (firstDistance, secondDistance float32, err error) { - firstProduct, secondProduct := floats.InnerProducts2(query, first, second) + firstProduct, secondProduct := innerProducts2(query, first, second) firstDistance, err = cosineDistanceFromProduct(firstProduct, queryMagnitude, firstMagnitude) if err != nil { return 0, 0, err @@ -92,7 +90,7 @@ func CosineDistances4WithMagnitudesPrevalidated( query, first, second, third, fourth []float32, queryMagnitude, firstMagnitude, secondMagnitude, thirdMagnitude, fourthMagnitude float32, ) (firstDistance, secondDistance, thirdDistance, fourthDistance float32, err error) { - firstProduct, secondProduct, thirdProduct, fourthProduct := floats.InnerProducts4(query, first, second, third, fourth) + firstProduct, secondProduct, thirdProduct, fourthProduct := innerProducts4(query, first, second, third, fourth) firstDistance, err = cosineDistanceFromProduct(firstProduct, queryMagnitude, firstMagnitude) if err != nil { return 0, 0, 0, 0, err diff --git a/internal/floats/floats.go b/internal/ailego/math/distance_utility.go similarity index 76% rename from internal/floats/floats.go rename to internal/ailego/math/distance_utility.go index 52c2242..4911cd0 100644 --- a/internal/floats/floats.go +++ b/internal/ailego/math/distance_utility.go @@ -12,9 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. -// Package floats provides allocation-free float32 vector kernels. Callers must -// pass equal, non-empty slices; validation belongs at the API boundary. -package floats +// The distance utility functions provide allocation-free float32 vector kernels. +// Callers must pass equal, non-empty slices; validation belongs at the API boundary. +package mathutil type binaryKernel func(left, right []float32) float32 type batch2Kernel func(query, first, second []float32) (firstProduct, secondProduct float32) @@ -28,42 +28,42 @@ var kernels = struct { dot4 batch4Kernel products productsKernel }{ - l2: l2SquaredScalar, + l2: squaredEuclideanScalar, dot: innerProductScalar, dot2: innerProducts2Scalar, dot4: innerProducts4Scalar, products: dotNormsScalar, } -// L2Squared returns the squared Euclidean distance between left and right. -func L2Squared(left, right []float32) float32 { +// squaredEuclidean returns the squared Euclidean distance between left and right. +func squaredEuclidean(left, right []float32) float32 { return kernels.l2(left, right) } -// InnerProduct returns the dot product of left and right. -func InnerProduct(left, right []float32) float32 { +// innerProduct returns the dot product of left and right. +func innerProduct(left, right []float32) float32 { return kernels.dot(left, right) } -// InnerProducts2 computes the dot product of one query with two candidates in +// innerProducts2 computes the dot product of one query with two candidates in // one pass. SIMD implementations reuse each loaded query block for both // candidates, which is the dominant HNSW one-to-many scoring pattern. -func InnerProducts2(query, first, second []float32) (firstProduct, secondProduct float32) { +func innerProducts2(query, first, second []float32) (firstProduct, secondProduct float32) { return kernels.dot2(query, first, second) } -// InnerProducts4 computes the dot product of one query with four candidates +// innerProducts4 computes the dot product of one query with four candidates // in one pass, amortizing each query load across four independent products. -func InnerProducts4(query, first, second, third, fourth []float32) (firstProduct, secondProduct, thirdProduct, fourthProduct float32) { +func innerProducts4(query, first, second, third, fourth []float32) (firstProduct, secondProduct, thirdProduct, fourthProduct float32) { return kernels.dot4(query, first, second, third, fourth) } -// DotNorms computes the dot product and both squared norms in one pass. -func DotNorms(left, right []float32) (dot, leftNorm, rightNorm float32) { +// dotNorms computes the dot product and both squared norms in one pass. +func dotNorms(left, right []float32) (dot, leftNorm, rightNorm float32) { return kernels.products(left, right) } -func l2SquaredScalar(left, right []float32) (sum float32) { +func squaredEuclideanScalar(left, right []float32) (sum float32) { for index, leftValue := range left { difference := leftValue - right[index] sum += difference * difference diff --git a/internal/floats/floats_amd64.go b/internal/ailego/math/distance_utility_amd64.go similarity index 58% rename from internal/floats/floats_amd64.go rename to internal/ailego/math/distance_utility_amd64.go index 89700ef..95cdeb8 100644 --- a/internal/floats/floats_amd64.go +++ b/internal/ailego/math/distance_utility_amd64.go @@ -14,7 +14,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package floats +package mathutil import ( "unsafe" @@ -22,43 +22,55 @@ import ( "golang.org/x/sys/cpu" ) -//go:generate go tool goat src/floats_avx.c -O3 -mavx -//go:generate go tool goat src/floats_batch_avx.c -O3 -mavx -//go:generate go tool goat src/floats_avx512.c -O3 -mavx -mfma -mavx512f +//go:generate make distance-avx +//go:generate make distance-avx512 func init() { - switch { - case cpu.X86.HasAVX && cpu.X86.HasFMA && cpu.X86.HasAVX512F: - kernels.l2 = l2SquaredAVX512 - kernels.dot = innerProductAVX512 - kernels.products = dotNormsAVX512 - case cpu.X86.HasAVX: - kernels.l2 = l2SquaredAVX + kernels.l2 = squaredEuclideanSSE + kernels.dot = innerProductSSE + kernels.products = dotNormsSSE + if cpu.X86.HasAVX { + kernels.l2 = squaredEuclideanAVX kernels.dot = innerProductAVX kernels.products = dotNormsAVX - } - if cpu.X86.HasAVX { kernels.dot2 = innerProducts2AVXBatch kernels.dot4 = innerProducts4AVXBatch } + if cpu.X86.HasAVX && cpu.X86.HasFMA && cpu.X86.HasAVX512F { + kernels.l2 = squaredEuclideanAVX512 + kernels.dot = innerProductAVX512 + kernels.products = dotNormsAVX512 + } +} + +func squaredEuclideanSSE(left, right []float32) float32 { + return squared_euclidean_distance_fp32_sse(unsafe.Pointer(&left[0]), unsafe.Pointer(&right[0]), int64(len(left))) +} + +func innerProductSSE(left, right []float32) float32 { + return inner_product_fp32_sse(unsafe.Pointer(&left[0]), unsafe.Pointer(&right[0]), int64(len(left))) +} + +func dotNormsSSE(left, right []float32) (dot, leftNorm, rightNorm float32) { + dot = inner_product_and_squared_norm_fp32_sse( + unsafe.Pointer(&left[0]), unsafe.Pointer(&right[0]), int64(len(left)), + unsafe.Pointer(&leftNorm), unsafe.Pointer(&rightNorm), + ) + return } -func l2SquaredAVX(left, right []float32) float32 { +func squaredEuclideanAVX(left, right []float32) float32 { if len(left) < 8 { - return l2SquaredScalar(left, right) + return squaredEuclideanSSE(left, right) } - var result float32 - xvec_avx_l2_squared(unsafe.Pointer(&left[0]), unsafe.Pointer(&right[0]), int64(len(left)), unsafe.Pointer(&result)) - return result + return squared_euclidean_distance_fp32_avx(unsafe.Pointer(&left[0]), unsafe.Pointer(&right[0]), int64(len(left))) } func innerProductAVX(left, right []float32) float32 { if len(left) < 8 { - return innerProductScalar(left, right) + return innerProductSSE(left, right) } - var result float32 - xvec_avx_inner_product(unsafe.Pointer(&left[0]), unsafe.Pointer(&right[0]), int64(len(left)), unsafe.Pointer(&result)) - return result + return inner_product_fp32_avx(unsafe.Pointer(&left[0]), unsafe.Pointer(&right[0]), int64(len(left))) } func innerProducts2AVXBatch(query, first, second []float32) (firstProduct, secondProduct float32) { @@ -87,40 +99,36 @@ func innerProducts4AVXBatch(query, first, second, third, fourth []float32) (firs func dotNormsAVX(left, right []float32) (dot, leftNorm, rightNorm float32) { if len(left) < 8 { - return dotNormsScalar(left, right) + return dotNormsSSE(left, right) } - xvec_avx_dot_norms( + dot = inner_product_and_squared_norm_fp32_avx( unsafe.Pointer(&left[0]), unsafe.Pointer(&right[0]), int64(len(left)), - unsafe.Pointer(&dot), unsafe.Pointer(&leftNorm), unsafe.Pointer(&rightNorm), + unsafe.Pointer(&leftNorm), unsafe.Pointer(&rightNorm), ) return } -func l2SquaredAVX512(left, right []float32) float32 { +func squaredEuclideanAVX512(left, right []float32) float32 { if len(left) < 16 { - return l2SquaredAVX(left, right) + return squaredEuclideanAVX(left, right) } - var result float32 - xvec_avx512_l2_squared(unsafe.Pointer(&left[0]), unsafe.Pointer(&right[0]), int64(len(left)), unsafe.Pointer(&result)) - return result + return squared_euclidean_distance_fp32_avx512(unsafe.Pointer(&left[0]), unsafe.Pointer(&right[0]), int64(len(left))) } func innerProductAVX512(left, right []float32) float32 { if len(left) < 16 { return innerProductAVX(left, right) } - var result float32 - xvec_avx512_inner_product(unsafe.Pointer(&left[0]), unsafe.Pointer(&right[0]), int64(len(left)), unsafe.Pointer(&result)) - return result + return inner_product_fp32_avx512(unsafe.Pointer(&left[0]), unsafe.Pointer(&right[0]), int64(len(left))) } func dotNormsAVX512(left, right []float32) (dot, leftNorm, rightNorm float32) { if len(left) < 16 { return dotNormsAVX(left, right) } - xvec_avx512_dot_norms( + dot = inner_product_and_squared_norm_fp32_avx512( unsafe.Pointer(&left[0]), unsafe.Pointer(&right[0]), int64(len(left)), - unsafe.Pointer(&dot), unsafe.Pointer(&leftNorm), unsafe.Pointer(&rightNorm), + unsafe.Pointer(&leftNorm), unsafe.Pointer(&rightNorm), ) return } diff --git a/internal/floats/floats_amd64_test.go b/internal/ailego/math/distance_utility_amd64_test.go similarity index 75% rename from internal/floats/floats_amd64_test.go rename to internal/ailego/math/distance_utility_amd64_test.go index c0f3f43..998360a 100644 --- a/internal/floats/floats_amd64_test.go +++ b/internal/ailego/math/distance_utility_amd64_test.go @@ -14,7 +14,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package floats +package mathutil import ( "testing" @@ -22,16 +22,20 @@ import ( "golang.org/x/sys/cpu" ) +func TestSSEDistanceKernels(t *testing.T) { + testArchitectureKernels(t, squaredEuclideanSSE, innerProductSSE, dotNormsSSE) +} + func TestAVXDistanceKernels(t *testing.T) { if !cpu.X86.HasAVX { t.Skip("AVX is not supported by this CPU") } - testArchitectureKernels(t, l2SquaredAVX, innerProductAVX, dotNormsAVX) + testArchitectureKernels(t, squaredEuclideanAVX, innerProductAVX, dotNormsAVX) } func TestAVX512DistanceKernels(t *testing.T) { if !cpu.X86.HasAVX || !cpu.X86.HasFMA || !cpu.X86.HasAVX512F { t.Skip("AVX-512/FMA is not supported by this CPU") } - testArchitectureKernels(t, l2SquaredAVX512, innerProductAVX512, dotNormsAVX512) + testArchitectureKernels(t, squaredEuclideanAVX512, innerProductAVX512, dotNormsAVX512) } diff --git a/internal/floats/floats_arm64.go b/internal/ailego/math/distance_utility_arm64.go similarity index 70% rename from internal/floats/floats_arm64.go rename to internal/ailego/math/distance_utility_arm64.go index 15fd6f2..bf07686 100644 --- a/internal/floats/floats_arm64.go +++ b/internal/ailego/math/distance_utility_arm64.go @@ -14,7 +14,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package floats +package mathutil import ( "unsafe" @@ -22,39 +22,35 @@ import ( "golang.org/x/sys/cpu" ) -//go:generate go tool goat src/floats_neon.c -O3 +//go:generate make distance-neon func init() { if cpu.ARM64.HasASIMD { - kernels.l2 = l2SquaredNEON + kernels.l2 = squaredEuclideanNEON kernels.dot = innerProductNEON kernels.products = dotNormsNEON } } -func l2SquaredNEON(left, right []float32) float32 { +func squaredEuclideanNEON(left, right []float32) float32 { if len(left) < 4 { - return l2SquaredScalar(left, right) + return squaredEuclideanScalar(left, right) } - var result float32 - xvec_neon_l2_squared(unsafe.Pointer(&left[0]), unsafe.Pointer(&right[0]), int64(len(left)), unsafe.Pointer(&result)) - return result + return squared_euclidean_distance_fp32_neon(unsafe.Pointer(&left[0]), unsafe.Pointer(&right[0]), int64(len(left))) } func innerProductNEON(left, right []float32) float32 { if len(left) < 4 { return innerProductScalar(left, right) } - var result float32 - xvec_neon_inner_product(unsafe.Pointer(&left[0]), unsafe.Pointer(&right[0]), int64(len(left)), unsafe.Pointer(&result)) - return result + return inner_product_fp32_neon(unsafe.Pointer(&left[0]), unsafe.Pointer(&right[0]), int64(len(left))) } func dotNormsNEON(left, right []float32) (dot, leftNorm, rightNorm float32) { if len(left) < 4 { return dotNormsScalar(left, right) } - xvec_neon_dot_norms( + inner_product_and_squared_norm_fp32_neon( unsafe.Pointer(&left[0]), unsafe.Pointer(&right[0]), int64(len(left)), unsafe.Pointer(&dot), unsafe.Pointer(&leftNorm), unsafe.Pointer(&rightNorm), ) diff --git a/internal/floats/floats_arm64_test.go b/internal/ailego/math/distance_utility_arm64_test.go similarity index 89% rename from internal/floats/floats_arm64_test.go rename to internal/ailego/math/distance_utility_arm64_test.go index c886ba1..c34e416 100644 --- a/internal/floats/floats_arm64_test.go +++ b/internal/ailego/math/distance_utility_arm64_test.go @@ -14,7 +14,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package floats +package mathutil import ( "testing" @@ -26,5 +26,5 @@ func TestNEONDistanceKernels(t *testing.T) { if !cpu.ARM64.HasASIMD { t.Skip("NEON/ASIMD is not supported by this CPU") } - testArchitectureKernels(t, l2SquaredNEON, innerProductNEON, dotNormsNEON) + testArchitectureKernels(t, squaredEuclideanNEON, innerProductNEON, dotNormsNEON) } diff --git a/internal/ailego/math/distance_utility_batch_avx.go b/internal/ailego/math/distance_utility_batch_avx.go new file mode 100644 index 0000000..5006d5f --- /dev/null +++ b/internal/ailego/math/distance_utility_batch_avx.go @@ -0,0 +1,17 @@ +//go:build !noasm && amd64 +// Code generated by GoAT. DO NOT EDIT. +// versions: +// clang 21.1.8 (6ubuntu1) +// objdump 2.46 +// flags: -mavx -O3 +// source: src/distance_utility_batch_avx.c + +package mathutil + +import "unsafe" + +//go:noescape +func xvec_avx_batch_inner_products2(query, first, second unsafe.Pointer, size int64, first_output, second_output unsafe.Pointer) + +//go:noescape +func xvec_avx_batch_inner_products4(query, first, second, third, fourth unsafe.Pointer, size int64, first_output, second_output, third_output, fourth_output unsafe.Pointer) diff --git a/internal/ailego/math/distance_utility_batch_avx.s b/internal/ailego/math/distance_utility_batch_avx.s new file mode 100644 index 0000000..0e3218d --- /dev/null +++ b/internal/ailego/math/distance_utility_batch_avx.s @@ -0,0 +1,471 @@ +//go:build !noasm && amd64 +// Code generated by GoAT. DO NOT EDIT. +// versions: +// clang 21.1.8 (6ubuntu1) +// objdump 2.46 +// flags: -mavx -O3 +// source: src/distance_utility_batch_avx.c + +TEXT ·xvec_avx_batch_inner_products2(SB), $0-48 + MOVQ query+0(FP), DI + MOVQ first+8(FP), SI + MOVQ second+16(FP), DX + MOVQ size+24(FP), CX + MOVQ first_output+32(FP), R8 + MOVQ second_output+40(FP), R9 + BYTE $0x55 // pushq %rbp + WORD $0x8948; BYTE $0xe5 // movq %rsp, %rbp + WORD $0x5641 // pushq %r14 + BYTE $0x53 // pushq %rbx + LONG $0xf8e48348 // andq $-8, %rsp + LONG $0x07518d4c // leaq 7(%rcx), %r10 + WORD $0x8548; BYTE $0xc9 // testq %rcx, %rcx + LONG $0xd1490f4c // cmovnsq %rcx, %r10 + WORD $0x894d; BYTE $0xd3 // movq %r10, %r11 + LONG $0xf8e38349 // andq $-8, %r11 + WORD $0x8948; BYTE $0xc8 // movq %rcx, %rax + WORD $0x294c; BYTE $0xd8 // subq %r11, %rax + LONG $0xc957f0c5 // vxorps %xmm1, %xmm1, %xmm1 + LONG $0x08f98348 // cmpq $8, %rcx + JL LBB0_1 + LONG $0x03fac149 // sarq $3, %r10 + QUAD $0xfffffffffff8bb49; WORD $0x7fff // movabsq $9223372036854775800, %r11 # imm = 0x7FFFFFFFFFFFFFF8 + WORD $0x214c; BYTE $0xd9 // andq %r11, %rcx + LONG $0x08f98348 // cmpq $8, %rcx + JNE LBB0_9 + LONG $0xc057f8c5 // vxorps %xmm0, %xmm0, %xmm0 + WORD $0x8948; BYTE $0xf9 // movq %rdi, %rcx + WORD $0x8949; BYTE $0xf3 // movq %rsi, %r11 + WORD $0x8948; BYTE $0xd3 // movq %rdx, %rbx + JMP LBB0_4 + +LBB0_1: + LONG $0xc057f8c5 // vxorps %xmm0, %xmm0, %xmm0 + JMP LBB0_7 + +LBB0_9: + QUAD $0xfffffffffffebe49; WORD $0x0fff // movabsq $1152921504606846974, %r14 # imm = 0xFFFFFFFFFFFFFFE + WORD $0x214d; BYTE $0xd6 // andq %r10, %r14 + LONG $0xc057f8c5 // vxorps %xmm0, %xmm0, %xmm0 + WORD $0x8948; BYTE $0xf9 // movq %rdi, %rcx + WORD $0x8949; BYTE $0xf3 // movq %rsi, %r11 + WORD $0x8948; BYTE $0xd3 // movq %rdx, %rbx + +LBB0_10: + LONG $0x1110fcc5 // vmovups (%rcx), %ymm2 + LONG $0x5910fcc5; BYTE $0x20 // vmovups 32(%rcx), %ymm3 + LONG $0x596cc1c4; BYTE $0x23 // vmulps (%r11), %ymm2, %ymm4 + LONG $0x1359ecc5 // vmulps (%rbx), %ymm2, %ymm2 + LONG $0xcc58f4c5 // vaddps %ymm4, %ymm1, %ymm1 + LONG $0xc258fcc5 // vaddps %ymm2, %ymm0, %ymm0 + LONG $0x5964c1c4; WORD $0x2053 // vmulps 32(%r11), %ymm3, %ymm2 + LONG $0xca58f4c5 // vaddps %ymm2, %ymm1, %ymm1 + LONG $0x5359e4c5; BYTE $0x20 // vmulps 32(%rbx), %ymm3, %ymm2 + LONG $0xc258fcc5 // vaddps %ymm2, %ymm0, %ymm0 + LONG $0x40c18348 // addq $64, %rcx + LONG $0x40c38349 // addq $64, %r11 + LONG $0x40c38348 // addq $64, %rbx + LONG $0xfec68349 // addq $-2, %r14 + JNE LBB0_10 + +LBB0_4: + LONG $0x01c2f641 // testb $1, %r10b + JE LBB0_6 + LONG $0x1110fcc5 // vmovups (%rcx), %ymm2 + LONG $0x596cc1c4; BYTE $0x1b // vmulps (%r11), %ymm2, %ymm3 + LONG $0xcb58f4c5 // vaddps %ymm3, %ymm1, %ymm1 + LONG $0x1359ecc5 // vmulps (%rbx), %ymm2, %ymm2 + LONG $0xc258fcc5 // vaddps %ymm2, %ymm0, %ymm0 + +LBB0_6: + LONG $0x05e2c149 // shlq $5, %r10 + WORD $0x014c; BYTE $0xd7 // addq %r10, %rdi + WORD $0x014c; BYTE $0xd6 // addq %r10, %rsi + WORD $0x014c; BYTE $0xd2 // addq %r10, %rdx + +LBB0_7: + LONG $0xd257e8c5 // vxorps %xmm2, %xmm2, %xmm2 + LONG $0xda58f2c5 // vaddss %xmm2, %xmm1, %xmm3 + LONG $0xe116fac5 // vmovshdup %xmm1, %xmm4 # xmm4 = xmm1[1,1,3,3] + LONG $0xdb58dac5 // vaddss %xmm3, %xmm4, %xmm3 + LONG $0xe1c6f1c5; BYTE $0x01 // vshufpd $1, %xmm1, %xmm1, %xmm4 # xmm4 = xmm1[1,0] + LONG $0xdb58dac5 // vaddss %xmm3, %xmm4, %xmm3 + LONG $0xe1c6f0c5; BYTE $0xff // vshufps $255, %xmm1, %xmm1, %xmm4 # xmm4 = xmm1[3,3,3,3] + LONG $0xdb58dac5 // vaddss %xmm3, %xmm4, %xmm3 + LONG $0x197de3c4; WORD $0x01c9 // vextractf128 $1, %ymm1, %xmm1 + LONG $0xdb58f2c5 // vaddss %xmm3, %xmm1, %xmm3 + LONG $0xe116fac5 // vmovshdup %xmm1, %xmm4 # xmm4 = xmm1[1,1,3,3] + LONG $0xdb58dac5 // vaddss %xmm3, %xmm4, %xmm3 + LONG $0xe1c6f1c5; BYTE $0x01 // vshufpd $1, %xmm1, %xmm1, %xmm4 # xmm4 = xmm1[1,0] + LONG $0xdb58dac5 // vaddss %xmm3, %xmm4, %xmm3 + LONG $0xc9c6f0c5; BYTE $0xff // vshufps $255, %xmm1, %xmm1, %xmm1 # xmm1 = xmm1[3,3,3,3] + LONG $0xcb58f2c5 // vaddss %xmm3, %xmm1, %xmm1 + LONG $0x117ac1c4; BYTE $0x08 // vmovss %xmm1, (%r8) + LONG $0xca58fac5 // vaddss %xmm2, %xmm0, %xmm1 + LONG $0xd016fac5 // vmovshdup %xmm0, %xmm2 # xmm2 = xmm0[1,1,3,3] + LONG $0xc958eac5 // vaddss %xmm1, %xmm2, %xmm1 + LONG $0xd0c6f9c5; BYTE $0x01 // vshufpd $1, %xmm0, %xmm0, %xmm2 # xmm2 = xmm0[1,0] + LONG $0xc958eac5 // vaddss %xmm1, %xmm2, %xmm1 + LONG $0xd0c6f8c5; BYTE $0xff // vshufps $255, %xmm0, %xmm0, %xmm2 # xmm2 = xmm0[3,3,3,3] + LONG $0xc958eac5 // vaddss %xmm1, %xmm2, %xmm1 + LONG $0x197de3c4; WORD $0x01c0 // vextractf128 $1, %ymm0, %xmm0 + LONG $0xc958fac5 // vaddss %xmm1, %xmm0, %xmm1 + LONG $0xd016fac5 // vmovshdup %xmm0, %xmm2 # xmm2 = xmm0[1,1,3,3] + LONG $0xc958eac5 // vaddss %xmm1, %xmm2, %xmm1 + LONG $0xd0c6f9c5; BYTE $0x01 // vshufpd $1, %xmm0, %xmm0, %xmm2 # xmm2 = xmm0[1,0] + LONG $0xc958eac5 // vaddss %xmm1, %xmm2, %xmm1 + LONG $0xc0c6f8c5; BYTE $0xff // vshufps $255, %xmm0, %xmm0, %xmm0 # xmm0 = xmm0[3,3,3,3] + LONG $0xc158fac5 // vaddss %xmm1, %xmm0, %xmm0 + LONG $0x117ac1c4; BYTE $0x01 // vmovss %xmm0, (%r9) + WORD $0x8548; BYTE $0xc0 // testq %rax, %rax + JLE LBB0_8 + LONG $0x0710fac5 // vmovss (%rdi), %xmm0 # xmm0 = mem[0],zero,zero,zero + LONG $0x0659fac5 // vmulss (%rsi), %xmm0, %xmm0 + LONG $0x587ac1c4; BYTE $0x00 // vaddss (%r8), %xmm0, %xmm0 + LONG $0x117ac1c4; BYTE $0x00 // vmovss %xmm0, (%r8) + LONG $0x0710fac5 // vmovss (%rdi), %xmm0 # xmm0 = mem[0],zero,zero,zero + LONG $0x0259fac5 // vmulss (%rdx), %xmm0, %xmm0 + LONG $0x587ac1c4; BYTE $0x01 // vaddss (%r9), %xmm0, %xmm0 + LONG $0x117ac1c4; BYTE $0x01 // vmovss %xmm0, (%r9) + LONG $0x01f88348 // cmpq $1, %rax + JE LBB0_8 + LONG $0x4710fac5; BYTE $0x04 // vmovss 4(%rdi), %xmm0 # xmm0 = mem[0],zero,zero,zero + LONG $0x4659fac5; BYTE $0x04 // vmulss 4(%rsi), %xmm0, %xmm0 + LONG $0x587ac1c4; BYTE $0x00 // vaddss (%r8), %xmm0, %xmm0 + LONG $0x117ac1c4; BYTE $0x00 // vmovss %xmm0, (%r8) + LONG $0x4710fac5; BYTE $0x04 // vmovss 4(%rdi), %xmm0 # xmm0 = mem[0],zero,zero,zero + LONG $0x4259fac5; BYTE $0x04 // vmulss 4(%rdx), %xmm0, %xmm0 + LONG $0x587ac1c4; BYTE $0x01 // vaddss (%r9), %xmm0, %xmm0 + LONG $0x117ac1c4; BYTE $0x01 // vmovss %xmm0, (%r9) + LONG $0x02f88348 // cmpq $2, %rax + JE LBB0_8 + LONG $0x4710fac5; BYTE $0x08 // vmovss 8(%rdi), %xmm0 # xmm0 = mem[0],zero,zero,zero + LONG $0x4659fac5; BYTE $0x08 // vmulss 8(%rsi), %xmm0, %xmm0 + LONG $0x587ac1c4; BYTE $0x00 // vaddss (%r8), %xmm0, %xmm0 + LONG $0x117ac1c4; BYTE $0x00 // vmovss %xmm0, (%r8) + LONG $0x4710fac5; BYTE $0x08 // vmovss 8(%rdi), %xmm0 # xmm0 = mem[0],zero,zero,zero + LONG $0x4259fac5; BYTE $0x08 // vmulss 8(%rdx), %xmm0, %xmm0 + LONG $0x587ac1c4; BYTE $0x01 // vaddss (%r9), %xmm0, %xmm0 + LONG $0x117ac1c4; BYTE $0x01 // vmovss %xmm0, (%r9) + LONG $0x03f88348 // cmpq $3, %rax + JE LBB0_8 + LONG $0x4710fac5; BYTE $0x0c // vmovss 12(%rdi), %xmm0 # xmm0 = mem[0],zero,zero,zero + LONG $0x4659fac5; BYTE $0x0c // vmulss 12(%rsi), %xmm0, %xmm0 + LONG $0x587ac1c4; BYTE $0x00 // vaddss (%r8), %xmm0, %xmm0 + LONG $0x117ac1c4; BYTE $0x00 // vmovss %xmm0, (%r8) + LONG $0x4710fac5; BYTE $0x0c // vmovss 12(%rdi), %xmm0 # xmm0 = mem[0],zero,zero,zero + LONG $0x4259fac5; BYTE $0x0c // vmulss 12(%rdx), %xmm0, %xmm0 + LONG $0x587ac1c4; BYTE $0x01 // vaddss (%r9), %xmm0, %xmm0 + LONG $0x117ac1c4; BYTE $0x01 // vmovss %xmm0, (%r9) + LONG $0x04f88348 // cmpq $4, %rax + JE LBB0_8 + LONG $0x4710fac5; BYTE $0x10 // vmovss 16(%rdi), %xmm0 # xmm0 = mem[0],zero,zero,zero + LONG $0x4659fac5; BYTE $0x10 // vmulss 16(%rsi), %xmm0, %xmm0 + LONG $0x587ac1c4; BYTE $0x00 // vaddss (%r8), %xmm0, %xmm0 + LONG $0x117ac1c4; BYTE $0x00 // vmovss %xmm0, (%r8) + LONG $0x4710fac5; BYTE $0x10 // vmovss 16(%rdi), %xmm0 # xmm0 = mem[0],zero,zero,zero + LONG $0x4259fac5; BYTE $0x10 // vmulss 16(%rdx), %xmm0, %xmm0 + LONG $0x587ac1c4; BYTE $0x01 // vaddss (%r9), %xmm0, %xmm0 + LONG $0x117ac1c4; BYTE $0x01 // vmovss %xmm0, (%r9) + LONG $0x05f88348 // cmpq $5, %rax + JE LBB0_8 + LONG $0x4710fac5; BYTE $0x14 // vmovss 20(%rdi), %xmm0 # xmm0 = mem[0],zero,zero,zero + LONG $0x4659fac5; BYTE $0x14 // vmulss 20(%rsi), %xmm0, %xmm0 + LONG $0x587ac1c4; BYTE $0x00 // vaddss (%r8), %xmm0, %xmm0 + LONG $0x117ac1c4; BYTE $0x00 // vmovss %xmm0, (%r8) + LONG $0x4710fac5; BYTE $0x14 // vmovss 20(%rdi), %xmm0 # xmm0 = mem[0],zero,zero,zero + LONG $0x4259fac5; BYTE $0x14 // vmulss 20(%rdx), %xmm0, %xmm0 + LONG $0x587ac1c4; BYTE $0x01 // vaddss (%r9), %xmm0, %xmm0 + LONG $0x117ac1c4; BYTE $0x01 // vmovss %xmm0, (%r9) + LONG $0x06f88348 // cmpq $6, %rax + JE LBB0_8 + LONG $0x4710fac5; BYTE $0x18 // vmovss 24(%rdi), %xmm0 # xmm0 = mem[0],zero,zero,zero + LONG $0x4659fac5; BYTE $0x18 // vmulss 24(%rsi), %xmm0, %xmm0 + LONG $0x587ac1c4; BYTE $0x00 // vaddss (%r8), %xmm0, %xmm0 + LONG $0x117ac1c4; BYTE $0x00 // vmovss %xmm0, (%r8) + LONG $0x4710fac5; BYTE $0x18 // vmovss 24(%rdi), %xmm0 # xmm0 = mem[0],zero,zero,zero + LONG $0x4259fac5; BYTE $0x18 // vmulss 24(%rdx), %xmm0, %xmm0 + LONG $0x587ac1c4; BYTE $0x01 // vaddss (%r9), %xmm0, %xmm0 + LONG $0x117ac1c4; BYTE $0x01 // vmovss %xmm0, (%r9) + +LBB0_8: + LONG $0xf0658d48 // leaq -16(%rbp), %rsp + BYTE $0x5b // popq %rbx + WORD $0x5e41 // popq %r14 + BYTE $0x5d // popq %rbp + WORD $0xf8c5; BYTE $0x77 // vzeroupper + RET + +TEXT ·xvec_avx_batch_inner_products4(SB), $0-80 + MOVQ query+0(FP), DI + MOVQ first+8(FP), SI + MOVQ second+16(FP), DX + MOVQ third+24(FP), CX + MOVQ fourth+32(FP), R8 + MOVQ size+40(FP), R9 + PUSHQ fourth_output+72(FP) + PUSHQ third_output+64(FP) + PUSHQ second_output+56(FP) + PUSHQ first_output+48(FP) + PUSHQ $0 + BYTE $0x55 // pushq %rbp + WORD $0x8948; BYTE $0xe5 // movq %rsp, %rbp + WORD $0x5741 // pushq %r15 + WORD $0x5641 // pushq %r14 + WORD $0x5441 // pushq %r12 + BYTE $0x53 // pushq %rbx + LONG $0xf8e48348 // andq $-8, %rsp + LONG $0x28458b48 // movq 40(%rbp), %rax + LONG $0x20558b4c // movq 32(%rbp), %r10 + LONG $0x185d8b4c // movq 24(%rbp), %r11 + LONG $0x105d8b48 // movq 16(%rbp), %rbx + LONG $0x07798d4d // leaq 7(%r9), %r15 + WORD $0x854d; BYTE $0xc9 // testq %r9, %r9 + LONG $0xf9490f4d // cmovnsq %r9, %r15 + WORD $0x894d; BYTE $0xfc // movq %r15, %r12 + LONG $0xf8e48349 // andq $-8, %r12 + WORD $0x894d; BYTE $0xce // movq %r9, %r14 + WORD $0x294d; BYTE $0xe6 // subq %r12, %r14 + LONG $0xdb57e0c5 // vxorps %xmm3, %xmm3, %xmm3 + LONG $0x08f98349 // cmpq $8, %r9 + JL LBB1_1 + LONG $0x03ffc149 // sarq $3, %r15 + LONG $0xc057f8c5 // vxorps %xmm0, %xmm0, %xmm0 + WORD $0x3145; BYTE $0xc9 // xorl %r9d, %r9d + WORD $0x894d; BYTE $0xfc // movq %r15, %r12 + LONG $0xc957f0c5 // vxorps %xmm1, %xmm1, %xmm1 + LONG $0xd257e8c5 // vxorps %xmm2, %xmm2, %xmm2 + +LBB1_6: + LONG $0x107ca1c4; WORD $0x0f24 // vmovups (%rdi,%r9), %ymm4 + LONG $0x595ca1c4; WORD $0x0e2c // vmulps (%rsi,%r9), %ymm4, %ymm5 + LONG $0xdd58e4c5 // vaddps %ymm5, %ymm3, %ymm3 + LONG $0x595ca1c4; WORD $0x0a2c // vmulps (%rdx,%r9), %ymm4, %ymm5 + LONG $0xd558ecc5 // vaddps %ymm5, %ymm2, %ymm2 + LONG $0x595ca1c4; WORD $0x092c // vmulps (%rcx,%r9), %ymm4, %ymm5 + LONG $0x595c81c4; WORD $0x0824 // vmulps (%r8,%r9), %ymm4, %ymm4 + LONG $0xcd58f4c5 // vaddps %ymm5, %ymm1, %ymm1 + LONG $0xc458fcc5 // vaddps %ymm4, %ymm0, %ymm0 + LONG $0x20c18349 // addq $32, %r9 + WORD $0xff49; BYTE $0xcc // decq %r12 + JNE LBB1_6 + LONG $0x05e7c149 // shlq $5, %r15 + WORD $0x014c; BYTE $0xff // addq %r15, %rdi + WORD $0x014c; BYTE $0xfe // addq %r15, %rsi + WORD $0x014c; BYTE $0xfa // addq %r15, %rdx + WORD $0x014c; BYTE $0xf9 // addq %r15, %rcx + WORD $0x014d; BYTE $0xf8 // addq %r15, %r8 + JMP LBB1_3 + +LBB1_1: + LONG $0xd257e8c5 // vxorps %xmm2, %xmm2, %xmm2 + LONG $0xc957f0c5 // vxorps %xmm1, %xmm1, %xmm1 + LONG $0xc057f8c5 // vxorps %xmm0, %xmm0, %xmm0 + +LBB1_3: + LONG $0xe457d8c5 // vxorps %xmm4, %xmm4, %xmm4 + LONG $0xec58e2c5 // vaddss %xmm4, %xmm3, %xmm5 + LONG $0xf316fac5 // vmovshdup %xmm3, %xmm6 # xmm6 = xmm3[1,1,3,3] + LONG $0xed58cac5 // vaddss %xmm5, %xmm6, %xmm5 + LONG $0xf3c6e1c5; BYTE $0x01 // vshufpd $1, %xmm3, %xmm3, %xmm6 # xmm6 = xmm3[1,0] + LONG $0xed58cac5 // vaddss %xmm5, %xmm6, %xmm5 + LONG $0xf3c6e0c5; BYTE $0xff // vshufps $255, %xmm3, %xmm3, %xmm6 # xmm6 = xmm3[3,3,3,3] + LONG $0xed58cac5 // vaddss %xmm5, %xmm6, %xmm5 + LONG $0x197de3c4; WORD $0x01db // vextractf128 $1, %ymm3, %xmm3 + LONG $0xed58e2c5 // vaddss %xmm5, %xmm3, %xmm5 + LONG $0xf316fac5 // vmovshdup %xmm3, %xmm6 # xmm6 = xmm3[1,1,3,3] + LONG $0xed58cac5 // vaddss %xmm5, %xmm6, %xmm5 + LONG $0xf3c6e1c5; BYTE $0x01 // vshufpd $1, %xmm3, %xmm3, %xmm6 # xmm6 = xmm3[1,0] + LONG $0xed58cac5 // vaddss %xmm5, %xmm6, %xmm5 + LONG $0xdbc6e0c5; BYTE $0xff // vshufps $255, %xmm3, %xmm3, %xmm3 # xmm3 = xmm3[3,3,3,3] + LONG $0xdd58e2c5 // vaddss %xmm5, %xmm3, %xmm3 + LONG $0x1b11fac5 // vmovss %xmm3, (%rbx) + LONG $0xdc58eac5 // vaddss %xmm4, %xmm2, %xmm3 + LONG $0xea16fac5 // vmovshdup %xmm2, %xmm5 # xmm5 = xmm2[1,1,3,3] + LONG $0xdb58d2c5 // vaddss %xmm3, %xmm5, %xmm3 + LONG $0xeac6e9c5; BYTE $0x01 // vshufpd $1, %xmm2, %xmm2, %xmm5 # xmm5 = xmm2[1,0] + LONG $0xdb58d2c5 // vaddss %xmm3, %xmm5, %xmm3 + LONG $0xeac6e8c5; BYTE $0xff // vshufps $255, %xmm2, %xmm2, %xmm5 # xmm5 = xmm2[3,3,3,3] + LONG $0xdb58d2c5 // vaddss %xmm3, %xmm5, %xmm3 + LONG $0x197de3c4; WORD $0x01d2 // vextractf128 $1, %ymm2, %xmm2 + LONG $0xdb58eac5 // vaddss %xmm3, %xmm2, %xmm3 + LONG $0xea16fac5 // vmovshdup %xmm2, %xmm5 # xmm5 = xmm2[1,1,3,3] + LONG $0xdb58d2c5 // vaddss %xmm3, %xmm5, %xmm3 + LONG $0xeac6e9c5; BYTE $0x01 // vshufpd $1, %xmm2, %xmm2, %xmm5 # xmm5 = xmm2[1,0] + LONG $0xdb58d2c5 // vaddss %xmm3, %xmm5, %xmm3 + LONG $0xd2c6e8c5; BYTE $0xff // vshufps $255, %xmm2, %xmm2, %xmm2 # xmm2 = xmm2[3,3,3,3] + LONG $0xd358eac5 // vaddss %xmm3, %xmm2, %xmm2 + LONG $0x117ac1c4; BYTE $0x13 // vmovss %xmm2, (%r11) + LONG $0xd458f2c5 // vaddss %xmm4, %xmm1, %xmm2 + LONG $0xd916fac5 // vmovshdup %xmm1, %xmm3 # xmm3 = xmm1[1,1,3,3] + LONG $0xd258e2c5 // vaddss %xmm2, %xmm3, %xmm2 + LONG $0xd9c6f1c5; BYTE $0x01 // vshufpd $1, %xmm1, %xmm1, %xmm3 # xmm3 = xmm1[1,0] + LONG $0xd258e2c5 // vaddss %xmm2, %xmm3, %xmm2 + LONG $0xd9c6f0c5; BYTE $0xff // vshufps $255, %xmm1, %xmm1, %xmm3 # xmm3 = xmm1[3,3,3,3] + LONG $0xd258e2c5 // vaddss %xmm2, %xmm3, %xmm2 + LONG $0x197de3c4; WORD $0x01c9 // vextractf128 $1, %ymm1, %xmm1 + LONG $0xd258f2c5 // vaddss %xmm2, %xmm1, %xmm2 + LONG $0xd916fac5 // vmovshdup %xmm1, %xmm3 # xmm3 = xmm1[1,1,3,3] + LONG $0xd258e2c5 // vaddss %xmm2, %xmm3, %xmm2 + LONG $0xd9c6f1c5; BYTE $0x01 // vshufpd $1, %xmm1, %xmm1, %xmm3 # xmm3 = xmm1[1,0] + LONG $0xd258e2c5 // vaddss %xmm2, %xmm3, %xmm2 + LONG $0xc9c6f0c5; BYTE $0xff // vshufps $255, %xmm1, %xmm1, %xmm1 # xmm1 = xmm1[3,3,3,3] + LONG $0xca58f2c5 // vaddss %xmm2, %xmm1, %xmm1 + LONG $0x117ac1c4; BYTE $0x0a // vmovss %xmm1, (%r10) + LONG $0xcc58fac5 // vaddss %xmm4, %xmm0, %xmm1 + LONG $0xd016fac5 // vmovshdup %xmm0, %xmm2 # xmm2 = xmm0[1,1,3,3] + LONG $0xc958eac5 // vaddss %xmm1, %xmm2, %xmm1 + LONG $0xd0c6f9c5; BYTE $0x01 // vshufpd $1, %xmm0, %xmm0, %xmm2 # xmm2 = xmm0[1,0] + LONG $0xc958eac5 // vaddss %xmm1, %xmm2, %xmm1 + LONG $0xd0c6f8c5; BYTE $0xff // vshufps $255, %xmm0, %xmm0, %xmm2 # xmm2 = xmm0[3,3,3,3] + LONG $0xc958eac5 // vaddss %xmm1, %xmm2, %xmm1 + LONG $0x197de3c4; WORD $0x01c0 // vextractf128 $1, %ymm0, %xmm0 + LONG $0xc958fac5 // vaddss %xmm1, %xmm0, %xmm1 + LONG $0xd016fac5 // vmovshdup %xmm0, %xmm2 # xmm2 = xmm0[1,1,3,3] + LONG $0xc958eac5 // vaddss %xmm1, %xmm2, %xmm1 + LONG $0xd0c6f9c5; BYTE $0x01 // vshufpd $1, %xmm0, %xmm0, %xmm2 # xmm2 = xmm0[1,0] + LONG $0xc958eac5 // vaddss %xmm1, %xmm2, %xmm1 + LONG $0xc0c6f8c5; BYTE $0xff // vshufps $255, %xmm0, %xmm0, %xmm0 # xmm0 = xmm0[3,3,3,3] + LONG $0xc158fac5 // vaddss %xmm1, %xmm0, %xmm0 + LONG $0x0011fac5 // vmovss %xmm0, (%rax) + WORD $0x854d; BYTE $0xf6 // testq %r14, %r14 + JLE LBB1_4 + LONG $0x0710fac5 // vmovss (%rdi), %xmm0 # xmm0 = mem[0],zero,zero,zero + LONG $0x0659fac5 // vmulss (%rsi), %xmm0, %xmm0 + LONG $0x0358fac5 // vaddss (%rbx), %xmm0, %xmm0 + LONG $0x0311fac5 // vmovss %xmm0, (%rbx) + LONG $0x0710fac5 // vmovss (%rdi), %xmm0 # xmm0 = mem[0],zero,zero,zero + LONG $0x0259fac5 // vmulss (%rdx), %xmm0, %xmm0 + LONG $0x587ac1c4; BYTE $0x03 // vaddss (%r11), %xmm0, %xmm0 + LONG $0x117ac1c4; BYTE $0x03 // vmovss %xmm0, (%r11) + LONG $0x0710fac5 // vmovss (%rdi), %xmm0 # xmm0 = mem[0],zero,zero,zero + LONG $0x0159fac5 // vmulss (%rcx), %xmm0, %xmm0 + LONG $0x587ac1c4; BYTE $0x02 // vaddss (%r10), %xmm0, %xmm0 + LONG $0x117ac1c4; BYTE $0x02 // vmovss %xmm0, (%r10) + LONG $0x0710fac5 // vmovss (%rdi), %xmm0 # xmm0 = mem[0],zero,zero,zero + LONG $0x597ac1c4; BYTE $0x00 // vmulss (%r8), %xmm0, %xmm0 + LONG $0x0058fac5 // vaddss (%rax), %xmm0, %xmm0 + LONG $0x0011fac5 // vmovss %xmm0, (%rax) + LONG $0x01fe8349 // cmpq $1, %r14 + JE LBB1_4 + LONG $0x4710fac5; BYTE $0x04 // vmovss 4(%rdi), %xmm0 # xmm0 = mem[0],zero,zero,zero + LONG $0x4659fac5; BYTE $0x04 // vmulss 4(%rsi), %xmm0, %xmm0 + LONG $0x0358fac5 // vaddss (%rbx), %xmm0, %xmm0 + LONG $0x0311fac5 // vmovss %xmm0, (%rbx) + LONG $0x4710fac5; BYTE $0x04 // vmovss 4(%rdi), %xmm0 # xmm0 = mem[0],zero,zero,zero + LONG $0x4259fac5; BYTE $0x04 // vmulss 4(%rdx), %xmm0, %xmm0 + LONG $0x587ac1c4; BYTE $0x03 // vaddss (%r11), %xmm0, %xmm0 + LONG $0x117ac1c4; BYTE $0x03 // vmovss %xmm0, (%r11) + LONG $0x4710fac5; BYTE $0x04 // vmovss 4(%rdi), %xmm0 # xmm0 = mem[0],zero,zero,zero + LONG $0x4159fac5; BYTE $0x04 // vmulss 4(%rcx), %xmm0, %xmm0 + LONG $0x587ac1c4; BYTE $0x02 // vaddss (%r10), %xmm0, %xmm0 + LONG $0x117ac1c4; BYTE $0x02 // vmovss %xmm0, (%r10) + LONG $0x4710fac5; BYTE $0x04 // vmovss 4(%rdi), %xmm0 # xmm0 = mem[0],zero,zero,zero + LONG $0x597ac1c4; WORD $0x0440 // vmulss 4(%r8), %xmm0, %xmm0 + LONG $0x0058fac5 // vaddss (%rax), %xmm0, %xmm0 + LONG $0x0011fac5 // vmovss %xmm0, (%rax) + LONG $0x02fe8349 // cmpq $2, %r14 + JE LBB1_4 + LONG $0x4710fac5; BYTE $0x08 // vmovss 8(%rdi), %xmm0 # xmm0 = mem[0],zero,zero,zero + LONG $0x4659fac5; BYTE $0x08 // vmulss 8(%rsi), %xmm0, %xmm0 + LONG $0x0358fac5 // vaddss (%rbx), %xmm0, %xmm0 + LONG $0x0311fac5 // vmovss %xmm0, (%rbx) + LONG $0x4710fac5; BYTE $0x08 // vmovss 8(%rdi), %xmm0 # xmm0 = mem[0],zero,zero,zero + LONG $0x4259fac5; BYTE $0x08 // vmulss 8(%rdx), %xmm0, %xmm0 + LONG $0x587ac1c4; BYTE $0x03 // vaddss (%r11), %xmm0, %xmm0 + LONG $0x117ac1c4; BYTE $0x03 // vmovss %xmm0, (%r11) + LONG $0x4710fac5; BYTE $0x08 // vmovss 8(%rdi), %xmm0 # xmm0 = mem[0],zero,zero,zero + LONG $0x4159fac5; BYTE $0x08 // vmulss 8(%rcx), %xmm0, %xmm0 + LONG $0x587ac1c4; BYTE $0x02 // vaddss (%r10), %xmm0, %xmm0 + LONG $0x117ac1c4; BYTE $0x02 // vmovss %xmm0, (%r10) + LONG $0x4710fac5; BYTE $0x08 // vmovss 8(%rdi), %xmm0 # xmm0 = mem[0],zero,zero,zero + LONG $0x597ac1c4; WORD $0x0840 // vmulss 8(%r8), %xmm0, %xmm0 + LONG $0x0058fac5 // vaddss (%rax), %xmm0, %xmm0 + LONG $0x0011fac5 // vmovss %xmm0, (%rax) + LONG $0x03fe8349 // cmpq $3, %r14 + JE LBB1_4 + LONG $0x4710fac5; BYTE $0x0c // vmovss 12(%rdi), %xmm0 # xmm0 = mem[0],zero,zero,zero + LONG $0x4659fac5; BYTE $0x0c // vmulss 12(%rsi), %xmm0, %xmm0 + LONG $0x0358fac5 // vaddss (%rbx), %xmm0, %xmm0 + LONG $0x0311fac5 // vmovss %xmm0, (%rbx) + LONG $0x4710fac5; BYTE $0x0c // vmovss 12(%rdi), %xmm0 # xmm0 = mem[0],zero,zero,zero + LONG $0x4259fac5; BYTE $0x0c // vmulss 12(%rdx), %xmm0, %xmm0 + LONG $0x587ac1c4; BYTE $0x03 // vaddss (%r11), %xmm0, %xmm0 + LONG $0x117ac1c4; BYTE $0x03 // vmovss %xmm0, (%r11) + LONG $0x4710fac5; BYTE $0x0c // vmovss 12(%rdi), %xmm0 # xmm0 = mem[0],zero,zero,zero + LONG $0x4159fac5; BYTE $0x0c // vmulss 12(%rcx), %xmm0, %xmm0 + LONG $0x587ac1c4; BYTE $0x02 // vaddss (%r10), %xmm0, %xmm0 + LONG $0x117ac1c4; BYTE $0x02 // vmovss %xmm0, (%r10) + LONG $0x4710fac5; BYTE $0x0c // vmovss 12(%rdi), %xmm0 # xmm0 = mem[0],zero,zero,zero + LONG $0x597ac1c4; WORD $0x0c40 // vmulss 12(%r8), %xmm0, %xmm0 + LONG $0x0058fac5 // vaddss (%rax), %xmm0, %xmm0 + LONG $0x0011fac5 // vmovss %xmm0, (%rax) + LONG $0x04fe8349 // cmpq $4, %r14 + JE LBB1_4 + LONG $0x4710fac5; BYTE $0x10 // vmovss 16(%rdi), %xmm0 # xmm0 = mem[0],zero,zero,zero + LONG $0x4659fac5; BYTE $0x10 // vmulss 16(%rsi), %xmm0, %xmm0 + LONG $0x0358fac5 // vaddss (%rbx), %xmm0, %xmm0 + LONG $0x0311fac5 // vmovss %xmm0, (%rbx) + LONG $0x4710fac5; BYTE $0x10 // vmovss 16(%rdi), %xmm0 # xmm0 = mem[0],zero,zero,zero + LONG $0x4259fac5; BYTE $0x10 // vmulss 16(%rdx), %xmm0, %xmm0 + LONG $0x587ac1c4; BYTE $0x03 // vaddss (%r11), %xmm0, %xmm0 + LONG $0x117ac1c4; BYTE $0x03 // vmovss %xmm0, (%r11) + LONG $0x4710fac5; BYTE $0x10 // vmovss 16(%rdi), %xmm0 # xmm0 = mem[0],zero,zero,zero + LONG $0x4159fac5; BYTE $0x10 // vmulss 16(%rcx), %xmm0, %xmm0 + LONG $0x587ac1c4; BYTE $0x02 // vaddss (%r10), %xmm0, %xmm0 + LONG $0x117ac1c4; BYTE $0x02 // vmovss %xmm0, (%r10) + LONG $0x4710fac5; BYTE $0x10 // vmovss 16(%rdi), %xmm0 # xmm0 = mem[0],zero,zero,zero + LONG $0x597ac1c4; WORD $0x1040 // vmulss 16(%r8), %xmm0, %xmm0 + LONG $0x0058fac5 // vaddss (%rax), %xmm0, %xmm0 + LONG $0x0011fac5 // vmovss %xmm0, (%rax) + LONG $0x05fe8349 // cmpq $5, %r14 + JE LBB1_4 + LONG $0x4710fac5; BYTE $0x14 // vmovss 20(%rdi), %xmm0 # xmm0 = mem[0],zero,zero,zero + LONG $0x4659fac5; BYTE $0x14 // vmulss 20(%rsi), %xmm0, %xmm0 + LONG $0x0358fac5 // vaddss (%rbx), %xmm0, %xmm0 + LONG $0x0311fac5 // vmovss %xmm0, (%rbx) + LONG $0x4710fac5; BYTE $0x14 // vmovss 20(%rdi), %xmm0 # xmm0 = mem[0],zero,zero,zero + LONG $0x4259fac5; BYTE $0x14 // vmulss 20(%rdx), %xmm0, %xmm0 + LONG $0x587ac1c4; BYTE $0x03 // vaddss (%r11), %xmm0, %xmm0 + LONG $0x117ac1c4; BYTE $0x03 // vmovss %xmm0, (%r11) + LONG $0x4710fac5; BYTE $0x14 // vmovss 20(%rdi), %xmm0 # xmm0 = mem[0],zero,zero,zero + LONG $0x4159fac5; BYTE $0x14 // vmulss 20(%rcx), %xmm0, %xmm0 + LONG $0x587ac1c4; BYTE $0x02 // vaddss (%r10), %xmm0, %xmm0 + LONG $0x117ac1c4; BYTE $0x02 // vmovss %xmm0, (%r10) + LONG $0x4710fac5; BYTE $0x14 // vmovss 20(%rdi), %xmm0 # xmm0 = mem[0],zero,zero,zero + LONG $0x597ac1c4; WORD $0x1440 // vmulss 20(%r8), %xmm0, %xmm0 + LONG $0x0058fac5 // vaddss (%rax), %xmm0, %xmm0 + LONG $0x0011fac5 // vmovss %xmm0, (%rax) + LONG $0x06fe8349 // cmpq $6, %r14 + JE LBB1_4 + LONG $0x4710fac5; BYTE $0x18 // vmovss 24(%rdi), %xmm0 # xmm0 = mem[0],zero,zero,zero + LONG $0x4659fac5; BYTE $0x18 // vmulss 24(%rsi), %xmm0, %xmm0 + LONG $0x0358fac5 // vaddss (%rbx), %xmm0, %xmm0 + LONG $0x0311fac5 // vmovss %xmm0, (%rbx) + LONG $0x4710fac5; BYTE $0x18 // vmovss 24(%rdi), %xmm0 # xmm0 = mem[0],zero,zero,zero + LONG $0x4259fac5; BYTE $0x18 // vmulss 24(%rdx), %xmm0, %xmm0 + LONG $0x587ac1c4; BYTE $0x03 // vaddss (%r11), %xmm0, %xmm0 + LONG $0x117ac1c4; BYTE $0x03 // vmovss %xmm0, (%r11) + LONG $0x4710fac5; BYTE $0x18 // vmovss 24(%rdi), %xmm0 # xmm0 = mem[0],zero,zero,zero + LONG $0x4159fac5; BYTE $0x18 // vmulss 24(%rcx), %xmm0, %xmm0 + LONG $0x587ac1c4; BYTE $0x02 // vaddss (%r10), %xmm0, %xmm0 + LONG $0x117ac1c4; BYTE $0x02 // vmovss %xmm0, (%r10) + LONG $0x4710fac5; BYTE $0x18 // vmovss 24(%rdi), %xmm0 # xmm0 = mem[0],zero,zero,zero + LONG $0x597ac1c4; WORD $0x1840 // vmulss 24(%r8), %xmm0, %xmm0 + LONG $0x0058fac5 // vaddss (%rax), %xmm0, %xmm0 + LONG $0x0011fac5 // vmovss %xmm0, (%rax) + +LBB1_4: + LONG $0xe0658d48 // leaq -32(%rbp), %rsp + BYTE $0x5b // popq %rbx + WORD $0x5c41 // popq %r12 + WORD $0x5e41 // popq %r14 + WORD $0x5f41 // popq %r15 + BYTE $0x5d // popq %rbp + WORD $0xf8c5; BYTE $0x77 // vzeroupper + POPQ DI + POPQ DI + POPQ DI + POPQ DI + POPQ DI + RET diff --git a/internal/floats/floats_lasx.go b/internal/ailego/math/distance_utility_lasx.go similarity index 89% rename from internal/floats/floats_lasx.go rename to internal/ailego/math/distance_utility_lasx.go index 14f4448..f1f0877 100644 --- a/internal/floats/floats_lasx.go +++ b/internal/ailego/math/distance_utility_lasx.go @@ -4,9 +4,9 @@ // clang 21.1.8 (6ubuntu1) // objdump 2.46 // flags: -mlasx -O3 -// source: src/floats_lasx.c +// source: src/distance_utility_lasx.c -package floats +package mathutil import "unsafe" diff --git a/internal/floats/floats_lasx.s b/internal/ailego/math/distance_utility_lasx.s similarity index 99% rename from internal/floats/floats_lasx.s rename to internal/ailego/math/distance_utility_lasx.s index 394c0d0..25c1fa8 100644 --- a/internal/floats/floats_lasx.s +++ b/internal/ailego/math/distance_utility_lasx.s @@ -4,7 +4,7 @@ // clang 21.1.8 (6ubuntu1) // objdump 2.46 // flags: -mlasx -O3 -// source: src/floats_lasx.c +// source: src/distance_utility_lasx.c TEXT ·xvec_lasx_l2_squared(SB), $0-32 MOVV left+0(FP), R4 diff --git a/internal/floats/floats_loong64.go b/internal/ailego/math/distance_utility_loong64.go similarity index 91% rename from internal/floats/floats_loong64.go rename to internal/ailego/math/distance_utility_loong64.go index 9b8d2b1..e9100db 100644 --- a/internal/floats/floats_loong64.go +++ b/internal/ailego/math/distance_utility_loong64.go @@ -14,7 +14,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package floats +package mathutil import ( "unsafe" @@ -26,15 +26,15 @@ import ( func init() { if cpu.Loong64.HasLASX { - kernels.l2 = l2SquaredLASX + kernels.l2 = squaredEuclideanLASX kernels.dot = innerProductLASX kernels.products = dotNormsLASX } } -func l2SquaredLASX(left, right []float32) float32 { +func squaredEuclideanLASX(left, right []float32) float32 { if len(left) < 8 { - return l2SquaredScalar(left, right) + return squaredEuclideanScalar(left, right) } var result float32 xvec_lasx_l2_squared(unsafe.Pointer(&left[0]), unsafe.Pointer(&right[0]), int64(len(left)), unsafe.Pointer(&result)) diff --git a/internal/floats/floats_loong64_test.go b/internal/ailego/math/distance_utility_loong64_test.go similarity index 89% rename from internal/floats/floats_loong64_test.go rename to internal/ailego/math/distance_utility_loong64_test.go index a3de2fa..2a65d20 100644 --- a/internal/floats/floats_loong64_test.go +++ b/internal/ailego/math/distance_utility_loong64_test.go @@ -14,7 +14,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package floats +package mathutil import ( "testing" @@ -26,5 +26,5 @@ func TestLASXDistanceKernels(t *testing.T) { if !cpu.Loong64.HasLASX { t.Skip("LASX is not supported by this CPU") } - testArchitectureKernels(t, l2SquaredLASX, innerProductLASX, dotNormsLASX) + testArchitectureKernels(t, squaredEuclideanLASX, innerProductLASX, dotNormsLASX) } diff --git a/internal/floats/floats_riscv64.go b/internal/ailego/math/distance_utility_riscv64.go similarity index 92% rename from internal/floats/floats_riscv64.go rename to internal/ailego/math/distance_utility_riscv64.go index d28530d..43126d8 100644 --- a/internal/floats/floats_riscv64.go +++ b/internal/ailego/math/distance_utility_riscv64.go @@ -14,7 +14,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package floats +package mathutil import ( "unsafe" @@ -26,13 +26,13 @@ import ( func init() { if cpu.RISCV64.HasV { - kernels.l2 = l2SquaredRVV + kernels.l2 = squaredEuclideanRVV kernels.dot = innerProductRVV kernels.products = dotNormsRVV } } -func l2SquaredRVV(left, right []float32) float32 { +func squaredEuclideanRVV(left, right []float32) float32 { var result float32 xvec_rvv_l2_squared(unsafe.Pointer(&left[0]), unsafe.Pointer(&right[0]), int64(len(left)), unsafe.Pointer(&result)) return result diff --git a/internal/floats/floats_riscv64_test.go b/internal/ailego/math/distance_utility_riscv64_test.go similarity index 89% rename from internal/floats/floats_riscv64_test.go rename to internal/ailego/math/distance_utility_riscv64_test.go index 210ce17..7029860 100644 --- a/internal/floats/floats_riscv64_test.go +++ b/internal/ailego/math/distance_utility_riscv64_test.go @@ -14,7 +14,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package floats +package mathutil import ( "testing" @@ -26,5 +26,5 @@ func TestRVVDistanceKernels(t *testing.T) { if !cpu.RISCV64.HasV { t.Skip("RVV is not supported by this CPU") } - testArchitectureKernels(t, l2SquaredRVV, innerProductRVV, dotNormsRVV) + testArchitectureKernels(t, squaredEuclideanRVV, innerProductRVV, dotNormsRVV) } diff --git a/internal/floats/floats_rvv.go b/internal/ailego/math/distance_utility_rvv.go similarity index 91% rename from internal/floats/floats_rvv.go rename to internal/ailego/math/distance_utility_rvv.go index e772c2b..89f1f2b 100644 --- a/internal/floats/floats_rvv.go +++ b/internal/ailego/math/distance_utility_rvv.go @@ -4,9 +4,9 @@ // clang 21.1.8 (6ubuntu1) // objdump 2.46 // flags: -march=rv64imafdv -isystem /usr/lib/llvm-21/lib/clang/21/include -O3 -// source: src/floats_rvv.c +// source: src/distance_utility_rvv.c -package floats +package mathutil import "unsafe" diff --git a/internal/floats/floats_rvv.s b/internal/ailego/math/distance_utility_rvv.s similarity index 99% rename from internal/floats/floats_rvv.s rename to internal/ailego/math/distance_utility_rvv.s index 9eb47a3..3d810bc 100644 --- a/internal/floats/floats_rvv.s +++ b/internal/ailego/math/distance_utility_rvv.s @@ -4,7 +4,7 @@ // clang 21.1.8 (6ubuntu1) // objdump 2.46 // flags: -march=rv64imafdv -isystem /usr/lib/llvm-21/lib/clang/21/include -O3 -// source: src/floats_rvv.c +// source: src/distance_utility_rvv.c TEXT ·xvec_rvv_l2_squared(SB), $0-32 MOV left+0(FP), A0 diff --git a/internal/floats/floats_test.go b/internal/ailego/math/distance_utility_test.go similarity index 84% rename from internal/floats/floats_test.go rename to internal/ailego/math/distance_utility_test.go index bf79ee7..ca0db4c 100644 --- a/internal/floats/floats_test.go +++ b/internal/ailego/math/distance_utility_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package floats +package mathutil import ( "fmt" @@ -40,9 +40,9 @@ func TestDistanceKernelsMatchFloat32Oracle(t *testing.T) { right = right[1:] wantL2, wantDot, wantLeftNorm, wantRightNorm := distanceOracle(left, right) - requireFloat32Close(t, wantL2, L2Squared(left, right)) - requireFloat32Close(t, wantDot, InnerProduct(left, right)) - dot, leftNorm, rightNorm := DotNorms(left, right) + requireFloat32Close(t, wantL2, squaredEuclidean(left, right)) + requireFloat32Close(t, wantDot, innerProduct(left, right)) + dot, leftNorm, rightNorm := dotNorms(left, right) requireFloat32Close(t, wantDot, dot) requireFloat32Close(t, wantLeftNorm, leftNorm) requireFloat32Close(t, wantRightNorm, rightNorm) @@ -56,8 +56,8 @@ func TestDistanceKernelsUseFloat32Accumulation(t *testing.T) { // At float32 precision, adding one after 2^24 no longer changes the sum. left := []float32{4096, 1} right := []float32{4096, 1} - require.Equal(t, float32(1<<24), InnerProduct(left, right)) - dot, leftNorm, rightNorm := DotNorms(left, right) + require.Equal(t, float32(1<<24), innerProduct(left, right)) + dot, leftNorm, rightNorm := dotNorms(left, right) require.Equal(t, float32(1<<24), dot) require.Equal(t, float32(1<<24), leftNorm) require.Equal(t, float32(1<<24), rightNorm) @@ -79,7 +79,7 @@ func TestInnerProducts2MatchesFloat32Oracle(t *testing.T) { query, first, second = query[1:], first[1:], second[1:] _, wantFirst, _, _ := distanceOracle(query, first) _, wantSecond, _, _ := distanceOracle(query, second) - gotFirst, gotSecond := InnerProducts2(query, first, second) + gotFirst, gotSecond := innerProducts2(query, first, second) requireFloat32Close(t, wantFirst, gotFirst) requireFloat32Close(t, wantSecond, gotSecond) }) @@ -99,7 +99,7 @@ func TestInnerProducts4MatchesFloat32Oracle(t *testing.T) { } vectors[vector] = vectors[vector][1:] } - first, second, third, fourth := InnerProducts4(vectors[0], vectors[1], vectors[2], vectors[3], vectors[4]) + first, second, third, fourth := innerProducts4(vectors[0], vectors[1], vectors[2], vectors[3], vectors[4]) for index, got := range []float32{first, second, third, fourth} { _, want, _, _ := distanceOracle(vectors[0], vectors[index+1]) requireFloat32Close(t, want, got) @@ -115,11 +115,11 @@ func TestDistanceKernelsDoNotAllocateOrMutate(t *testing.T) { rightCopy := append([]float32(nil), right...) require.Zero(t, testing.AllocsPerRun(100, func() { - benchmarkL2 = L2Squared(left, right) - benchmarkInnerProduct = InnerProduct(left, right) - benchmarkBatch2First, benchmarkBatch2Second = InnerProducts2(left, right, right) - benchmarkBatch4First, benchmarkBatch4Second, benchmarkBatch4Third, benchmarkBatch4Fourth = InnerProducts4(left, right, right, right, right) - benchmarkDot, benchmarkLeftNorm, benchmarkRightNorm = DotNorms(left, right) + benchmarkL2 = squaredEuclidean(left, right) + benchmarkInnerProduct = innerProduct(left, right) + benchmarkBatch2First, benchmarkBatch2Second = innerProducts2(left, right, right) + benchmarkBatch4First, benchmarkBatch4Second, benchmarkBatch4Third, benchmarkBatch4Fourth = innerProducts4(left, right, right, right, right) + benchmarkDot, benchmarkLeftNorm, benchmarkRightNorm = dotNorms(left, right) })) require.Equal(t, leftCopy, left) require.Equal(t, rightCopy, right) @@ -135,41 +135,41 @@ func BenchmarkDistanceKernels(b *testing.B) { } b.Run(fmt.Sprintf("L2/%d", dimension), func(b *testing.B) { for b.Loop() { - benchmarkL2 = L2Squared(left, right) + benchmarkL2 = squaredEuclidean(left, right) } }) b.Run(fmt.Sprintf("InnerProduct/%d", dimension), func(b *testing.B) { for b.Loop() { - benchmarkDot = InnerProduct(left, right) + benchmarkDot = innerProduct(left, right) } }) b.Run(fmt.Sprintf("InnerProductSequential2/%d", dimension), func(b *testing.B) { for b.Loop() { - benchmarkDot = InnerProduct(left, right) - benchmarkDot2 = InnerProduct(left, right) + benchmarkDot = innerProduct(left, right) + benchmarkDot2 = innerProduct(left, right) } }) b.Run(fmt.Sprintf("InnerProducts2/%d", dimension), func(b *testing.B) { for b.Loop() { - benchmarkDot, benchmarkDot2 = InnerProducts2(left, right, right) + benchmarkDot, benchmarkDot2 = innerProducts2(left, right, right) } }) b.Run(fmt.Sprintf("InnerProductSequential4/%d", dimension), func(b *testing.B) { for b.Loop() { - benchmarkDot = InnerProduct(left, right) - benchmarkDot2 = InnerProduct(left, right) - benchmarkDot3 = InnerProduct(left, right) - benchmarkDot4 = InnerProduct(left, right) + benchmarkDot = innerProduct(left, right) + benchmarkDot2 = innerProduct(left, right) + benchmarkDot3 = innerProduct(left, right) + benchmarkDot4 = innerProduct(left, right) } }) b.Run(fmt.Sprintf("InnerProducts4/%d", dimension), func(b *testing.B) { for b.Loop() { - benchmarkDot, benchmarkDot2, benchmarkDot3, benchmarkDot4 = InnerProducts4(left, right, right, right, right) + benchmarkDot, benchmarkDot2, benchmarkDot3, benchmarkDot4 = innerProducts4(left, right, right, right, right) } }) b.Run(fmt.Sprintf("DotNorms/%d", dimension), func(b *testing.B) { for b.Loop() { - benchmarkDot, benchmarkLeftNorm, benchmarkRightNorm = DotNorms(left, right) + benchmarkDot, benchmarkLeftNorm, benchmarkRightNorm = dotNorms(left, right) } }) } diff --git a/internal/ailego/math/euclidean_distance_matrix_fp32_avx.go b/internal/ailego/math/euclidean_distance_matrix_fp32_avx.go new file mode 100644 index 0000000..122d0ba --- /dev/null +++ b/internal/ailego/math/euclidean_distance_matrix_fp32_avx.go @@ -0,0 +1,14 @@ +//go:build !noasm && amd64 +// Code generated by GoAT. DO NOT EDIT. +// versions: +// clang 21.1.8 (6ubuntu1) +// objdump 2.46 +// flags: -mavx -O3 +// source: src/euclidean_distance_matrix_fp32_avx.c + +package mathutil + +import "unsafe" + +//go:noescape +func squared_euclidean_distance_fp32_avx(lhs, rhs unsafe.Pointer, size int64) (result float32) diff --git a/internal/ailego/math/euclidean_distance_matrix_fp32_avx.s b/internal/ailego/math/euclidean_distance_matrix_fp32_avx.s new file mode 100644 index 0000000..28094f4 --- /dev/null +++ b/internal/ailego/math/euclidean_distance_matrix_fp32_avx.s @@ -0,0 +1,238 @@ +//go:build !noasm && amd64 +// Code generated by GoAT. DO NOT EDIT. +// versions: +// clang 21.1.8 (6ubuntu1) +// objdump 2.46 +// flags: -mavx -O3 +// source: src/euclidean_distance_matrix_fp32_avx.c + +TEXT ·squared_euclidean_distance_fp32_avx(SB), $8-28 + MOVQ lhs+0(FP), DI + MOVQ rhs+8(FP), SI + MOVQ size+16(FP), DX + BYTE $0x55 // pushq %rbp + WORD $0x8948; BYTE $0xe5 // movq %rsp, %rbp + LONG $0xf8e48348 // andq $-8, %rsp + LONG $0x97048d48 // leaq (%rdi,%rdx,4), %rax + LONG $0x02e2c148 // shlq $2, %rdx + WORD $0x8949; BYTE $0xd0 // movq %rdx, %r8 + LONG $0xc0e08349 // andq $-64, %r8 + LONG $0x070c8d4a // leaq (%rdi,%r8), %rcx + WORD $0x8941; BYTE $0xf1 // movl %esi, %r9d + WORD $0x0941; BYTE $0xf9 // orl %edi, %r9d + LONG $0x1fc1f641 // testb $31, %r9b + JE LBB0_11 + WORD $0x854d; BYTE $0xc0 // testq %r8, %r8 + JE LBB0_2 + LONG $0xc0c28348 // addq $-64, %rdx + WORD $0xc2f6; BYTE $0x40 // testb $64, %dl + JNE LBB0_6 + LONG $0x0710fcc5 // vmovups (%rdi), %ymm0 + LONG $0x065cfcc5 // vsubps (%rsi), %ymm0, %ymm0 + LONG $0x4f10fcc5; BYTE $0x20 // vmovups 32(%rdi), %ymm1 + LONG $0x4e5cf4c5; BYTE $0x20 // vsubps 32(%rsi), %ymm1, %ymm1 + LONG $0xc059fcc5 // vmulps %ymm0, %ymm0, %ymm0 + LONG $0xc959f4c5 // vmulps %ymm1, %ymm1, %ymm1 + LONG $0x40c78348 // addq $64, %rdi + LONG $0x40c68348 // addq $64, %rsi + WORD $0x8949; BYTE $0xf8 // movq %rdi, %r8 + WORD $0x8948; BYTE $0xcf // movq %rcx, %rdi + LONG $0x40fa8348 // cmpq $64, %rdx + JAE LBB0_9 + JMP LBB0_3 + +LBB0_11: + WORD $0x854d; BYTE $0xc0 // testq %r8, %r8 + JE LBB0_12 + LONG $0xc0c28348 // addq $-64, %rdx + WORD $0xc2f6; BYTE $0x40 // testb $64, %dl + JNE LBB0_14 + LONG $0x0728fcc5 // vmovaps (%rdi), %ymm0 + LONG $0x065cfcc5 // vsubps (%rsi), %ymm0, %ymm0 + LONG $0x4f28fcc5; BYTE $0x20 // vmovaps 32(%rdi), %ymm1 + LONG $0x4e5cf4c5; BYTE $0x20 // vsubps 32(%rsi), %ymm1, %ymm1 + LONG $0xc059fcc5 // vmulps %ymm0, %ymm0, %ymm0 + LONG $0xc959f4c5 // vmulps %ymm1, %ymm1, %ymm1 + LONG $0x40c78348 // addq $64, %rdi + LONG $0x40c68348 // addq $64, %rsi + WORD $0x8949; BYTE $0xf8 // movq %rdi, %r8 + WORD $0x8948; BYTE $0xcf // movq %rcx, %rdi + LONG $0x40fa8348 // cmpq $64, %rdx + JAE LBB0_17 + JMP LBB0_19 + +LBB0_2: + LONG $0xc957f0c5 // vxorps %xmm1, %xmm1, %xmm1 + LONG $0xc057f8c5 // vxorps %xmm0, %xmm0, %xmm0 + JMP LBB0_3 + +LBB0_6: + LONG $0xc057f8c5 // vxorps %xmm0, %xmm0, %xmm0 + WORD $0x8949; BYTE $0xf8 // movq %rdi, %r8 + LONG $0xc957f0c5 // vxorps %xmm1, %xmm1, %xmm1 + WORD $0x8948; BYTE $0xcf // movq %rcx, %rdi + LONG $0x40fa8348 // cmpq $64, %rdx + JB LBB0_3 + +LBB0_9: + LONG $0x107cc1c4; BYTE $0x10 // vmovups (%r8), %ymm2 + LONG $0x107cc1c4; WORD $0x2058 // vmovups 32(%r8), %ymm3 + LONG $0x107cc1c4; WORD $0x4060 // vmovups 64(%r8), %ymm4 + LONG $0x165cecc5 // vsubps (%rsi), %ymm2, %ymm2 + LONG $0x5e5ce4c5; BYTE $0x20 // vsubps 32(%rsi), %ymm3, %ymm3 + LONG $0x107cc1c4; WORD $0x6068 // vmovups 96(%r8), %ymm5 + LONG $0xd259ecc5 // vmulps %ymm2, %ymm2, %ymm2 + LONG $0xc258fcc5 // vaddps %ymm2, %ymm0, %ymm0 + LONG $0xd359e4c5 // vmulps %ymm3, %ymm3, %ymm2 + LONG $0x5e5cdcc5; BYTE $0x40 // vsubps 64(%rsi), %ymm4, %ymm3 + LONG $0x665cd4c5; BYTE $0x60 // vsubps 96(%rsi), %ymm5, %ymm4 + LONG $0xca58f4c5 // vaddps %ymm2, %ymm1, %ymm1 + LONG $0xd359e4c5 // vmulps %ymm3, %ymm3, %ymm2 + LONG $0xc258fcc5 // vaddps %ymm2, %ymm0, %ymm0 + LONG $0xd459dcc5 // vmulps %ymm4, %ymm4, %ymm2 + LONG $0xca58f4c5 // vaddps %ymm2, %ymm1, %ymm1 + LONG $0x80e88349 // subq $-128, %r8 + LONG $0x80ee8348 // subq $-128, %rsi + WORD $0x3949; BYTE $0xc8 // cmpq %rcx, %r8 + JNE LBB0_9 + WORD $0x8948; BYTE $0xcf // movq %rcx, %rdi + +LBB0_3: + LONG $0x20c18348 // addq $32, %rcx + WORD $0x3948; BYTE $0xc8 // cmpq %rcx, %rax + JB LBB0_22 + LONG $0x1710fcc5 // vmovups (%rdi), %ymm2 + JMP LBB0_21 + +LBB0_12: + LONG $0xc957f0c5 // vxorps %xmm1, %xmm1, %xmm1 + LONG $0xc057f8c5 // vxorps %xmm0, %xmm0, %xmm0 + JMP LBB0_19 + +LBB0_14: + LONG $0xc057f8c5 // vxorps %xmm0, %xmm0, %xmm0 + WORD $0x8949; BYTE $0xf8 // movq %rdi, %r8 + LONG $0xc957f0c5 // vxorps %xmm1, %xmm1, %xmm1 + WORD $0x8948; BYTE $0xcf // movq %rcx, %rdi + LONG $0x40fa8348 // cmpq $64, %rdx + JB LBB0_19 + +LBB0_17: + LONG $0x287cc1c4; BYTE $0x10 // vmovaps (%r8), %ymm2 + LONG $0x287cc1c4; WORD $0x2058 // vmovaps 32(%r8), %ymm3 + LONG $0x287cc1c4; WORD $0x4060 // vmovaps 64(%r8), %ymm4 + LONG $0x165cecc5 // vsubps (%rsi), %ymm2, %ymm2 + LONG $0x5e5ce4c5; BYTE $0x20 // vsubps 32(%rsi), %ymm3, %ymm3 + LONG $0x287cc1c4; WORD $0x6068 // vmovaps 96(%r8), %ymm5 + LONG $0xd259ecc5 // vmulps %ymm2, %ymm2, %ymm2 + LONG $0xc258fcc5 // vaddps %ymm2, %ymm0, %ymm0 + LONG $0xd359e4c5 // vmulps %ymm3, %ymm3, %ymm2 + LONG $0x5e5cdcc5; BYTE $0x40 // vsubps 64(%rsi), %ymm4, %ymm3 + LONG $0x665cd4c5; BYTE $0x60 // vsubps 96(%rsi), %ymm5, %ymm4 + LONG $0xca58f4c5 // vaddps %ymm2, %ymm1, %ymm1 + LONG $0xd359e4c5 // vmulps %ymm3, %ymm3, %ymm2 + LONG $0xc258fcc5 // vaddps %ymm2, %ymm0, %ymm0 + LONG $0xd459dcc5 // vmulps %ymm4, %ymm4, %ymm2 + LONG $0xca58f4c5 // vaddps %ymm2, %ymm1, %ymm1 + LONG $0x80e88349 // subq $-128, %r8 + LONG $0x80ee8348 // subq $-128, %rsi + WORD $0x3949; BYTE $0xc8 // cmpq %rcx, %r8 + JNE LBB0_17 + WORD $0x8948; BYTE $0xcf // movq %rcx, %rdi + +LBB0_19: + LONG $0x20c18348 // addq $32, %rcx + WORD $0x3948; BYTE $0xc8 // cmpq %rcx, %rax + JB LBB0_22 + LONG $0x1728fcc5 // vmovaps (%rdi), %ymm2 + +LBB0_21: + LONG $0x165cecc5 // vsubps (%rsi), %ymm2, %ymm2 + LONG $0xd259ecc5 // vmulps %ymm2, %ymm2, %ymm2 + LONG $0xc258fcc5 // vaddps %ymm2, %ymm0, %ymm0 + LONG $0x20c78348 // addq $32, %rdi + LONG $0x20c68348 // addq $32, %rsi + +LBB0_22: + LONG $0xc058f4c5 // vaddps %ymm0, %ymm1, %ymm0 + LONG $0xc07cffc5 // vhaddps %ymm0, %ymm0, %ymm0 + LONG $0xc07cffc5 // vhaddps %ymm0, %ymm0, %ymm0 + LONG $0x197de3c4; WORD $0x01c1 // vextractf128 $1, %ymm0, %xmm1 + LONG $0xc158fac5 // vaddss %xmm1, %xmm0, %xmm0 + WORD $0x2948; BYTE $0xf8 // subq %rdi, %rax + LONG $0x02f8c148 // sarq $2, %rax + LONG $0x03f88348 // cmpq $3, %rax + JLE LBB0_23 + LONG $0x05f88348 // cmpq $5, %rax + JG LBB0_29 + LONG $0x04f88348 // cmpq $4, %rax + JE LBB0_34 + LONG $0x05f88348 // cmpq $5, %rax + JE LBB0_33 + +LBB0_38: + WORD $0x8948; BYTE $0xec // movq %rbp, %rsp + BYTE $0x5d // popq %rbp + WORD $0xf8c5; BYTE $0x77 // vzeroupper + MOVSS X0, result+24(FP) + RET + +LBB0_23: + LONG $0x01f88348 // cmpq $1, %rax + JE LBB0_37 + LONG $0x02f88348 // cmpq $2, %rax + JE LBB0_36 + LONG $0x03f88348 // cmpq $3, %rax + JE LBB0_35 + JMP LBB0_38 + +LBB0_29: + LONG $0x06f88348 // cmpq $6, %rax + JE LBB0_32 + LONG $0x07f88348 // cmpq $7, %rax + JNE LBB0_38 + LONG $0x4f10fac5; BYTE $0x18 // vmovss 24(%rdi), %xmm1 # xmm1 = mem[0],zero,zero,zero + LONG $0x4e5cf2c5; BYTE $0x18 // vsubss 24(%rsi), %xmm1, %xmm1 + LONG $0xc959f2c5 // vmulss %xmm1, %xmm1, %xmm1 + LONG $0xc058f2c5 // vaddss %xmm0, %xmm1, %xmm0 + +LBB0_32: + LONG $0x4f10fac5; BYTE $0x14 // vmovss 20(%rdi), %xmm1 # xmm1 = mem[0],zero,zero,zero + LONG $0x4e5cf2c5; BYTE $0x14 // vsubss 20(%rsi), %xmm1, %xmm1 + LONG $0xc959f2c5 // vmulss %xmm1, %xmm1, %xmm1 + LONG $0xc058f2c5 // vaddss %xmm0, %xmm1, %xmm0 + +LBB0_33: + LONG $0x4f10fac5; BYTE $0x10 // vmovss 16(%rdi), %xmm1 # xmm1 = mem[0],zero,zero,zero + LONG $0x4e5cf2c5; BYTE $0x10 // vsubss 16(%rsi), %xmm1, %xmm1 + LONG $0xc959f2c5 // vmulss %xmm1, %xmm1, %xmm1 + LONG $0xc058f2c5 // vaddss %xmm0, %xmm1, %xmm0 + +LBB0_34: + LONG $0x4f10fac5; BYTE $0x0c // vmovss 12(%rdi), %xmm1 # xmm1 = mem[0],zero,zero,zero + LONG $0x4e5cf2c5; BYTE $0x0c // vsubss 12(%rsi), %xmm1, %xmm1 + LONG $0xc959f2c5 // vmulss %xmm1, %xmm1, %xmm1 + LONG $0xc058f2c5 // vaddss %xmm0, %xmm1, %xmm0 + +LBB0_35: + LONG $0x4f10fac5; BYTE $0x08 // vmovss 8(%rdi), %xmm1 # xmm1 = mem[0],zero,zero,zero + LONG $0x4e5cf2c5; BYTE $0x08 // vsubss 8(%rsi), %xmm1, %xmm1 + LONG $0xc959f2c5 // vmulss %xmm1, %xmm1, %xmm1 + LONG $0xc058f2c5 // vaddss %xmm0, %xmm1, %xmm0 + +LBB0_36: + LONG $0x4f10fac5; BYTE $0x04 // vmovss 4(%rdi), %xmm1 # xmm1 = mem[0],zero,zero,zero + LONG $0x4e5cf2c5; BYTE $0x04 // vsubss 4(%rsi), %xmm1, %xmm1 + LONG $0xc959f2c5 // vmulss %xmm1, %xmm1, %xmm1 + LONG $0xc058f2c5 // vaddss %xmm0, %xmm1, %xmm0 + +LBB0_37: + LONG $0x0f10fac5 // vmovss (%rdi), %xmm1 # xmm1 = mem[0],zero,zero,zero + LONG $0x0e5cf2c5 // vsubss (%rsi), %xmm1, %xmm1 + LONG $0xc959f2c5 // vmulss %xmm1, %xmm1, %xmm1 + LONG $0xc058f2c5 // vaddss %xmm0, %xmm1, %xmm0 + WORD $0x8948; BYTE $0xec // movq %rbp, %rsp + BYTE $0x5d // popq %rbp + WORD $0xf8c5; BYTE $0x77 // vzeroupper + MOVSS X0, result+24(FP) + RET diff --git a/internal/ailego/math/euclidean_distance_matrix_fp32_avx512.go b/internal/ailego/math/euclidean_distance_matrix_fp32_avx512.go new file mode 100644 index 0000000..2915882 --- /dev/null +++ b/internal/ailego/math/euclidean_distance_matrix_fp32_avx512.go @@ -0,0 +1,14 @@ +//go:build !noasm && amd64 +// Code generated by GoAT. DO NOT EDIT. +// versions: +// clang 21.1.8 (6ubuntu1) +// objdump 2.46 +// flags: -mavx -mfma -mavx512f -O3 +// source: src/euclidean_distance_matrix_fp32_avx512.c + +package mathutil + +import "unsafe" + +//go:noescape +func squared_euclidean_distance_fp32_avx512(lhs, rhs unsafe.Pointer, size int64) (result float32) diff --git a/internal/ailego/math/euclidean_distance_matrix_fp32_avx512.s b/internal/ailego/math/euclidean_distance_matrix_fp32_avx512.s new file mode 100644 index 0000000..9ba0294 --- /dev/null +++ b/internal/ailego/math/euclidean_distance_matrix_fp32_avx512.s @@ -0,0 +1,175 @@ +//go:build !noasm && amd64 +// Code generated by GoAT. DO NOT EDIT. +// versions: +// clang 21.1.8 (6ubuntu1) +// objdump 2.46 +// flags: -mavx -mfma -mavx512f -O3 +// source: src/euclidean_distance_matrix_fp32_avx512.c + +TEXT ·squared_euclidean_distance_fp32_avx512(SB), $8-28 + MOVQ lhs+0(FP), DI + MOVQ rhs+8(FP), SI + MOVQ size+16(FP), DX + BYTE $0x55 // pushq %rbp + WORD $0x8948; BYTE $0xe5 // movq %rsp, %rbp + LONG $0xf8e48348 // andq $-8, %rsp + LONG $0x970c8d48 // leaq (%rdi,%rdx,4), %rcx + LONG $0x02e2c148 // shlq $2, %rdx + WORD $0x8949; BYTE $0xd0 // movq %rdx, %r8 + LONG $0x80e08349 // andq $-128, %r8 + LONG $0x07048d4a // leaq (%rdi,%r8), %rax + WORD $0x8941; BYTE $0xf1 // movl %esi, %r9d + WORD $0x0941; BYTE $0xf9 // orl %edi, %r9d + LONG $0x3fc1f641 // testb $63, %r9b + JE LBB0_11 + WORD $0x854d; BYTE $0xc0 // testq %r8, %r8 + JE LBB0_2 + LONG $0x80c28348 // addq $-128, %rdx + WORD $0xd284 // testb %dl, %dl + JS LBB0_6 + LONG $0x487cf162; WORD $0x0710 // vmovups (%rdi), %zmm0 + LONG $0x487cf162; WORD $0x4f10; BYTE $0x01 // vmovups 64(%rdi), %zmm1 + LONG $0x487cf162; WORD $0x065c // vsubps (%rsi), %zmm0, %zmm0 + LONG $0x4874f162; WORD $0x4e5c; BYTE $0x01 // vsubps 64(%rsi), %zmm1, %zmm1 + LONG $0xd257e8c5 // vxorps %xmm2, %xmm2, %xmm2 + LONG $0x487df262; WORD $0xc2a8 // vfmadd213ps %zmm2, %zmm0, %zmm0 # zmm0 = (zmm0 * zmm0) + zmm2 + LONG $0x4875f262; WORD $0xcaa8 // vfmadd213ps %zmm2, %zmm1, %zmm1 # zmm1 = (zmm1 * zmm1) + zmm2 + LONG $0x80ef8348 // subq $-128, %rdi + LONG $0x80ee8348 // subq $-128, %rsi + WORD $0x8949; BYTE $0xf8 // movq %rdi, %r8 + WORD $0x8948; BYTE $0xc7 // movq %rax, %rdi + LONG $0x80fa8148; WORD $0x0000; BYTE $0x00 // cmpq $128, %rdx + JAE LBB0_9 + JMP LBB0_3 + +LBB0_11: + WORD $0x854d; BYTE $0xc0 // testq %r8, %r8 + JE LBB0_12 + LONG $0x80c28348 // addq $-128, %rdx + WORD $0xd284 // testb %dl, %dl + JS LBB0_14 + LONG $0x487cf162; WORD $0x0728 // vmovaps (%rdi), %zmm0 + LONG $0x487cf162; WORD $0x4f28; BYTE $0x01 // vmovaps 64(%rdi), %zmm1 + LONG $0x487cf162; WORD $0x065c // vsubps (%rsi), %zmm0, %zmm0 + LONG $0x4874f162; WORD $0x4e5c; BYTE $0x01 // vsubps 64(%rsi), %zmm1, %zmm1 + LONG $0xd257e8c5 // vxorps %xmm2, %xmm2, %xmm2 + LONG $0x487df262; WORD $0xc2a8 // vfmadd213ps %zmm2, %zmm0, %zmm0 # zmm0 = (zmm0 * zmm0) + zmm2 + LONG $0x4875f262; WORD $0xcaa8 // vfmadd213ps %zmm2, %zmm1, %zmm1 # zmm1 = (zmm1 * zmm1) + zmm2 + LONG $0x80ef8348 // subq $-128, %rdi + LONG $0x80ee8348 // subq $-128, %rsi + WORD $0x8949; BYTE $0xf8 // movq %rdi, %r8 + WORD $0x8948; BYTE $0xc7 // movq %rax, %rdi + LONG $0x80fa8148; WORD $0x0000; BYTE $0x00 // cmpq $128, %rdx + JAE LBB0_17 + JMP LBB0_19 + +LBB0_2: + LONG $0xc957f0c5 // vxorps %xmm1, %xmm1, %xmm1 + LONG $0xc057f8c5 // vxorps %xmm0, %xmm0, %xmm0 + JMP LBB0_3 + +LBB0_6: + LONG $0xc057f8c5 // vxorps %xmm0, %xmm0, %xmm0 + WORD $0x8949; BYTE $0xf8 // movq %rdi, %r8 + LONG $0xc957f0c5 // vxorps %xmm1, %xmm1, %xmm1 + WORD $0x8948; BYTE $0xc7 // movq %rax, %rdi + LONG $0x80fa8148; WORD $0x0000; BYTE $0x00 // cmpq $128, %rdx + JB LBB0_3 + +LBB0_9: + LONG $0x487cd162; WORD $0x1010 // vmovups (%r8), %zmm2 + LONG $0x487cd162; WORD $0x5810; BYTE $0x01 // vmovups 64(%r8), %zmm3 + LONG $0x487cd162; WORD $0x6010; BYTE $0x02 // vmovups 128(%r8), %zmm4 + LONG $0x486cf162; WORD $0x165c // vsubps (%rsi), %zmm2, %zmm2 + LONG $0x4864f162; WORD $0x5e5c; BYTE $0x01 // vsubps 64(%rsi), %zmm3, %zmm3 + LONG $0x487cd162; WORD $0x6810; BYTE $0x03 // vmovups 192(%r8), %zmm5 + LONG $0x486df262; WORD $0xd0a8 // vfmadd213ps %zmm0, %zmm2, %zmm2 # zmm2 = (zmm2 * zmm2) + zmm0 + LONG $0x485cf162; WORD $0x465c; BYTE $0x02 // vsubps 128(%rsi), %zmm4, %zmm0 + LONG $0x4865f262; WORD $0xd9a8 // vfmadd213ps %zmm1, %zmm3, %zmm3 # zmm3 = (zmm3 * zmm3) + zmm1 + LONG $0x4854f162; WORD $0x4e5c; BYTE $0x03 // vsubps 192(%rsi), %zmm5, %zmm1 + LONG $0x487df262; WORD $0xc2a8 // vfmadd213ps %zmm2, %zmm0, %zmm0 # zmm0 = (zmm0 * zmm0) + zmm2 + LONG $0x4875f262; WORD $0xcba8 // vfmadd213ps %zmm3, %zmm1, %zmm1 # zmm1 = (zmm1 * zmm1) + zmm3 + LONG $0x00c08149; WORD $0x0001; BYTE $0x00 // addq $256, %r8 # imm = 0x100 + LONG $0x00c68148; WORD $0x0001; BYTE $0x00 // addq $256, %rsi # imm = 0x100 + WORD $0x3949; BYTE $0xc0 // cmpq %rax, %r8 + JNE LBB0_9 + WORD $0x8948; BYTE $0xc7 // movq %rax, %rdi + +LBB0_3: + LONG $0x40c08348 // addq $64, %rax + WORD $0x3948; BYTE $0xc1 // cmpq %rax, %rcx + JB LBB0_22 + LONG $0x487cf162; WORD $0x1710 // vmovups (%rdi), %zmm2 + JMP LBB0_21 + +LBB0_12: + LONG $0xc957f0c5 // vxorps %xmm1, %xmm1, %xmm1 + LONG $0xc057f8c5 // vxorps %xmm0, %xmm0, %xmm0 + JMP LBB0_19 + +LBB0_14: + LONG $0xc057f8c5 // vxorps %xmm0, %xmm0, %xmm0 + WORD $0x8949; BYTE $0xf8 // movq %rdi, %r8 + LONG $0xc957f0c5 // vxorps %xmm1, %xmm1, %xmm1 + WORD $0x8948; BYTE $0xc7 // movq %rax, %rdi + LONG $0x80fa8148; WORD $0x0000; BYTE $0x00 // cmpq $128, %rdx + JB LBB0_19 + +LBB0_17: + LONG $0x487cd162; WORD $0x1028 // vmovaps (%r8), %zmm2 + LONG $0x487cd162; WORD $0x5828; BYTE $0x01 // vmovaps 64(%r8), %zmm3 + LONG $0x487cd162; WORD $0x6028; BYTE $0x02 // vmovaps 128(%r8), %zmm4 + LONG $0x486cf162; WORD $0x165c // vsubps (%rsi), %zmm2, %zmm2 + LONG $0x4864f162; WORD $0x5e5c; BYTE $0x01 // vsubps 64(%rsi), %zmm3, %zmm3 + LONG $0x487cd162; WORD $0x6828; BYTE $0x03 // vmovaps 192(%r8), %zmm5 + LONG $0x486df262; WORD $0xd0a8 // vfmadd213ps %zmm0, %zmm2, %zmm2 # zmm2 = (zmm2 * zmm2) + zmm0 + LONG $0x485cf162; WORD $0x465c; BYTE $0x02 // vsubps 128(%rsi), %zmm4, %zmm0 + LONG $0x4865f262; WORD $0xd9a8 // vfmadd213ps %zmm1, %zmm3, %zmm3 # zmm3 = (zmm3 * zmm3) + zmm1 + LONG $0x4854f162; WORD $0x4e5c; BYTE $0x03 // vsubps 192(%rsi), %zmm5, %zmm1 + LONG $0x487df262; WORD $0xc2a8 // vfmadd213ps %zmm2, %zmm0, %zmm0 # zmm0 = (zmm0 * zmm0) + zmm2 + LONG $0x4875f262; WORD $0xcba8 // vfmadd213ps %zmm3, %zmm1, %zmm1 # zmm1 = (zmm1 * zmm1) + zmm3 + LONG $0x00c08149; WORD $0x0001; BYTE $0x00 // addq $256, %r8 # imm = 0x100 + LONG $0x00c68148; WORD $0x0001; BYTE $0x00 // addq $256, %rsi # imm = 0x100 + WORD $0x3949; BYTE $0xc0 // cmpq %rax, %r8 + JNE LBB0_17 + WORD $0x8948; BYTE $0xc7 // movq %rax, %rdi + +LBB0_19: + LONG $0x40c08348 // addq $64, %rax + WORD $0x3948; BYTE $0xc1 // cmpq %rax, %rcx + JB LBB0_22 + LONG $0x487cf162; WORD $0x1728 // vmovaps (%rdi), %zmm2 + +LBB0_21: + LONG $0x486cf162; WORD $0x165c // vsubps (%rsi), %zmm2, %zmm2 + LONG $0x486df262; WORD $0xc2b8 // vfmadd231ps %zmm2, %zmm2, %zmm0 # zmm0 = (zmm2 * zmm2) + zmm0 + LONG $0x40c78348 // addq $64, %rdi + LONG $0x40c68348 // addq $64, %rsi + +LBB0_22: + LONG $0x4874f162; WORD $0xc058 // vaddps %zmm0, %zmm1, %zmm0 + WORD $0x3948; BYTE $0xcf // cmpq %rcx, %rdi + JE LBB0_24 + WORD $0xf929 // subl %edi, %ecx + WORD $0xe9c1; BYTE $0x02 // shrl $2, %ecx + LONG $0xffffffb8; BYTE $0xff // movl $-1, %eax + WORD $0xe0d3 // shll %cl, %eax + WORD $0xd0f7 // notl %eax + LONG $0xc892f8c5 // kmovw %eax, %k1 + LONG $0xc97cf162; WORD $0x0f10 // vmovups (%rdi), %zmm1 {%k1} {z} + LONG $0xc97cf162; WORD $0x1610 // vmovups (%rsi), %zmm2 {%k1} {z} + LONG $0xc974f162; WORD $0xca5c // vsubps %zmm2, %zmm1, %zmm1 {%k1} {z} + LONG $0x4975f262; WORD $0xc1b8 // vfmadd231ps %zmm1, %zmm1, %zmm0 {%k1} # zmm0 {%k1} = (zmm1 * zmm1) + zmm0 + +LBB0_24: + LONG $0x48fdf362; WORD $0xc11b; BYTE $0x01 // vextractf64x4 $1, %zmm0, %ymm1 + LONG $0xc158fcc5 // vaddps %ymm1, %ymm0, %ymm0 + LONG $0xc07cffc5 // vhaddps %ymm0, %ymm0, %ymm0 + LONG $0xc07cffc5 // vhaddps %ymm0, %ymm0, %ymm0 + LONG $0x197de3c4; WORD $0x01c1 // vextractf128 $1, %ymm0, %xmm1 + LONG $0xc158fac5 // vaddss %xmm1, %xmm0, %xmm0 + WORD $0x8948; BYTE $0xec // movq %rbp, %rsp + BYTE $0x5d // popq %rbp + WORD $0xf8c5; BYTE $0x77 // vzeroupper + MOVSS X0, result+24(FP) + RET diff --git a/internal/ailego/math/euclidean_distance_matrix_fp32_neon.go b/internal/ailego/math/euclidean_distance_matrix_fp32_neon.go new file mode 100644 index 0000000..d97fa45 --- /dev/null +++ b/internal/ailego/math/euclidean_distance_matrix_fp32_neon.go @@ -0,0 +1,14 @@ +//go:build !noasm && arm64 +// Code generated by GoAT. DO NOT EDIT. +// versions: +// clang 21.1.8 (6ubuntu1) +// objdump 2.46 +// flags: -O3 +// source: src/euclidean_distance_matrix_fp32_neon.c + +package mathutil + +import "unsafe" + +//go:noescape +func squared_euclidean_distance_fp32_neon(lhs, rhs unsafe.Pointer, size int64) (result float32) diff --git a/internal/ailego/math/euclidean_distance_matrix_fp32_neon.s b/internal/ailego/math/euclidean_distance_matrix_fp32_neon.s new file mode 100644 index 0000000..384ee7b --- /dev/null +++ b/internal/ailego/math/euclidean_distance_matrix_fp32_neon.s @@ -0,0 +1,80 @@ +//go:build !noasm && arm64 +// Code generated by GoAT. DO NOT EDIT. +// versions: +// clang 21.1.8 (6ubuntu1) +// objdump 2.46 +// flags: -O3 +// source: src/euclidean_distance_matrix_fp32_neon.c + +TEXT ·squared_euclidean_distance_fp32_neon(SB), $0-28 + MOVD lhs+0(FP), R0 + MOVD rhs+8(FP), R1 + MOVD size+16(FP), R2 + WORD $0xd37ef448 // lsl x8, x2, #2 + WORD $0xf27be909 // ands x9, x8, #0xffffffffffffffe0 + WORD $0x8b080008 // add x8, x0, x8 + WORD $0x8b090009 // add x9, x0, x9 + BEQ LBB0_12 + WORD $0x6f00e400 // movi v0.2d, #0000000000000000 + WORD $0x6f00e401 // movi v1.2d, #0000000000000000 + +LBB0_2: + WORD $0xacc10c02 // ldp q2, q3, [x0], #32 + WORD $0xacc11424 // ldp q4, q5, [x1], #32 + WORD $0xeb09001f // cmp x0, x9 + WORD $0x4ea4d442 // fsub v2.4s, v2.4s, v4.4s + WORD $0x4ea5d463 // fsub v3.4s, v3.4s, v5.4s + WORD $0x4e22cc40 // fmla v0.4s, v2.4s, v2.4s + WORD $0x4e23cc61 // fmla v1.4s, v3.4s, v3.4s + BNE LBB0_2 + WORD $0xaa0903e0 // mov x0, x9 + WORD $0x91004129 // add x9, x9, #16 + WORD $0xeb09011f // cmp x8, x9 + BLO LBB0_5 + +LBB0_4: + WORD $0x3cc10402 // ldr q2, [x0], #16 + WORD $0x3cc10423 // ldr q3, [x1], #16 + WORD $0x4ea3d442 // fsub v2.4s, v2.4s, v3.4s + WORD $0x4e22cc40 // fmla v0.4s, v2.4s, v2.4s + +LBB0_5: + WORD $0x4e20d420 // fadd v0.4s, v1.4s, v0.4s + WORD $0xcb000108 // sub x8, x8, x0 + WORD $0x9342fd08 // asr x8, x8, #2 + WORD $0xf100051f // cmp x8, #1 + WORD $0x6e20d400 // faddp v0.4s, v0.4s, v0.4s + WORD $0x7e30d800 // faddp s0, v0.2s + BEQ LBB0_10 + WORD $0xf100091f // cmp x8, #2 + BEQ LBB0_9 + WORD $0xf1000d1f // cmp x8, #3 + BNE LBB0_11 + WORD $0xbd400801 // ldr s1, [x0, #8] + WORD $0xbd400822 // ldr s2, [x1, #8] + WORD $0x1e223821 // fsub s1, s1, s2 + WORD $0x1f010020 // fmadd s0, s1, s1, s0 + +LBB0_9: + WORD $0xbd400401 // ldr s1, [x0, #4] + WORD $0xbd400422 // ldr s2, [x1, #4] + WORD $0x1e223821 // fsub s1, s1, s2 + WORD $0x1f010020 // fmadd s0, s1, s1, s0 + +LBB0_10: + WORD $0xbd400001 // ldr s1, [x0] + WORD $0xbd400022 // ldr s2, [x1] + WORD $0x1e223821 // fsub s1, s1, s2 + WORD $0x1f010020 // fmadd s0, s1, s1, s0 + +LBB0_11: + FMOVS F0, result+24(FP) + RET + +LBB0_12: + WORD $0x6f00e401 // movi v1.2d, #0000000000000000 + WORD $0x6f00e400 // movi v0.2d, #0000000000000000 + WORD $0x91004129 // add x9, x9, #16 + WORD $0xeb09011f // cmp x8, x9 + BHS LBB0_4 + B LBB0_5 diff --git a/internal/ailego/math/euclidean_distance_matrix_fp32_sse.go b/internal/ailego/math/euclidean_distance_matrix_fp32_sse.go new file mode 100644 index 0000000..99db00d --- /dev/null +++ b/internal/ailego/math/euclidean_distance_matrix_fp32_sse.go @@ -0,0 +1,14 @@ +//go:build !noasm && amd64 +// Code generated by GoAT. DO NOT EDIT. +// versions: +// clang 21.1.8 (6ubuntu1) +// objdump 2.46 +// flags: -msse2 -O3 +// source: src/euclidean_distance_matrix_fp32_sse.c + +package mathutil + +import "unsafe" + +//go:noescape +func squared_euclidean_distance_fp32_sse(lhs, rhs unsafe.Pointer, size int64) (result float32) diff --git a/internal/ailego/math/euclidean_distance_matrix_fp32_sse.s b/internal/ailego/math/euclidean_distance_matrix_fp32_sse.s new file mode 100644 index 0000000..54d0e87 --- /dev/null +++ b/internal/ailego/math/euclidean_distance_matrix_fp32_sse.s @@ -0,0 +1,202 @@ +//go:build !noasm && amd64 +// Code generated by GoAT. DO NOT EDIT. +// versions: +// clang 21.1.8 (6ubuntu1) +// objdump 2.46 +// flags: -msse2 -O3 +// source: src/euclidean_distance_matrix_fp32_sse.c + +TEXT ·squared_euclidean_distance_fp32_sse(SB), $8-28 + MOVQ lhs+0(FP), DI + MOVQ rhs+8(FP), SI + MOVQ size+16(FP), DX + BYTE $0x55 // pushq %rbp + WORD $0x8948; BYTE $0xe5 // movq %rsp, %rbp + LONG $0xf8e48348 // andq $-8, %rsp + LONG $0x97048d48 // leaq (%rdi,%rdx,4), %rax + LONG $0x02e2c148 // shlq $2, %rdx + WORD $0x8949; BYTE $0xd0 // movq %rdx, %r8 + LONG $0xe0e08349 // andq $-32, %r8 + LONG $0x070c8d4a // leaq (%rdi,%r8), %rcx + WORD $0x8941; BYTE $0xf1 // movl %esi, %r9d + WORD $0x0941; BYTE $0xf9 // orl %edi, %r9d + LONG $0x0fc1f641 // testb $15, %r9b + JE LBB0_11 + WORD $0x854d; BYTE $0xc0 // testq %r8, %r8 + JE LBB0_2 + LONG $0xe0c28348 // addq $-32, %rdx + WORD $0xc2f6; BYTE $0x20 // testb $32, %dl + JNE LBB0_6 + WORD $0x100f; BYTE $0x07 // movups (%rdi), %xmm0 + LONG $0x104f100f // movups 16(%rdi), %xmm1 + WORD $0x100f; BYTE $0x16 // movups (%rsi), %xmm2 + WORD $0x5c0f; BYTE $0xc2 // subps %xmm2, %xmm0 + LONG $0x1056100f // movups 16(%rsi), %xmm2 + WORD $0x5c0f; BYTE $0xca // subps %xmm2, %xmm1 + WORD $0x590f; BYTE $0xc0 // mulps %xmm0, %xmm0 + WORD $0x590f; BYTE $0xc9 // mulps %xmm1, %xmm1 + LONG $0x20c78348 // addq $32, %rdi + LONG $0x20c68348 // addq $32, %rsi + WORD $0x8949; BYTE $0xf8 // movq %rdi, %r8 + WORD $0x8948; BYTE $0xcf // movq %rcx, %rdi + LONG $0x20fa8348 // cmpq $32, %rdx + JAE LBB0_9 + JMP LBB0_3 + +LBB0_11: + WORD $0x854d; BYTE $0xc0 // testq %r8, %r8 + JE LBB0_12 + LONG $0xe0c28348 // addq $-32, %rdx + WORD $0xc2f6; BYTE $0x20 // testb $32, %dl + JNE LBB0_14 + WORD $0x280f; BYTE $0x07 // movaps (%rdi), %xmm0 + WORD $0x5c0f; BYTE $0x06 // subps (%rsi), %xmm0 + LONG $0x104f280f // movaps 16(%rdi), %xmm1 + LONG $0x104e5c0f // subps 16(%rsi), %xmm1 + WORD $0x590f; BYTE $0xc0 // mulps %xmm0, %xmm0 + WORD $0x590f; BYTE $0xc9 // mulps %xmm1, %xmm1 + LONG $0x20c78348 // addq $32, %rdi + LONG $0x20c68348 // addq $32, %rsi + WORD $0x8949; BYTE $0xf8 // movq %rdi, %r8 + WORD $0x8948; BYTE $0xcf // movq %rcx, %rdi + LONG $0x20fa8348 // cmpq $32, %rdx + JAE LBB0_17 + JMP LBB0_19 + +LBB0_2: + WORD $0x570f; BYTE $0xc9 // xorps %xmm1, %xmm1 + WORD $0x570f; BYTE $0xc0 // xorps %xmm0, %xmm0 + JMP LBB0_3 + +LBB0_6: + WORD $0x570f; BYTE $0xc0 // xorps %xmm0, %xmm0 + WORD $0x8949; BYTE $0xf8 // movq %rdi, %r8 + WORD $0x570f; BYTE $0xc9 // xorps %xmm1, %xmm1 + WORD $0x8948; BYTE $0xcf // movq %rcx, %rdi + LONG $0x20fa8348 // cmpq $32, %rdx + JB LBB0_3 + +LBB0_9: + LONG $0x18100f41 // movups (%r8), %xmm3 + LONG $0x50100f41; BYTE $0x10 // movups 16(%r8), %xmm2 + WORD $0x100f; BYTE $0x26 // movups (%rsi), %xmm4 + WORD $0x5c0f; BYTE $0xdc // subps %xmm4, %xmm3 + WORD $0x590f; BYTE $0xdb // mulps %xmm3, %xmm3 + WORD $0x580f; BYTE $0xd8 // addps %xmm0, %xmm3 + LONG $0x40100f41; BYTE $0x20 // movups 32(%r8), %xmm0 + LONG $0x1066100f // movups 16(%rsi), %xmm4 + WORD $0x5c0f; BYTE $0xd4 // subps %xmm4, %xmm2 + WORD $0x590f; BYTE $0xd2 // mulps %xmm2, %xmm2 + WORD $0x580f; BYTE $0xd1 // addps %xmm1, %xmm2 + LONG $0x48100f41; BYTE $0x30 // movups 48(%r8), %xmm1 + LONG $0x2066100f // movups 32(%rsi), %xmm4 + WORD $0x5c0f; BYTE $0xc4 // subps %xmm4, %xmm0 + LONG $0x3066100f // movups 48(%rsi), %xmm4 + WORD $0x5c0f; BYTE $0xcc // subps %xmm4, %xmm1 + WORD $0x590f; BYTE $0xc0 // mulps %xmm0, %xmm0 + WORD $0x580f; BYTE $0xc3 // addps %xmm3, %xmm0 + WORD $0x590f; BYTE $0xc9 // mulps %xmm1, %xmm1 + WORD $0x580f; BYTE $0xca // addps %xmm2, %xmm1 + LONG $0x40c08349 // addq $64, %r8 + LONG $0x40c68348 // addq $64, %rsi + WORD $0x3949; BYTE $0xc8 // cmpq %rcx, %r8 + JNE LBB0_9 + WORD $0x8948; BYTE $0xcf // movq %rcx, %rdi + +LBB0_3: + LONG $0x10c18348 // addq $16, %rcx + WORD $0x3948; BYTE $0xc8 // cmpq %rcx, %rax + JB LBB0_22 + WORD $0x100f; BYTE $0x17 // movups (%rdi), %xmm2 + WORD $0x100f; BYTE $0x1e // movups (%rsi), %xmm3 + WORD $0x5c0f; BYTE $0xd3 // subps %xmm3, %xmm2 + JMP LBB0_21 + +LBB0_12: + WORD $0x570f; BYTE $0xc9 // xorps %xmm1, %xmm1 + WORD $0x570f; BYTE $0xc0 // xorps %xmm0, %xmm0 + JMP LBB0_19 + +LBB0_14: + WORD $0x570f; BYTE $0xc0 // xorps %xmm0, %xmm0 + WORD $0x8949; BYTE $0xf8 // movq %rdi, %r8 + WORD $0x570f; BYTE $0xc9 // xorps %xmm1, %xmm1 + WORD $0x8948; BYTE $0xcf // movq %rcx, %rdi + LONG $0x20fa8348 // cmpq $32, %rdx + JB LBB0_19 + +LBB0_17: + LONG $0x10280f41 // movaps (%r8), %xmm2 + LONG $0x58280f41; BYTE $0x10 // movaps 16(%r8), %xmm3 + WORD $0x5c0f; BYTE $0x16 // subps (%rsi), %xmm2 + WORD $0x590f; BYTE $0xd2 // mulps %xmm2, %xmm2 + WORD $0x580f; BYTE $0xd0 // addps %xmm0, %xmm2 + LONG $0x40280f41; BYTE $0x20 // movaps 32(%r8), %xmm0 + LONG $0x105e5c0f // subps 16(%rsi), %xmm3 + WORD $0x590f; BYTE $0xdb // mulps %xmm3, %xmm3 + WORD $0x580f; BYTE $0xd9 // addps %xmm1, %xmm3 + LONG $0x20465c0f // subps 32(%rsi), %xmm0 + LONG $0x48280f41; BYTE $0x30 // movaps 48(%r8), %xmm1 + LONG $0x304e5c0f // subps 48(%rsi), %xmm1 + WORD $0x590f; BYTE $0xc0 // mulps %xmm0, %xmm0 + WORD $0x580f; BYTE $0xc2 // addps %xmm2, %xmm0 + WORD $0x590f; BYTE $0xc9 // mulps %xmm1, %xmm1 + WORD $0x580f; BYTE $0xcb // addps %xmm3, %xmm1 + LONG $0x40c08349 // addq $64, %r8 + LONG $0x40c68348 // addq $64, %rsi + WORD $0x3949; BYTE $0xc8 // cmpq %rcx, %r8 + JNE LBB0_17 + WORD $0x8948; BYTE $0xcf // movq %rcx, %rdi + +LBB0_19: + LONG $0x10c18348 // addq $16, %rcx + WORD $0x3948; BYTE $0xc8 // cmpq %rcx, %rax + JB LBB0_22 + WORD $0x280f; BYTE $0x17 // movaps (%rdi), %xmm2 + WORD $0x5c0f; BYTE $0x16 // subps (%rsi), %xmm2 + +LBB0_21: + WORD $0x590f; BYTE $0xd2 // mulps %xmm2, %xmm2 + WORD $0x580f; BYTE $0xc2 // addps %xmm2, %xmm0 + LONG $0x10c78348 // addq $16, %rdi + LONG $0x10c68348 // addq $16, %rsi + +LBB0_22: + WORD $0x580f; BYTE $0xc8 // addps %xmm0, %xmm1 + WORD $0x280f; BYTE $0xd1 // movaps %xmm1, %xmm2 + LONG $0xd1150f66 // unpckhpd %xmm1, %xmm2 # xmm2 = xmm2[1],xmm1[1] + WORD $0x580f; BYTE $0xd1 // addps %xmm1, %xmm2 + WORD $0x280f; BYTE $0xc2 // movaps %xmm2, %xmm0 + LONG $0x55c2c60f // shufps $85, %xmm2, %xmm0 # xmm0 = xmm0[1,1],xmm2[1,1] + LONG $0xc2580ff3 // addss %xmm2, %xmm0 + WORD $0x2948; BYTE $0xf8 // subq %rdi, %rax + LONG $0x02f8c148 // sarq $2, %rax + LONG $0x01f88348 // cmpq $1, %rax + JE LBB0_27 + LONG $0x02f88348 // cmpq $2, %rax + JE LBB0_26 + LONG $0x03f88348 // cmpq $3, %rax + JNE LBB0_28 + LONG $0x4f100ff3; BYTE $0x08 // movss 8(%rdi), %xmm1 # xmm1 = mem[0],zero,zero,zero + LONG $0x4e5c0ff3; BYTE $0x08 // subss 8(%rsi), %xmm1 + LONG $0xc9590ff3 // mulss %xmm1, %xmm1 + LONG $0xc1580ff3 // addss %xmm1, %xmm0 + +LBB0_26: + LONG $0x4f100ff3; BYTE $0x04 // movss 4(%rdi), %xmm1 # xmm1 = mem[0],zero,zero,zero + LONG $0x4e5c0ff3; BYTE $0x04 // subss 4(%rsi), %xmm1 + LONG $0xc9590ff3 // mulss %xmm1, %xmm1 + LONG $0xc1580ff3 // addss %xmm1, %xmm0 + +LBB0_27: + LONG $0x0f100ff3 // movss (%rdi), %xmm1 # xmm1 = mem[0],zero,zero,zero + LONG $0x0e5c0ff3 // subss (%rsi), %xmm1 + LONG $0xc9590ff3 // mulss %xmm1, %xmm1 + LONG $0xc8580ff3 // addss %xmm0, %xmm1 + WORD $0x280f; BYTE $0xc1 // movaps %xmm1, %xmm0 + +LBB0_28: + WORD $0x8948; BYTE $0xec // movq %rbp, %rsp + BYTE $0x5d // popq %rbp + MOVSS X0, result+24(FP) + RET diff --git a/internal/ailego/math/inner_product_matrix_fp32_avx.go b/internal/ailego/math/inner_product_matrix_fp32_avx.go new file mode 100644 index 0000000..c86e2f1 --- /dev/null +++ b/internal/ailego/math/inner_product_matrix_fp32_avx.go @@ -0,0 +1,14 @@ +//go:build !noasm && amd64 +// Code generated by GoAT. DO NOT EDIT. +// versions: +// clang 21.1.8 (6ubuntu1) +// objdump 2.46 +// flags: -mavx -O3 +// source: src/inner_product_matrix_fp32_avx.c + +package mathutil + +import "unsafe" + +//go:noescape +func inner_product_fp32_avx(lhs, rhs unsafe.Pointer, size int64) (result float32) diff --git a/internal/ailego/math/inner_product_matrix_fp32_avx.s b/internal/ailego/math/inner_product_matrix_fp32_avx.s new file mode 100644 index 0000000..ac260b3 --- /dev/null +++ b/internal/ailego/math/inner_product_matrix_fp32_avx.s @@ -0,0 +1,224 @@ +//go:build !noasm && amd64 +// Code generated by GoAT. DO NOT EDIT. +// versions: +// clang 21.1.8 (6ubuntu1) +// objdump 2.46 +// flags: -mavx -O3 +// source: src/inner_product_matrix_fp32_avx.c + +TEXT ·inner_product_fp32_avx(SB), $8-28 + MOVQ lhs+0(FP), DI + MOVQ rhs+8(FP), SI + MOVQ size+16(FP), DX + BYTE $0x55 // pushq %rbp + WORD $0x8948; BYTE $0xe5 // movq %rsp, %rbp + LONG $0xf8e48348 // andq $-8, %rsp + LONG $0x97048d48 // leaq (%rdi,%rdx,4), %rax + LONG $0x02e2c148 // shlq $2, %rdx + WORD $0x8949; BYTE $0xd0 // movq %rdx, %r8 + LONG $0xc0e08349 // andq $-64, %r8 + LONG $0x070c8d4a // leaq (%rdi,%r8), %rcx + WORD $0x8941; BYTE $0xf1 // movl %esi, %r9d + WORD $0x0941; BYTE $0xf9 // orl %edi, %r9d + LONG $0x1fc1f641 // testb $31, %r9b + JE LBB0_11 + WORD $0x854d; BYTE $0xc0 // testq %r8, %r8 + JE LBB0_2 + LONG $0xc0c28348 // addq $-64, %rdx + WORD $0xc2f6; BYTE $0x40 // testb $64, %dl + JNE LBB0_6 + LONG $0x0710fcc5 // vmovups (%rdi), %ymm0 + LONG $0x4f10fcc5; BYTE $0x20 // vmovups 32(%rdi), %ymm1 + LONG $0x0659fcc5 // vmulps (%rsi), %ymm0, %ymm0 + LONG $0xd257e8c5 // vxorps %xmm2, %xmm2, %xmm2 + LONG $0xc258fcc5 // vaddps %ymm2, %ymm0, %ymm0 + LONG $0x4e59f4c5; BYTE $0x20 // vmulps 32(%rsi), %ymm1, %ymm1 + LONG $0xca58f4c5 // vaddps %ymm2, %ymm1, %ymm1 + LONG $0x40c78348 // addq $64, %rdi + LONG $0x40c68348 // addq $64, %rsi + WORD $0x8949; BYTE $0xf8 // movq %rdi, %r8 + WORD $0x8948; BYTE $0xcf // movq %rcx, %rdi + LONG $0x40fa8348 // cmpq $64, %rdx + JAE LBB0_9 + JMP LBB0_3 + +LBB0_11: + WORD $0x854d; BYTE $0xc0 // testq %r8, %r8 + JE LBB0_12 + LONG $0xc0c28348 // addq $-64, %rdx + WORD $0xc2f6; BYTE $0x40 // testb $64, %dl + JNE LBB0_14 + LONG $0x0728fcc5 // vmovaps (%rdi), %ymm0 + LONG $0x4f28fcc5; BYTE $0x20 // vmovaps 32(%rdi), %ymm1 + LONG $0x0659fcc5 // vmulps (%rsi), %ymm0, %ymm0 + LONG $0xd257e8c5 // vxorps %xmm2, %xmm2, %xmm2 + LONG $0xc258fcc5 // vaddps %ymm2, %ymm0, %ymm0 + LONG $0x4e59f4c5; BYTE $0x20 // vmulps 32(%rsi), %ymm1, %ymm1 + LONG $0xca58f4c5 // vaddps %ymm2, %ymm1, %ymm1 + LONG $0x40c78348 // addq $64, %rdi + LONG $0x40c68348 // addq $64, %rsi + WORD $0x8949; BYTE $0xf8 // movq %rdi, %r8 + WORD $0x8948; BYTE $0xcf // movq %rcx, %rdi + LONG $0x40fa8348 // cmpq $64, %rdx + JAE LBB0_17 + JMP LBB0_19 + +LBB0_2: + LONG $0xc957f0c5 // vxorps %xmm1, %xmm1, %xmm1 + LONG $0xc057f8c5 // vxorps %xmm0, %xmm0, %xmm0 + JMP LBB0_3 + +LBB0_6: + LONG $0xc057f8c5 // vxorps %xmm0, %xmm0, %xmm0 + WORD $0x8949; BYTE $0xf8 // movq %rdi, %r8 + LONG $0xc957f0c5 // vxorps %xmm1, %xmm1, %xmm1 + WORD $0x8948; BYTE $0xcf // movq %rcx, %rdi + LONG $0x40fa8348 // cmpq $64, %rdx + JB LBB0_3 + +LBB0_9: + LONG $0x107cc1c4; BYTE $0x10 // vmovups (%r8), %ymm2 + LONG $0x107cc1c4; WORD $0x2058 // vmovups 32(%r8), %ymm3 + LONG $0x107cc1c4; WORD $0x4060 // vmovups 64(%r8), %ymm4 + LONG $0x107cc1c4; WORD $0x6068 // vmovups 96(%r8), %ymm5 + LONG $0x1659ecc5 // vmulps (%rsi), %ymm2, %ymm2 + LONG $0xc258fcc5 // vaddps %ymm2, %ymm0, %ymm0 + LONG $0x5659e4c5; BYTE $0x20 // vmulps 32(%rsi), %ymm3, %ymm2 + LONG $0x5e59dcc5; BYTE $0x40 // vmulps 64(%rsi), %ymm4, %ymm3 + LONG $0xca58f4c5 // vaddps %ymm2, %ymm1, %ymm1 + LONG $0xc358fcc5 // vaddps %ymm3, %ymm0, %ymm0 + LONG $0x5659d4c5; BYTE $0x60 // vmulps 96(%rsi), %ymm5, %ymm2 + LONG $0xca58f4c5 // vaddps %ymm2, %ymm1, %ymm1 + LONG $0x80e88349 // subq $-128, %r8 + LONG $0x80ee8348 // subq $-128, %rsi + WORD $0x3949; BYTE $0xc8 // cmpq %rcx, %r8 + JNE LBB0_9 + WORD $0x8948; BYTE $0xcf // movq %rcx, %rdi + +LBB0_3: + LONG $0x20c18348 // addq $32, %rcx + WORD $0x3948; BYTE $0xc8 // cmpq %rcx, %rax + JB LBB0_22 + LONG $0x1710fcc5 // vmovups (%rdi), %ymm2 + JMP LBB0_21 + +LBB0_12: + LONG $0xc957f0c5 // vxorps %xmm1, %xmm1, %xmm1 + LONG $0xc057f8c5 // vxorps %xmm0, %xmm0, %xmm0 + JMP LBB0_19 + +LBB0_14: + LONG $0xc057f8c5 // vxorps %xmm0, %xmm0, %xmm0 + WORD $0x8949; BYTE $0xf8 // movq %rdi, %r8 + LONG $0xc957f0c5 // vxorps %xmm1, %xmm1, %xmm1 + WORD $0x8948; BYTE $0xcf // movq %rcx, %rdi + LONG $0x40fa8348 // cmpq $64, %rdx + JB LBB0_19 + +LBB0_17: + LONG $0x287cc1c4; BYTE $0x10 // vmovaps (%r8), %ymm2 + LONG $0x287cc1c4; WORD $0x2058 // vmovaps 32(%r8), %ymm3 + LONG $0x287cc1c4; WORD $0x4060 // vmovaps 64(%r8), %ymm4 + LONG $0x287cc1c4; WORD $0x6068 // vmovaps 96(%r8), %ymm5 + LONG $0x1659ecc5 // vmulps (%rsi), %ymm2, %ymm2 + LONG $0xc258fcc5 // vaddps %ymm2, %ymm0, %ymm0 + LONG $0x5659e4c5; BYTE $0x20 // vmulps 32(%rsi), %ymm3, %ymm2 + LONG $0x5e59dcc5; BYTE $0x40 // vmulps 64(%rsi), %ymm4, %ymm3 + LONG $0xca58f4c5 // vaddps %ymm2, %ymm1, %ymm1 + LONG $0xc358fcc5 // vaddps %ymm3, %ymm0, %ymm0 + LONG $0x5659d4c5; BYTE $0x60 // vmulps 96(%rsi), %ymm5, %ymm2 + LONG $0xca58f4c5 // vaddps %ymm2, %ymm1, %ymm1 + LONG $0x80e88349 // subq $-128, %r8 + LONG $0x80ee8348 // subq $-128, %rsi + WORD $0x3949; BYTE $0xc8 // cmpq %rcx, %r8 + JNE LBB0_17 + WORD $0x8948; BYTE $0xcf // movq %rcx, %rdi + +LBB0_19: + LONG $0x20c18348 // addq $32, %rcx + WORD $0x3948; BYTE $0xc8 // cmpq %rcx, %rax + JB LBB0_22 + LONG $0x1728fcc5 // vmovaps (%rdi), %ymm2 + +LBB0_21: + LONG $0x1659ecc5 // vmulps (%rsi), %ymm2, %ymm2 + LONG $0xc258fcc5 // vaddps %ymm2, %ymm0, %ymm0 + LONG $0x20c78348 // addq $32, %rdi + LONG $0x20c68348 // addq $32, %rsi + +LBB0_22: + LONG $0xc058f4c5 // vaddps %ymm0, %ymm1, %ymm0 + LONG $0xc07cffc5 // vhaddps %ymm0, %ymm0, %ymm0 + LONG $0xc07cffc5 // vhaddps %ymm0, %ymm0, %ymm0 + LONG $0x197de3c4; WORD $0x01c1 // vextractf128 $1, %ymm0, %xmm1 + LONG $0xc158fac5 // vaddss %xmm1, %xmm0, %xmm0 + WORD $0x2948; BYTE $0xf8 // subq %rdi, %rax + LONG $0x02f8c148 // sarq $2, %rax + LONG $0x03f88348 // cmpq $3, %rax + JLE LBB0_23 + LONG $0x05f88348 // cmpq $5, %rax + JG LBB0_29 + LONG $0x04f88348 // cmpq $4, %rax + JE LBB0_34 + LONG $0x05f88348 // cmpq $5, %rax + JE LBB0_33 + +LBB0_38: + WORD $0x8948; BYTE $0xec // movq %rbp, %rsp + BYTE $0x5d // popq %rbp + WORD $0xf8c5; BYTE $0x77 // vzeroupper + MOVSS X0, result+24(FP) + RET + +LBB0_23: + LONG $0x01f88348 // cmpq $1, %rax + JE LBB0_37 + LONG $0x02f88348 // cmpq $2, %rax + JE LBB0_36 + LONG $0x03f88348 // cmpq $3, %rax + JE LBB0_35 + JMP LBB0_38 + +LBB0_29: + LONG $0x06f88348 // cmpq $6, %rax + JE LBB0_32 + LONG $0x07f88348 // cmpq $7, %rax + JNE LBB0_38 + LONG $0x4f10fac5; BYTE $0x18 // vmovss 24(%rdi), %xmm1 # xmm1 = mem[0],zero,zero,zero + LONG $0x4e59f2c5; BYTE $0x18 // vmulss 24(%rsi), %xmm1, %xmm1 + LONG $0xc058f2c5 // vaddss %xmm0, %xmm1, %xmm0 + +LBB0_32: + LONG $0x4f10fac5; BYTE $0x14 // vmovss 20(%rdi), %xmm1 # xmm1 = mem[0],zero,zero,zero + LONG $0x4e59f2c5; BYTE $0x14 // vmulss 20(%rsi), %xmm1, %xmm1 + LONG $0xc058f2c5 // vaddss %xmm0, %xmm1, %xmm0 + +LBB0_33: + LONG $0x4f10fac5; BYTE $0x10 // vmovss 16(%rdi), %xmm1 # xmm1 = mem[0],zero,zero,zero + LONG $0x4e59f2c5; BYTE $0x10 // vmulss 16(%rsi), %xmm1, %xmm1 + LONG $0xc058f2c5 // vaddss %xmm0, %xmm1, %xmm0 + +LBB0_34: + LONG $0x4f10fac5; BYTE $0x0c // vmovss 12(%rdi), %xmm1 # xmm1 = mem[0],zero,zero,zero + LONG $0x4e59f2c5; BYTE $0x0c // vmulss 12(%rsi), %xmm1, %xmm1 + LONG $0xc058f2c5 // vaddss %xmm0, %xmm1, %xmm0 + +LBB0_35: + LONG $0x4f10fac5; BYTE $0x08 // vmovss 8(%rdi), %xmm1 # xmm1 = mem[0],zero,zero,zero + LONG $0x4e59f2c5; BYTE $0x08 // vmulss 8(%rsi), %xmm1, %xmm1 + LONG $0xc058f2c5 // vaddss %xmm0, %xmm1, %xmm0 + +LBB0_36: + LONG $0x4f10fac5; BYTE $0x04 // vmovss 4(%rdi), %xmm1 # xmm1 = mem[0],zero,zero,zero + LONG $0x4e59f2c5; BYTE $0x04 // vmulss 4(%rsi), %xmm1, %xmm1 + LONG $0xc058f2c5 // vaddss %xmm0, %xmm1, %xmm0 + +LBB0_37: + LONG $0x0f10fac5 // vmovss (%rdi), %xmm1 # xmm1 = mem[0],zero,zero,zero + LONG $0x0e59f2c5 // vmulss (%rsi), %xmm1, %xmm1 + LONG $0xc058f2c5 // vaddss %xmm0, %xmm1, %xmm0 + WORD $0x8948; BYTE $0xec // movq %rbp, %rsp + BYTE $0x5d // popq %rbp + WORD $0xf8c5; BYTE $0x77 // vzeroupper + MOVSS X0, result+24(FP) + RET diff --git a/internal/ailego/math/inner_product_matrix_fp32_avx512.go b/internal/ailego/math/inner_product_matrix_fp32_avx512.go new file mode 100644 index 0000000..1c64071 --- /dev/null +++ b/internal/ailego/math/inner_product_matrix_fp32_avx512.go @@ -0,0 +1,14 @@ +//go:build !noasm && amd64 +// Code generated by GoAT. DO NOT EDIT. +// versions: +// clang 21.1.8 (6ubuntu1) +// objdump 2.46 +// flags: -mavx -mfma -mavx512f -O3 +// source: src/inner_product_matrix_fp32_avx512.c + +package mathutil + +import "unsafe" + +//go:noescape +func inner_product_fp32_avx512(lhs, rhs unsafe.Pointer, size int64) (result float32) diff --git a/internal/ailego/math/inner_product_matrix_fp32_avx512.s b/internal/ailego/math/inner_product_matrix_fp32_avx512.s new file mode 100644 index 0000000..b0a37ff --- /dev/null +++ b/internal/ailego/math/inner_product_matrix_fp32_avx512.s @@ -0,0 +1,199 @@ +//go:build !noasm && amd64 +// Code generated by GoAT. DO NOT EDIT. +// versions: +// clang 21.1.8 (6ubuntu1) +// objdump 2.46 +// flags: -mavx -mfma -mavx512f -O3 +// source: src/inner_product_matrix_fp32_avx512.c + +TEXT ·inner_product_fp32_avx512(SB), $8-28 + MOVQ lhs+0(FP), DI + MOVQ rhs+8(FP), SI + MOVQ size+16(FP), DX + BYTE $0x55 // pushq %rbp + WORD $0x8948; BYTE $0xe5 // movq %rsp, %rbp + LONG $0xf8e48348 // andq $-8, %rsp + LONG $0x970c8d48 // leaq (%rdi,%rdx,4), %rcx + LONG $0x02e2c148 // shlq $2, %rdx + WORD $0x8949; BYTE $0xd0 // movq %rdx, %r8 + LONG $0x80e08349 // andq $-128, %r8 + LONG $0x07048d4a // leaq (%rdi,%r8), %rax + WORD $0x8941; BYTE $0xf1 // movl %esi, %r9d + WORD $0x0941; BYTE $0xf9 // orl %edi, %r9d + LONG $0x3fc1f641 // testb $63, %r9b + JE LBB0_12 + WORD $0x854d; BYTE $0xc0 // testq %r8, %r8 + JE LBB0_2 + LONG $0x80c28348 // addq $-128, %rdx + WORD $0x8941; BYTE $0xd0 // movl %edx, %r8d + WORD $0xf741; BYTE $0xd0 // notl %r8d + LONG $0x80c0f741; WORD $0x0001; BYTE $0x00 // testl $384, %r8d # imm = 0x180 + JNE LBB0_7 + LONG $0xc057f8c5 // vxorps %xmm0, %xmm0, %xmm0 + WORD $0x8949; BYTE $0xf8 // movq %rdi, %r8 + LONG $0xc957f0c5 // vxorps %xmm1, %xmm1, %xmm1 + JMP LBB0_9 + +LBB0_12: + WORD $0x854d; BYTE $0xc0 // testq %r8, %r8 + JE LBB0_13 + LONG $0x80c28348 // addq $-128, %rdx + WORD $0x8941; BYTE $0xd0 // movl %edx, %r8d + WORD $0xf741; BYTE $0xd0 // notl %r8d + LONG $0x80c0f741; WORD $0x0001; BYTE $0x00 // testl $384, %r8d # imm = 0x180 + JNE LBB0_16 + LONG $0xc057f8c5 // vxorps %xmm0, %xmm0, %xmm0 + WORD $0x8949; BYTE $0xf8 // movq %rdi, %r8 + LONG $0xc957f0c5 // vxorps %xmm1, %xmm1, %xmm1 + JMP LBB0_18 + +LBB0_2: + LONG $0xc957f0c5 // vxorps %xmm1, %xmm1, %xmm1 + LONG $0xc057f8c5 // vxorps %xmm0, %xmm0, %xmm0 + JMP LBB0_3 + +LBB0_7: + WORD $0x8941; BYTE $0xd1 // movl %edx, %r9d + LONG $0x07e9c141 // shrl $7, %r9d + WORD $0xff41; BYTE $0xc1 // incl %r9d + LONG $0x03e18341 // andl $3, %r9d + WORD $0xf749; BYTE $0xd9 // negq %r9 + LONG $0xc057f8c5 // vxorps %xmm0, %xmm0, %xmm0 + WORD $0x8949; BYTE $0xf8 // movq %rdi, %r8 + LONG $0xc957f0c5 // vxorps %xmm1, %xmm1, %xmm1 + +LBB0_8: + LONG $0x487cd162; WORD $0x1010 // vmovups (%r8), %zmm2 + LONG $0x487cd162; WORD $0x5810; BYTE $0x01 // vmovups 64(%r8), %zmm3 + LONG $0x486df262; WORD $0x0eb8 // vfmadd231ps (%rsi), %zmm2, %zmm1 # zmm1 = (zmm2 * mem) + zmm1 + LONG $0x4865f262; WORD $0x46b8; BYTE $0x01 // vfmadd231ps 64(%rsi), %zmm3, %zmm0 # zmm0 = (zmm3 * mem) + zmm0 + LONG $0x80e88349 // subq $-128, %r8 + LONG $0x80ee8348 // subq $-128, %rsi + WORD $0xff49; BYTE $0xc1 // incq %r9 + JNE LBB0_8 + +LBB0_9: + WORD $0x8948; BYTE $0xc7 // movq %rax, %rdi + LONG $0x80fa8148; WORD $0x0001; BYTE $0x00 // cmpq $384, %rdx # imm = 0x180 + JB LBB0_3 + +LBB0_10: + LONG $0x487cd162; WORD $0x1010 // vmovups (%r8), %zmm2 + LONG $0x487cd162; WORD $0x5810; BYTE $0x01 // vmovups 64(%r8), %zmm3 + LONG $0x487cd162; WORD $0x6010; BYTE $0x02 // vmovups 128(%r8), %zmm4 + LONG $0x487cd162; WORD $0x6810; BYTE $0x03 // vmovups 192(%r8), %zmm5 + LONG $0x4875f262; WORD $0x1698 // vfmadd132ps (%rsi), %zmm1, %zmm2 # zmm2 = (zmm2 * mem) + zmm1 + LONG $0x487df262; WORD $0x5e98; BYTE $0x01 // vfmadd132ps 64(%rsi), %zmm0, %zmm3 # zmm3 = (zmm3 * mem) + zmm0 + LONG $0x485df262; WORD $0x56b8; BYTE $0x02 // vfmadd231ps 128(%rsi), %zmm4, %zmm2 # zmm2 = (zmm4 * mem) + zmm2 + LONG $0x4855f262; WORD $0x5eb8; BYTE $0x03 // vfmadd231ps 192(%rsi), %zmm5, %zmm3 # zmm3 = (zmm5 * mem) + zmm3 + LONG $0x487cd162; WORD $0x4010; BYTE $0x04 // vmovups 256(%r8), %zmm0 + LONG $0x486df262; WORD $0x4698; BYTE $0x04 // vfmadd132ps 256(%rsi), %zmm2, %zmm0 # zmm0 = (zmm0 * mem) + zmm2 + LONG $0x487cd162; WORD $0x5010; BYTE $0x05 // vmovups 320(%r8), %zmm2 + LONG $0x4865f262; WORD $0x5698; BYTE $0x05 // vfmadd132ps 320(%rsi), %zmm3, %zmm2 # zmm2 = (zmm2 * mem) + zmm3 + LONG $0x487cd162; WORD $0x4810; BYTE $0x06 // vmovups 384(%r8), %zmm1 + LONG $0x487df262; WORD $0x4e98; BYTE $0x06 // vfmadd132ps 384(%rsi), %zmm0, %zmm1 # zmm1 = (zmm1 * mem) + zmm0 + LONG $0x487cd162; WORD $0x4010; BYTE $0x07 // vmovups 448(%r8), %zmm0 + LONG $0x486df262; WORD $0x4698; BYTE $0x07 // vfmadd132ps 448(%rsi), %zmm2, %zmm0 # zmm0 = (zmm0 * mem) + zmm2 + LONG $0x00c08149; WORD $0x0002; BYTE $0x00 // addq $512, %r8 # imm = 0x200 + LONG $0x00c68148; WORD $0x0002; BYTE $0x00 // addq $512, %rsi # imm = 0x200 + WORD $0x3949; BYTE $0xc0 // cmpq %rax, %r8 + JNE LBB0_10 + WORD $0x8948; BYTE $0xc7 // movq %rax, %rdi + +LBB0_3: + LONG $0x40c08348 // addq $64, %rax + WORD $0x3948; BYTE $0xc1 // cmpq %rax, %rcx + JB LBB0_24 + LONG $0x487cf162; WORD $0x1710 // vmovups (%rdi), %zmm2 + JMP LBB0_23 + +LBB0_13: + LONG $0xc957f0c5 // vxorps %xmm1, %xmm1, %xmm1 + LONG $0xc057f8c5 // vxorps %xmm0, %xmm0, %xmm0 + JMP LBB0_21 + +LBB0_16: + WORD $0x8941; BYTE $0xd1 // movl %edx, %r9d + LONG $0x07e9c141 // shrl $7, %r9d + WORD $0xff41; BYTE $0xc1 // incl %r9d + LONG $0x03e18341 // andl $3, %r9d + WORD $0xf749; BYTE $0xd9 // negq %r9 + LONG $0xc057f8c5 // vxorps %xmm0, %xmm0, %xmm0 + WORD $0x8949; BYTE $0xf8 // movq %rdi, %r8 + LONG $0xc957f0c5 // vxorps %xmm1, %xmm1, %xmm1 + +LBB0_17: + LONG $0x487cd162; WORD $0x1028 // vmovaps (%r8), %zmm2 + LONG $0x487cd162; WORD $0x5828; BYTE $0x01 // vmovaps 64(%r8), %zmm3 + LONG $0x486df262; WORD $0x0eb8 // vfmadd231ps (%rsi), %zmm2, %zmm1 # zmm1 = (zmm2 * mem) + zmm1 + LONG $0x4865f262; WORD $0x46b8; BYTE $0x01 // vfmadd231ps 64(%rsi), %zmm3, %zmm0 # zmm0 = (zmm3 * mem) + zmm0 + LONG $0x80e88349 // subq $-128, %r8 + LONG $0x80ee8348 // subq $-128, %rsi + WORD $0xff49; BYTE $0xc1 // incq %r9 + JNE LBB0_17 + +LBB0_18: + WORD $0x8948; BYTE $0xc7 // movq %rax, %rdi + LONG $0x80fa8148; WORD $0x0001; BYTE $0x00 // cmpq $384, %rdx # imm = 0x180 + JB LBB0_21 + +LBB0_19: + LONG $0x487cd162; WORD $0x1028 // vmovaps (%r8), %zmm2 + LONG $0x487cd162; WORD $0x5828; BYTE $0x01 // vmovaps 64(%r8), %zmm3 + LONG $0x487cd162; WORD $0x6028; BYTE $0x02 // vmovaps 128(%r8), %zmm4 + LONG $0x487cd162; WORD $0x6828; BYTE $0x03 // vmovaps 192(%r8), %zmm5 + LONG $0x4875f262; WORD $0x1698 // vfmadd132ps (%rsi), %zmm1, %zmm2 # zmm2 = (zmm2 * mem) + zmm1 + LONG $0x487df262; WORD $0x5e98; BYTE $0x01 // vfmadd132ps 64(%rsi), %zmm0, %zmm3 # zmm3 = (zmm3 * mem) + zmm0 + LONG $0x485df262; WORD $0x56b8; BYTE $0x02 // vfmadd231ps 128(%rsi), %zmm4, %zmm2 # zmm2 = (zmm4 * mem) + zmm2 + LONG $0x4855f262; WORD $0x5eb8; BYTE $0x03 // vfmadd231ps 192(%rsi), %zmm5, %zmm3 # zmm3 = (zmm5 * mem) + zmm3 + LONG $0x487cd162; WORD $0x4028; BYTE $0x04 // vmovaps 256(%r8), %zmm0 + LONG $0x486df262; WORD $0x4698; BYTE $0x04 // vfmadd132ps 256(%rsi), %zmm2, %zmm0 # zmm0 = (zmm0 * mem) + zmm2 + LONG $0x487cd162; WORD $0x5028; BYTE $0x05 // vmovaps 320(%r8), %zmm2 + LONG $0x4865f262; WORD $0x5698; BYTE $0x05 // vfmadd132ps 320(%rsi), %zmm3, %zmm2 # zmm2 = (zmm2 * mem) + zmm3 + LONG $0x487cd162; WORD $0x4828; BYTE $0x06 // vmovaps 384(%r8), %zmm1 + LONG $0x487df262; WORD $0x4e98; BYTE $0x06 // vfmadd132ps 384(%rsi), %zmm0, %zmm1 # zmm1 = (zmm1 * mem) + zmm0 + LONG $0x487cd162; WORD $0x4028; BYTE $0x07 // vmovaps 448(%r8), %zmm0 + LONG $0x486df262; WORD $0x4698; BYTE $0x07 // vfmadd132ps 448(%rsi), %zmm2, %zmm0 # zmm0 = (zmm0 * mem) + zmm2 + LONG $0x00c08149; WORD $0x0002; BYTE $0x00 // addq $512, %r8 # imm = 0x200 + LONG $0x00c68148; WORD $0x0002; BYTE $0x00 // addq $512, %rsi # imm = 0x200 + WORD $0x3949; BYTE $0xc0 // cmpq %rax, %r8 + JNE LBB0_19 + WORD $0x8948; BYTE $0xc7 // movq %rax, %rdi + +LBB0_21: + LONG $0x40c08348 // addq $64, %rax + WORD $0x3948; BYTE $0xc1 // cmpq %rax, %rcx + JB LBB0_24 + LONG $0x487cf162; WORD $0x1728 // vmovaps (%rdi), %zmm2 + +LBB0_23: + LONG $0x486df262; WORD $0x0eb8 // vfmadd231ps (%rsi), %zmm2, %zmm1 # zmm1 = (zmm2 * mem) + zmm1 + LONG $0x40c78348 // addq $64, %rdi + LONG $0x40c68348 // addq $64, %rsi + +LBB0_24: + LONG $0x4874f162; WORD $0xc058 // vaddps %zmm0, %zmm1, %zmm0 + WORD $0x3948; BYTE $0xcf // cmpq %rcx, %rdi + JE LBB0_26 + WORD $0xf929 // subl %edi, %ecx + WORD $0xe9c1; BYTE $0x02 // shrl $2, %ecx + LONG $0xffffffb8; BYTE $0xff // movl $-1, %eax + WORD $0xe0d3 // shll %cl, %eax + WORD $0xd0f7 // notl %eax + LONG $0xc892f8c5 // kmovw %eax, %k1 + LONG $0xc97cf162; WORD $0x0f10 // vmovups (%rdi), %zmm1 {%k1} {z} + LONG $0xc97cf162; WORD $0x1610 // vmovups (%rsi), %zmm2 {%k1} {z} + LONG $0x4975f262; WORD $0xc2b8 // vfmadd231ps %zmm2, %zmm1, %zmm0 {%k1} # zmm0 {%k1} = (zmm1 * zmm2) + zmm0 + +LBB0_26: + LONG $0x48fdf362; WORD $0xc11b; BYTE $0x01 // vextractf64x4 $1, %zmm0, %ymm1 + LONG $0xc158fcc5 // vaddps %ymm1, %ymm0, %ymm0 + LONG $0xc07cffc5 // vhaddps %ymm0, %ymm0, %ymm0 + LONG $0xc07cffc5 // vhaddps %ymm0, %ymm0, %ymm0 + LONG $0x197de3c4; WORD $0x01c1 // vextractf128 $1, %ymm0, %xmm1 + LONG $0xc158fac5 // vaddss %xmm1, %xmm0, %xmm0 + WORD $0x8948; BYTE $0xec // movq %rbp, %rsp + BYTE $0x5d // popq %rbp + WORD $0xf8c5; BYTE $0x77 // vzeroupper + MOVSS X0, result+24(FP) + RET diff --git a/internal/ailego/math/inner_product_matrix_fp32_neon.go b/internal/ailego/math/inner_product_matrix_fp32_neon.go new file mode 100644 index 0000000..1937279 --- /dev/null +++ b/internal/ailego/math/inner_product_matrix_fp32_neon.go @@ -0,0 +1,14 @@ +//go:build !noasm && arm64 +// Code generated by GoAT. DO NOT EDIT. +// versions: +// clang 21.1.8 (6ubuntu1) +// objdump 2.46 +// flags: -O3 +// source: src/inner_product_matrix_fp32_neon.c + +package mathutil + +import "unsafe" + +//go:noescape +func inner_product_fp32_neon(lhs, rhs unsafe.Pointer, size int64) (result float32) diff --git a/internal/ailego/math/inner_product_matrix_fp32_neon.s b/internal/ailego/math/inner_product_matrix_fp32_neon.s new file mode 100644 index 0000000..4c9926c --- /dev/null +++ b/internal/ailego/math/inner_product_matrix_fp32_neon.s @@ -0,0 +1,74 @@ +//go:build !noasm && arm64 +// Code generated by GoAT. DO NOT EDIT. +// versions: +// clang 21.1.8 (6ubuntu1) +// objdump 2.46 +// flags: -O3 +// source: src/inner_product_matrix_fp32_neon.c + +TEXT ·inner_product_fp32_neon(SB), $0-28 + MOVD lhs+0(FP), R0 + MOVD rhs+8(FP), R1 + MOVD size+16(FP), R2 + WORD $0xd37ef448 // lsl x8, x2, #2 + WORD $0xf27be909 // ands x9, x8, #0xffffffffffffffe0 + WORD $0x8b080008 // add x8, x0, x8 + WORD $0x8b090009 // add x9, x0, x9 + BEQ LBB0_12 + WORD $0x6f00e400 // movi v0.2d, #0000000000000000 + WORD $0x6f00e401 // movi v1.2d, #0000000000000000 + +LBB0_2: + WORD $0xacc10c02 // ldp q2, q3, [x0], #32 + WORD $0xacc11424 // ldp q4, q5, [x1], #32 + WORD $0xeb09001f // cmp x0, x9 + WORD $0x4e22cc80 // fmla v0.4s, v4.4s, v2.4s + WORD $0x4e23cca1 // fmla v1.4s, v5.4s, v3.4s + BNE LBB0_2 + WORD $0xaa0903e0 // mov x0, x9 + WORD $0x91004129 // add x9, x9, #16 + WORD $0xeb09011f // cmp x8, x9 + BLO LBB0_5 + +LBB0_4: + WORD $0x3cc10402 // ldr q2, [x0], #16 + WORD $0x3cc10423 // ldr q3, [x1], #16 + WORD $0x4e22cc60 // fmla v0.4s, v3.4s, v2.4s + +LBB0_5: + WORD $0x4e20d420 // fadd v0.4s, v1.4s, v0.4s + WORD $0xcb000108 // sub x8, x8, x0 + WORD $0x9342fd08 // asr x8, x8, #2 + WORD $0xf100051f // cmp x8, #1 + WORD $0x6e20d400 // faddp v0.4s, v0.4s, v0.4s + WORD $0x7e30d800 // faddp s0, v0.2s + BEQ LBB0_10 + WORD $0xf100091f // cmp x8, #2 + BEQ LBB0_9 + WORD $0xf1000d1f // cmp x8, #3 + BNE LBB0_11 + WORD $0xbd400801 // ldr s1, [x0, #8] + WORD $0xbd400822 // ldr s2, [x1, #8] + WORD $0x1f020020 // fmadd s0, s1, s2, s0 + +LBB0_9: + WORD $0xbd400401 // ldr s1, [x0, #4] + WORD $0xbd400422 // ldr s2, [x1, #4] + WORD $0x1f020020 // fmadd s0, s1, s2, s0 + +LBB0_10: + WORD $0xbd400001 // ldr s1, [x0] + WORD $0xbd400022 // ldr s2, [x1] + WORD $0x1f020020 // fmadd s0, s1, s2, s0 + +LBB0_11: + FMOVS F0, result+24(FP) + RET + +LBB0_12: + WORD $0x6f00e401 // movi v1.2d, #0000000000000000 + WORD $0x6f00e400 // movi v0.2d, #0000000000000000 + WORD $0x91004129 // add x9, x9, #16 + WORD $0xeb09011f // cmp x8, x9 + BHS LBB0_4 + B LBB0_5 diff --git a/internal/ailego/math/inner_product_matrix_fp32_sse.go b/internal/ailego/math/inner_product_matrix_fp32_sse.go new file mode 100644 index 0000000..453c1ed --- /dev/null +++ b/internal/ailego/math/inner_product_matrix_fp32_sse.go @@ -0,0 +1,14 @@ +//go:build !noasm && amd64 +// Code generated by GoAT. DO NOT EDIT. +// versions: +// clang 21.1.8 (6ubuntu1) +// objdump 2.46 +// flags: -msse2 -O3 +// source: src/inner_product_matrix_fp32_sse.c + +package mathutil + +import "unsafe" + +//go:noescape +func inner_product_fp32_sse(lhs, rhs unsafe.Pointer, size int64) (result float32) diff --git a/internal/ailego/math/inner_product_matrix_fp32_sse.s b/internal/ailego/math/inner_product_matrix_fp32_sse.s new file mode 100644 index 0000000..1b274b4 --- /dev/null +++ b/internal/ailego/math/inner_product_matrix_fp32_sse.s @@ -0,0 +1,193 @@ +//go:build !noasm && amd64 +// Code generated by GoAT. DO NOT EDIT. +// versions: +// clang 21.1.8 (6ubuntu1) +// objdump 2.46 +// flags: -msse2 -O3 +// source: src/inner_product_matrix_fp32_sse.c + +TEXT ·inner_product_fp32_sse(SB), $8-28 + MOVQ lhs+0(FP), DI + MOVQ rhs+8(FP), SI + MOVQ size+16(FP), DX + BYTE $0x55 // pushq %rbp + WORD $0x8948; BYTE $0xe5 // movq %rsp, %rbp + LONG $0xf8e48348 // andq $-8, %rsp + LONG $0x97048d48 // leaq (%rdi,%rdx,4), %rax + LONG $0x02e2c148 // shlq $2, %rdx + WORD $0x8949; BYTE $0xd0 // movq %rdx, %r8 + LONG $0xe0e08349 // andq $-32, %r8 + LONG $0x070c8d4a // leaq (%rdi,%r8), %rcx + WORD $0x8941; BYTE $0xf1 // movl %esi, %r9d + WORD $0x0941; BYTE $0xf9 // orl %edi, %r9d + LONG $0x0fc1f641 // testb $15, %r9b + JE LBB0_11 + WORD $0x854d; BYTE $0xc0 // testq %r8, %r8 + JE LBB0_2 + LONG $0xe0c28348 // addq $-32, %rdx + WORD $0xc2f6; BYTE $0x20 // testb $32, %dl + JNE LBB0_6 + WORD $0x100f; BYTE $0x0f // movups (%rdi), %xmm1 + LONG $0x1057100f // movups 16(%rdi), %xmm2 + WORD $0x100f; BYTE $0x06 // movups (%rsi), %xmm0 + WORD $0x590f; BYTE $0xc1 // mulps %xmm1, %xmm0 + LONG $0x104e100f // movups 16(%rsi), %xmm1 + WORD $0x590f; BYTE $0xca // mulps %xmm2, %xmm1 + WORD $0x570f; BYTE $0xd2 // xorps %xmm2, %xmm2 + WORD $0x580f; BYTE $0xc2 // addps %xmm2, %xmm0 + WORD $0x580f; BYTE $0xca // addps %xmm2, %xmm1 + LONG $0x20c78348 // addq $32, %rdi + LONG $0x20c68348 // addq $32, %rsi + WORD $0x8949; BYTE $0xf8 // movq %rdi, %r8 + WORD $0x8948; BYTE $0xcf // movq %rcx, %rdi + LONG $0x20fa8348 // cmpq $32, %rdx + JAE LBB0_9 + JMP LBB0_3 + +LBB0_11: + WORD $0x854d; BYTE $0xc0 // testq %r8, %r8 + JE LBB0_12 + LONG $0xe0c28348 // addq $-32, %rdx + WORD $0xc2f6; BYTE $0x20 // testb $32, %dl + JNE LBB0_14 + WORD $0x280f; BYTE $0x07 // movaps (%rdi), %xmm0 + LONG $0x104f280f // movaps 16(%rdi), %xmm1 + WORD $0x590f; BYTE $0x06 // mulps (%rsi), %xmm0 + WORD $0x570f; BYTE $0xd2 // xorps %xmm2, %xmm2 + WORD $0x580f; BYTE $0xc2 // addps %xmm2, %xmm0 + LONG $0x104e590f // mulps 16(%rsi), %xmm1 + WORD $0x580f; BYTE $0xca // addps %xmm2, %xmm1 + LONG $0x20c78348 // addq $32, %rdi + LONG $0x20c68348 // addq $32, %rsi + WORD $0x8949; BYTE $0xf8 // movq %rdi, %r8 + WORD $0x8948; BYTE $0xcf // movq %rcx, %rdi + LONG $0x20fa8348 // cmpq $32, %rdx + JAE LBB0_17 + JMP LBB0_19 + +LBB0_2: + WORD $0x570f; BYTE $0xc9 // xorps %xmm1, %xmm1 + WORD $0x570f; BYTE $0xc0 // xorps %xmm0, %xmm0 + JMP LBB0_3 + +LBB0_6: + WORD $0x570f; BYTE $0xc0 // xorps %xmm0, %xmm0 + WORD $0x8949; BYTE $0xf8 // movq %rdi, %r8 + WORD $0x570f; BYTE $0xc9 // xorps %xmm1, %xmm1 + WORD $0x8948; BYTE $0xcf // movq %rcx, %rdi + LONG $0x20fa8348 // cmpq $32, %rdx + JB LBB0_3 + +LBB0_9: + LONG $0x10100f41 // movups (%r8), %xmm2 + LONG $0x58100f41; BYTE $0x10 // movups 16(%r8), %xmm3 + LONG $0x60100f41; BYTE $0x20 // movups 32(%r8), %xmm4 + LONG $0x68100f41; BYTE $0x30 // movups 48(%r8), %xmm5 + WORD $0x100f; BYTE $0x36 // movups (%rsi), %xmm6 + WORD $0x590f; BYTE $0xf2 // mulps %xmm2, %xmm6 + WORD $0x580f; BYTE $0xf0 // addps %xmm0, %xmm6 + LONG $0x1056100f // movups 16(%rsi), %xmm2 + WORD $0x590f; BYTE $0xd3 // mulps %xmm3, %xmm2 + WORD $0x580f; BYTE $0xd1 // addps %xmm1, %xmm2 + LONG $0x2046100f // movups 32(%rsi), %xmm0 + WORD $0x590f; BYTE $0xc4 // mulps %xmm4, %xmm0 + WORD $0x580f; BYTE $0xc6 // addps %xmm6, %xmm0 + LONG $0x304e100f // movups 48(%rsi), %xmm1 + WORD $0x590f; BYTE $0xcd // mulps %xmm5, %xmm1 + WORD $0x580f; BYTE $0xca // addps %xmm2, %xmm1 + LONG $0x40c08349 // addq $64, %r8 + LONG $0x40c68348 // addq $64, %rsi + WORD $0x3949; BYTE $0xc8 // cmpq %rcx, %r8 + JNE LBB0_9 + WORD $0x8948; BYTE $0xcf // movq %rcx, %rdi + +LBB0_3: + LONG $0x10c18348 // addq $16, %rcx + WORD $0x3948; BYTE $0xc8 // cmpq %rcx, %rax + JB LBB0_22 + WORD $0x100f; BYTE $0x17 // movups (%rdi), %xmm2 + WORD $0x100f; BYTE $0x1e // movups (%rsi), %xmm3 + WORD $0x590f; BYTE $0xda // mulps %xmm2, %xmm3 + WORD $0x580f; BYTE $0xc3 // addps %xmm3, %xmm0 + JMP LBB0_21 + +LBB0_12: + WORD $0x570f; BYTE $0xc9 // xorps %xmm1, %xmm1 + WORD $0x570f; BYTE $0xc0 // xorps %xmm0, %xmm0 + JMP LBB0_19 + +LBB0_14: + WORD $0x570f; BYTE $0xc0 // xorps %xmm0, %xmm0 + WORD $0x8949; BYTE $0xf8 // movq %rdi, %r8 + WORD $0x570f; BYTE $0xc9 // xorps %xmm1, %xmm1 + WORD $0x8948; BYTE $0xcf // movq %rcx, %rdi + LONG $0x20fa8348 // cmpq $32, %rdx + JB LBB0_19 + +LBB0_17: + LONG $0x10280f41 // movaps (%r8), %xmm2 + LONG $0x58280f41; BYTE $0x10 // movaps 16(%r8), %xmm3 + WORD $0x590f; BYTE $0x16 // mulps (%rsi), %xmm2 + WORD $0x580f; BYTE $0xd0 // addps %xmm0, %xmm2 + LONG $0x40280f41; BYTE $0x20 // movaps 32(%r8), %xmm0 + LONG $0x105e590f // mulps 16(%rsi), %xmm3 + WORD $0x580f; BYTE $0xd9 // addps %xmm1, %xmm3 + LONG $0x48280f41; BYTE $0x30 // movaps 48(%r8), %xmm1 + LONG $0x2046590f // mulps 32(%rsi), %xmm0 + WORD $0x580f; BYTE $0xc2 // addps %xmm2, %xmm0 + LONG $0x304e590f // mulps 48(%rsi), %xmm1 + WORD $0x580f; BYTE $0xcb // addps %xmm3, %xmm1 + LONG $0x40c08349 // addq $64, %r8 + LONG $0x40c68348 // addq $64, %rsi + WORD $0x3949; BYTE $0xc8 // cmpq %rcx, %r8 + JNE LBB0_17 + WORD $0x8948; BYTE $0xcf // movq %rcx, %rdi + +LBB0_19: + LONG $0x10c18348 // addq $16, %rcx + WORD $0x3948; BYTE $0xc8 // cmpq %rcx, %rax + JB LBB0_22 + WORD $0x280f; BYTE $0x17 // movaps (%rdi), %xmm2 + WORD $0x590f; BYTE $0x16 // mulps (%rsi), %xmm2 + WORD $0x580f; BYTE $0xc2 // addps %xmm2, %xmm0 + +LBB0_21: + LONG $0x10c78348 // addq $16, %rdi + LONG $0x10c68348 // addq $16, %rsi + +LBB0_22: + WORD $0x580f; BYTE $0xc8 // addps %xmm0, %xmm1 + WORD $0x280f; BYTE $0xd1 // movaps %xmm1, %xmm2 + LONG $0xd1150f66 // unpckhpd %xmm1, %xmm2 # xmm2 = xmm2[1],xmm1[1] + WORD $0x580f; BYTE $0xd1 // addps %xmm1, %xmm2 + WORD $0x280f; BYTE $0xc2 // movaps %xmm2, %xmm0 + LONG $0x55c2c60f // shufps $85, %xmm2, %xmm0 # xmm0 = xmm0[1,1],xmm2[1,1] + LONG $0xc2580ff3 // addss %xmm2, %xmm0 + WORD $0x2948; BYTE $0xf8 // subq %rdi, %rax + LONG $0x02f8c148 // sarq $2, %rax + LONG $0x01f88348 // cmpq $1, %rax + JE LBB0_27 + LONG $0x02f88348 // cmpq $2, %rax + JE LBB0_26 + LONG $0x03f88348 // cmpq $3, %rax + JNE LBB0_28 + LONG $0x4f100ff3; BYTE $0x08 // movss 8(%rdi), %xmm1 # xmm1 = mem[0],zero,zero,zero + LONG $0x4e590ff3; BYTE $0x08 // mulss 8(%rsi), %xmm1 + LONG $0xc1580ff3 // addss %xmm1, %xmm0 + +LBB0_26: + LONG $0x4f100ff3; BYTE $0x04 // movss 4(%rdi), %xmm1 # xmm1 = mem[0],zero,zero,zero + LONG $0x4e590ff3; BYTE $0x04 // mulss 4(%rsi), %xmm1 + LONG $0xc1580ff3 // addss %xmm1, %xmm0 + +LBB0_27: + LONG $0x0f100ff3 // movss (%rdi), %xmm1 # xmm1 = mem[0],zero,zero,zero + LONG $0x0e590ff3 // mulss (%rsi), %xmm1 + LONG $0xc8580ff3 // addss %xmm0, %xmm1 + WORD $0x280f; BYTE $0xc1 // movaps %xmm1, %xmm0 + +LBB0_28: + WORD $0x8948; BYTE $0xec // movq %rbp, %rsp + BYTE $0x5d // popq %rbp + MOVSS X0, result+24(FP) + RET diff --git a/internal/ailego/math/mips_euclidean_distance_matrix_fp32_avx.go b/internal/ailego/math/mips_euclidean_distance_matrix_fp32_avx.go new file mode 100644 index 0000000..7bd469f --- /dev/null +++ b/internal/ailego/math/mips_euclidean_distance_matrix_fp32_avx.go @@ -0,0 +1,14 @@ +//go:build !noasm && amd64 +// Code generated by GoAT. DO NOT EDIT. +// versions: +// clang 21.1.8 (6ubuntu1) +// objdump 2.46 +// flags: -mavx -O3 +// source: src/mips_euclidean_distance_matrix_fp32_avx.c + +package mathutil + +import "unsafe" + +//go:noescape +func inner_product_and_squared_norm_fp32_avx(lhs, rhs unsafe.Pointer, size int64, lhs_norm, rhs_norm unsafe.Pointer) (result float32) diff --git a/internal/ailego/math/mips_euclidean_distance_matrix_fp32_avx.s b/internal/ailego/math/mips_euclidean_distance_matrix_fp32_avx.s new file mode 100644 index 0000000..941ceed --- /dev/null +++ b/internal/ailego/math/mips_euclidean_distance_matrix_fp32_avx.s @@ -0,0 +1,248 @@ +//go:build !noasm && amd64 +// Code generated by GoAT. DO NOT EDIT. +// versions: +// clang 21.1.8 (6ubuntu1) +// objdump 2.46 +// flags: -mavx -O3 +// source: src/mips_euclidean_distance_matrix_fp32_avx.c + +TEXT ·inner_product_and_squared_norm_fp32_avx(SB), $8-44 + MOVQ lhs+0(FP), DI + MOVQ rhs+8(FP), SI + MOVQ size+16(FP), DX + MOVQ lhs_norm+24(FP), CX + MOVQ rhs_norm+32(FP), R8 + BYTE $0x55 // pushq %rbp + WORD $0x8948; BYTE $0xe5 // movq %rsp, %rbp + LONG $0xf8e48348 // andq $-8, %rsp + LONG $0x97048d48 // leaq (%rdi,%rdx,4), %rax + LONG $0x02e2c148 // shlq $2, %rdx + LONG $0xc0e28348 // andq $-64, %rdx + LONG $0x170c8d4c // leaq (%rdi,%rdx), %r9 + WORD $0x8941; BYTE $0xf2 // movl %esi, %r10d + WORD $0x0941; BYTE $0xfa // orl %edi, %r10d + LONG $0x1fc2f641 // testb $31, %r10b + JE LBB0_3 + WORD $0x8548; BYTE $0xd2 // testq %rdx, %rdx + JE LBB0_2 + LONG $0xc057f8c5 // vxorps %xmm0, %xmm0, %xmm0 + LONG $0xdb57e0c5 // vxorps %xmm3, %xmm3, %xmm3 + LONG $0xd257e8c5 // vxorps %xmm2, %xmm2, %xmm2 + LONG $0xc957f0c5 // vxorps %xmm1, %xmm1, %xmm1 + +LBB0_11: + LONG $0x2710fcc5 // vmovups (%rdi), %ymm4 + LONG $0x6f10fcc5; BYTE $0x20 // vmovups 32(%rdi), %ymm5 + LONG $0x3610fcc5 // vmovups (%rsi), %ymm6 + LONG $0x7e10fcc5; BYTE $0x20 // vmovups 32(%rsi), %ymm7 + LONG $0xc6595cc5 // vmulps %ymm6, %ymm4, %ymm8 + LONG $0xc058bcc5 // vaddps %ymm0, %ymm8, %ymm0 + LONG $0xc75954c5 // vmulps %ymm7, %ymm5, %ymm8 + LONG $0xdb58bcc5 // vaddps %ymm3, %ymm8, %ymm3 + LONG $0xe459dcc5 // vmulps %ymm4, %ymm4, %ymm4 + LONG $0xd458ecc5 // vaddps %ymm4, %ymm2, %ymm2 + LONG $0xe559d4c5 // vmulps %ymm5, %ymm5, %ymm4 + LONG $0xd258dcc5 // vaddps %ymm2, %ymm4, %ymm2 + LONG $0xe659ccc5 // vmulps %ymm6, %ymm6, %ymm4 + LONG $0xcc58f4c5 // vaddps %ymm4, %ymm1, %ymm1 + LONG $0xe759c4c5 // vmulps %ymm7, %ymm7, %ymm4 + LONG $0xc958dcc5 // vaddps %ymm1, %ymm4, %ymm1 + LONG $0x40c78348 // addq $64, %rdi + LONG $0x40c68348 // addq $64, %rsi + WORD $0x394c; BYTE $0xcf // cmpq %r9, %rdi + JNE LBB0_11 + WORD $0x894c; BYTE $0xcf // movq %r9, %rdi + LONG $0x20c18349 // addq $32, %r9 + WORD $0x394c; BYTE $0xc8 // cmpq %r9, %rax + JAE LBB0_14 + JMP LBB0_16 + +LBB0_3: + WORD $0x8548; BYTE $0xd2 // testq %rdx, %rdx + JE LBB0_4 + LONG $0xc057f8c5 // vxorps %xmm0, %xmm0, %xmm0 + LONG $0xdb57e0c5 // vxorps %xmm3, %xmm3, %xmm3 + LONG $0xd257e8c5 // vxorps %xmm2, %xmm2, %xmm2 + LONG $0xc957f0c5 // vxorps %xmm1, %xmm1, %xmm1 + +LBB0_6: + LONG $0x2728fcc5 // vmovaps (%rdi), %ymm4 + LONG $0x6f28fcc5; BYTE $0x20 // vmovaps 32(%rdi), %ymm5 + LONG $0x3628fcc5 // vmovaps (%rsi), %ymm6 + LONG $0x7e28fcc5; BYTE $0x20 // vmovaps 32(%rsi), %ymm7 + LONG $0xc6595cc5 // vmulps %ymm6, %ymm4, %ymm8 + LONG $0xc058bcc5 // vaddps %ymm0, %ymm8, %ymm0 + LONG $0xc75954c5 // vmulps %ymm7, %ymm5, %ymm8 + LONG $0xdb58bcc5 // vaddps %ymm3, %ymm8, %ymm3 + LONG $0xe459dcc5 // vmulps %ymm4, %ymm4, %ymm4 + LONG $0xd458ecc5 // vaddps %ymm4, %ymm2, %ymm2 + LONG $0xe559d4c5 // vmulps %ymm5, %ymm5, %ymm4 + LONG $0xd258dcc5 // vaddps %ymm2, %ymm4, %ymm2 + LONG $0xe659ccc5 // vmulps %ymm6, %ymm6, %ymm4 + LONG $0xcc58f4c5 // vaddps %ymm4, %ymm1, %ymm1 + LONG $0xe759c4c5 // vmulps %ymm7, %ymm7, %ymm4 + LONG $0xc958dcc5 // vaddps %ymm1, %ymm4, %ymm1 + LONG $0x40c78348 // addq $64, %rdi + LONG $0x40c68348 // addq $64, %rsi + WORD $0x394c; BYTE $0xcf // cmpq %r9, %rdi + JNE LBB0_6 + WORD $0x894c; BYTE $0xcf // movq %r9, %rdi + LONG $0x20c18349 // addq $32, %r9 + WORD $0x394c; BYTE $0xc8 // cmpq %r9, %rax + JAE LBB0_9 + JMP LBB0_16 + +LBB0_2: + LONG $0xc957f0c5 // vxorps %xmm1, %xmm1, %xmm1 + LONG $0xd257e8c5 // vxorps %xmm2, %xmm2, %xmm2 + LONG $0xdb57e0c5 // vxorps %xmm3, %xmm3, %xmm3 + LONG $0xc057f8c5 // vxorps %xmm0, %xmm0, %xmm0 + LONG $0x20c18349 // addq $32, %r9 + WORD $0x394c; BYTE $0xc8 // cmpq %r9, %rax + JB LBB0_16 + +LBB0_14: + LONG $0x2710fcc5 // vmovups (%rdi), %ymm4 + LONG $0x2e10fcc5 // vmovups (%rsi), %ymm5 + JMP LBB0_15 + +LBB0_4: + LONG $0xc957f0c5 // vxorps %xmm1, %xmm1, %xmm1 + LONG $0xd257e8c5 // vxorps %xmm2, %xmm2, %xmm2 + LONG $0xdb57e0c5 // vxorps %xmm3, %xmm3, %xmm3 + LONG $0xc057f8c5 // vxorps %xmm0, %xmm0, %xmm0 + LONG $0x20c18349 // addq $32, %r9 + WORD $0x394c; BYTE $0xc8 // cmpq %r9, %rax + JB LBB0_16 + +LBB0_9: + LONG $0x2728fcc5 // vmovaps (%rdi), %ymm4 + LONG $0x2e28fcc5 // vmovaps (%rsi), %ymm5 + +LBB0_15: + LONG $0xf559dcc5 // vmulps %ymm5, %ymm4, %ymm6 + LONG $0xc658fcc5 // vaddps %ymm6, %ymm0, %ymm0 + LONG $0xe459dcc5 // vmulps %ymm4, %ymm4, %ymm4 + LONG $0xd458ecc5 // vaddps %ymm4, %ymm2, %ymm2 + LONG $0xe559d4c5 // vmulps %ymm5, %ymm5, %ymm4 + LONG $0xcc58f4c5 // vaddps %ymm4, %ymm1, %ymm1 + LONG $0x20c78348 // addq $32, %rdi + LONG $0x20c68348 // addq $32, %rsi + +LBB0_16: + LONG $0xc058e4c5 // vaddps %ymm0, %ymm3, %ymm0 + LONG $0xc07cffc5 // vhaddps %ymm0, %ymm0, %ymm0 + LONG $0xc07cffc5 // vhaddps %ymm0, %ymm0, %ymm0 + LONG $0x197de3c4; WORD $0x01c3 // vextractf128 $1, %ymm0, %xmm3 + LONG $0xc358fac5 // vaddss %xmm3, %xmm0, %xmm0 + LONG $0xd27cefc5 // vhaddps %ymm2, %ymm2, %ymm2 + LONG $0xd27cefc5 // vhaddps %ymm2, %ymm2, %ymm2 + LONG $0x197de3c4; WORD $0x01d3 // vextractf128 $1, %ymm2, %xmm3 + LONG $0xd358eac5 // vaddss %xmm3, %xmm2, %xmm2 + LONG $0xc97cf7c5 // vhaddps %ymm1, %ymm1, %ymm1 + LONG $0xc97cf7c5 // vhaddps %ymm1, %ymm1, %ymm1 + LONG $0x197de3c4; WORD $0x01cb // vextractf128 $1, %ymm1, %xmm3 + LONG $0xcb58f2c5 // vaddss %xmm3, %xmm1, %xmm1 + WORD $0x2948; BYTE $0xf8 // subq %rdi, %rax + LONG $0x02f8c148 // sarq $2, %rax + LONG $0x03f88348 // cmpq $3, %rax + JLE LBB0_17 + LONG $0x05f88348 // cmpq $5, %rax + JG LBB0_23 + LONG $0x04f88348 // cmpq $4, %rax + JE LBB0_28 + LONG $0x05f88348 // cmpq $5, %rax + JE LBB0_27 + JMP LBB0_32 + +LBB0_17: + LONG $0x01f88348 // cmpq $1, %rax + JE LBB0_31 + LONG $0x02f88348 // cmpq $2, %rax + JE LBB0_30 + LONG $0x03f88348 // cmpq $3, %rax + JE LBB0_29 + JMP LBB0_32 + +LBB0_23: + LONG $0x06f88348 // cmpq $6, %rax + JE LBB0_26 + LONG $0x07f88348 // cmpq $7, %rax + JNE LBB0_32 + LONG $0x5f10fac5; BYTE $0x18 // vmovss 24(%rdi), %xmm3 # xmm3 = mem[0],zero,zero,zero + LONG $0x6610fac5; BYTE $0x18 // vmovss 24(%rsi), %xmm4 # xmm4 = mem[0],zero,zero,zero + LONG $0xec59e2c5 // vmulss %xmm4, %xmm3, %xmm5 + LONG $0xc058d2c5 // vaddss %xmm0, %xmm5, %xmm0 + LONG $0xdb59e2c5 // vmulss %xmm3, %xmm3, %xmm3 + LONG $0xd258e2c5 // vaddss %xmm2, %xmm3, %xmm2 + LONG $0xdc59dac5 // vmulss %xmm4, %xmm4, %xmm3 + LONG $0xc958e2c5 // vaddss %xmm1, %xmm3, %xmm1 + +LBB0_26: + LONG $0x5f10fac5; BYTE $0x14 // vmovss 20(%rdi), %xmm3 # xmm3 = mem[0],zero,zero,zero + LONG $0x6610fac5; BYTE $0x14 // vmovss 20(%rsi), %xmm4 # xmm4 = mem[0],zero,zero,zero + LONG $0xec59e2c5 // vmulss %xmm4, %xmm3, %xmm5 + LONG $0xc058d2c5 // vaddss %xmm0, %xmm5, %xmm0 + LONG $0xdb59e2c5 // vmulss %xmm3, %xmm3, %xmm3 + LONG $0xd258e2c5 // vaddss %xmm2, %xmm3, %xmm2 + LONG $0xdc59dac5 // vmulss %xmm4, %xmm4, %xmm3 + LONG $0xc958e2c5 // vaddss %xmm1, %xmm3, %xmm1 + +LBB0_27: + LONG $0x5f10fac5; BYTE $0x10 // vmovss 16(%rdi), %xmm3 # xmm3 = mem[0],zero,zero,zero + LONG $0x6610fac5; BYTE $0x10 // vmovss 16(%rsi), %xmm4 # xmm4 = mem[0],zero,zero,zero + LONG $0xec59e2c5 // vmulss %xmm4, %xmm3, %xmm5 + LONG $0xc058d2c5 // vaddss %xmm0, %xmm5, %xmm0 + LONG $0xdb59e2c5 // vmulss %xmm3, %xmm3, %xmm3 + LONG $0xd258e2c5 // vaddss %xmm2, %xmm3, %xmm2 + LONG $0xdc59dac5 // vmulss %xmm4, %xmm4, %xmm3 + LONG $0xc958e2c5 // vaddss %xmm1, %xmm3, %xmm1 + +LBB0_28: + LONG $0x5f10fac5; BYTE $0x0c // vmovss 12(%rdi), %xmm3 # xmm3 = mem[0],zero,zero,zero + LONG $0x6610fac5; BYTE $0x0c // vmovss 12(%rsi), %xmm4 # xmm4 = mem[0],zero,zero,zero + LONG $0xec59e2c5 // vmulss %xmm4, %xmm3, %xmm5 + LONG $0xc058d2c5 // vaddss %xmm0, %xmm5, %xmm0 + LONG $0xdb59e2c5 // vmulss %xmm3, %xmm3, %xmm3 + LONG $0xd258e2c5 // vaddss %xmm2, %xmm3, %xmm2 + LONG $0xdc59dac5 // vmulss %xmm4, %xmm4, %xmm3 + LONG $0xc958e2c5 // vaddss %xmm1, %xmm3, %xmm1 + +LBB0_29: + LONG $0x5f10fac5; BYTE $0x08 // vmovss 8(%rdi), %xmm3 # xmm3 = mem[0],zero,zero,zero + LONG $0x6610fac5; BYTE $0x08 // vmovss 8(%rsi), %xmm4 # xmm4 = mem[0],zero,zero,zero + LONG $0xec59e2c5 // vmulss %xmm4, %xmm3, %xmm5 + LONG $0xc058d2c5 // vaddss %xmm0, %xmm5, %xmm0 + LONG $0xdb59e2c5 // vmulss %xmm3, %xmm3, %xmm3 + LONG $0xd258e2c5 // vaddss %xmm2, %xmm3, %xmm2 + LONG $0xdc59dac5 // vmulss %xmm4, %xmm4, %xmm3 + LONG $0xc958e2c5 // vaddss %xmm1, %xmm3, %xmm1 + +LBB0_30: + LONG $0x5f10fac5; BYTE $0x04 // vmovss 4(%rdi), %xmm3 # xmm3 = mem[0],zero,zero,zero + LONG $0x6610fac5; BYTE $0x04 // vmovss 4(%rsi), %xmm4 # xmm4 = mem[0],zero,zero,zero + LONG $0xec59e2c5 // vmulss %xmm4, %xmm3, %xmm5 + LONG $0xc058d2c5 // vaddss %xmm0, %xmm5, %xmm0 + LONG $0xdb59e2c5 // vmulss %xmm3, %xmm3, %xmm3 + LONG $0xd258e2c5 // vaddss %xmm2, %xmm3, %xmm2 + LONG $0xdc59dac5 // vmulss %xmm4, %xmm4, %xmm3 + LONG $0xc958e2c5 // vaddss %xmm1, %xmm3, %xmm1 + +LBB0_31: + LONG $0x1f10fac5 // vmovss (%rdi), %xmm3 # xmm3 = mem[0],zero,zero,zero + LONG $0x2610fac5 // vmovss (%rsi), %xmm4 # xmm4 = mem[0],zero,zero,zero + LONG $0xec59e2c5 // vmulss %xmm4, %xmm3, %xmm5 + LONG $0xc058d2c5 // vaddss %xmm0, %xmm5, %xmm0 + LONG $0xdb59e2c5 // vmulss %xmm3, %xmm3, %xmm3 + LONG $0xd258e2c5 // vaddss %xmm2, %xmm3, %xmm2 + LONG $0xdc59dac5 // vmulss %xmm4, %xmm4, %xmm3 + LONG $0xc958e2c5 // vaddss %xmm1, %xmm3, %xmm1 + +LBB0_32: + LONG $0x1111fac5 // vmovss %xmm2, (%rcx) + LONG $0x117ac1c4; BYTE $0x08 // vmovss %xmm1, (%r8) + WORD $0x8948; BYTE $0xec // movq %rbp, %rsp + BYTE $0x5d // popq %rbp + WORD $0xf8c5; BYTE $0x77 // vzeroupper + MOVSS X0, result+40(FP) + RET diff --git a/internal/ailego/math/mips_euclidean_distance_matrix_fp32_avx512.go b/internal/ailego/math/mips_euclidean_distance_matrix_fp32_avx512.go new file mode 100644 index 0000000..c015e11 --- /dev/null +++ b/internal/ailego/math/mips_euclidean_distance_matrix_fp32_avx512.go @@ -0,0 +1,14 @@ +//go:build !noasm && amd64 +// Code generated by GoAT. DO NOT EDIT. +// versions: +// clang 21.1.8 (6ubuntu1) +// objdump 2.46 +// flags: -mavx -mfma -mavx512f -O3 +// source: src/mips_euclidean_distance_matrix_fp32_avx512.c + +package mathutil + +import "unsafe" + +//go:noescape +func inner_product_and_squared_norm_fp32_avx512(lhs, rhs unsafe.Pointer, size int64, lhs_norm, rhs_norm unsafe.Pointer) (result float32) diff --git a/internal/ailego/math/mips_euclidean_distance_matrix_fp32_avx512.s b/internal/ailego/math/mips_euclidean_distance_matrix_fp32_avx512.s new file mode 100644 index 0000000..10efa0b --- /dev/null +++ b/internal/ailego/math/mips_euclidean_distance_matrix_fp32_avx512.s @@ -0,0 +1,230 @@ +//go:build !noasm && amd64 +// Code generated by GoAT. DO NOT EDIT. +// versions: +// clang 21.1.8 (6ubuntu1) +// objdump 2.46 +// flags: -mavx -mfma -mavx512f -O3 +// source: src/mips_euclidean_distance_matrix_fp32_avx512.c + +TEXT ·inner_product_and_squared_norm_fp32_avx512(SB), $8-44 + MOVQ lhs+0(FP), DI + MOVQ rhs+8(FP), SI + MOVQ size+16(FP), DX + MOVQ lhs_norm+24(FP), CX + MOVQ rhs_norm+32(FP), R8 + BYTE $0x55 // pushq %rbp + WORD $0x8948; BYTE $0xe5 // movq %rsp, %rbp + LONG $0xf8e48348 // andq $-8, %rsp + WORD $0x8948; BYTE $0xc8 // movq %rcx, %rax + LONG $0x970c8d48 // leaq (%rdi,%rdx,4), %rcx + LONG $0x02e2c148 // shlq $2, %rdx + WORD $0x8949; BYTE $0xd2 // movq %rdx, %r10 + LONG $0x80e28349 // andq $-128, %r10 + LONG $0x170c8d4e // leaq (%rdi,%r10), %r9 + WORD $0x8941; BYTE $0xf3 // movl %esi, %r11d + WORD $0x0941; BYTE $0xfb // orl %edi, %r11d + LONG $0x3fc3f641 // testb $63, %r11b + JE LBB0_11 + WORD $0x854d; BYTE $0xd2 // testq %r10, %r10 + JE LBB0_2 + LONG $0x80c28348 // addq $-128, %rdx + WORD $0xd284 // testb %dl, %dl + JS LBB0_6 + LONG $0x487cf162; WORD $0x0f10 // vmovups (%rdi), %zmm1 + LONG $0x487cf162; WORD $0x6710; BYTE $0x01 // vmovups 64(%rdi), %zmm4 + LONG $0x487cf162; WORD $0x0610 // vmovups (%rsi), %zmm0 + LONG $0x487cf162; WORD $0x5e10; BYTE $0x01 // vmovups 64(%rsi), %zmm3 + LONG $0xed57d0c5 // vxorps %xmm5, %xmm5, %xmm5 + LONG $0x487cf162; WORD $0xd028 // vmovaps %zmm0, %zmm2 + LONG $0x4875f262; WORD $0xd5a8 // vfmadd213ps %zmm5, %zmm1, %zmm2 # zmm2 = (zmm1 * zmm2) + zmm5 + LONG $0x487df262; WORD $0xc5a8 // vfmadd213ps %zmm5, %zmm0, %zmm0 # zmm0 = (zmm0 * zmm0) + zmm5 + LONG $0x4865f262; WORD $0xc3b8 // vfmadd231ps %zmm3, %zmm3, %zmm0 # zmm0 = (zmm3 * zmm3) + zmm0 + LONG $0x485df262; WORD $0xdda8 // vfmadd213ps %zmm5, %zmm4, %zmm3 # zmm3 = (zmm4 * zmm3) + zmm5 + LONG $0x4875f262; WORD $0xcda8 // vfmadd213ps %zmm5, %zmm1, %zmm1 # zmm1 = (zmm1 * zmm1) + zmm5 + LONG $0x485df262; WORD $0xccb8 // vfmadd231ps %zmm4, %zmm4, %zmm1 # zmm1 = (zmm4 * zmm4) + zmm1 + LONG $0x80ef8348 // subq $-128, %rdi + LONG $0x80ee8348 // subq $-128, %rsi + WORD $0x8949; BYTE $0xfa // movq %rdi, %r10 + WORD $0x894c; BYTE $0xcf // movq %r9, %rdi + LONG $0x80fa8148; WORD $0x0000; BYTE $0x00 // cmpq $128, %rdx + JAE LBB0_9 + JMP LBB0_3 + +LBB0_11: + WORD $0x854d; BYTE $0xd2 // testq %r10, %r10 + JE LBB0_12 + LONG $0x80c28348 // addq $-128, %rdx + WORD $0xd284 // testb %dl, %dl + JS LBB0_14 + LONG $0x487cf162; WORD $0x0f28 // vmovaps (%rdi), %zmm1 + LONG $0x487cf162; WORD $0x6728; BYTE $0x01 // vmovaps 64(%rdi), %zmm4 + LONG $0x487cf162; WORD $0x0628 // vmovaps (%rsi), %zmm0 + LONG $0x487cf162; WORD $0x5e28; BYTE $0x01 // vmovaps 64(%rsi), %zmm3 + LONG $0xed57d0c5 // vxorps %xmm5, %xmm5, %xmm5 + LONG $0x487cf162; WORD $0xd028 // vmovaps %zmm0, %zmm2 + LONG $0x4875f262; WORD $0xd5a8 // vfmadd213ps %zmm5, %zmm1, %zmm2 # zmm2 = (zmm1 * zmm2) + zmm5 + LONG $0x487df262; WORD $0xc5a8 // vfmadd213ps %zmm5, %zmm0, %zmm0 # zmm0 = (zmm0 * zmm0) + zmm5 + LONG $0x4865f262; WORD $0xc3b8 // vfmadd231ps %zmm3, %zmm3, %zmm0 # zmm0 = (zmm3 * zmm3) + zmm0 + LONG $0x485df262; WORD $0xdda8 // vfmadd213ps %zmm5, %zmm4, %zmm3 # zmm3 = (zmm4 * zmm3) + zmm5 + LONG $0x4875f262; WORD $0xcda8 // vfmadd213ps %zmm5, %zmm1, %zmm1 # zmm1 = (zmm1 * zmm1) + zmm5 + LONG $0x485df262; WORD $0xccb8 // vfmadd231ps %zmm4, %zmm4, %zmm1 # zmm1 = (zmm4 * zmm4) + zmm1 + LONG $0x80ef8348 // subq $-128, %rdi + LONG $0x80ee8348 // subq $-128, %rsi + WORD $0x8949; BYTE $0xfa // movq %rdi, %r10 + WORD $0x894c; BYTE $0xcf // movq %r9, %rdi + LONG $0x80fa8148; WORD $0x0000; BYTE $0x00 // cmpq $128, %rdx + JAE LBB0_17 + JMP LBB0_19 + +LBB0_2: + LONG $0xc057f8c5 // vxorps %xmm0, %xmm0, %xmm0 + LONG $0xc957f0c5 // vxorps %xmm1, %xmm1, %xmm1 + LONG $0xdb57e0c5 // vxorps %xmm3, %xmm3, %xmm3 + LONG $0xd257e8c5 // vxorps %xmm2, %xmm2, %xmm2 + JMP LBB0_3 + +LBB0_6: + LONG $0xd257e8c5 // vxorps %xmm2, %xmm2, %xmm2 + WORD $0x8949; BYTE $0xfa // movq %rdi, %r10 + LONG $0xdb57e0c5 // vxorps %xmm3, %xmm3, %xmm3 + LONG $0xc957f0c5 // vxorps %xmm1, %xmm1, %xmm1 + LONG $0xc057f8c5 // vxorps %xmm0, %xmm0, %xmm0 + WORD $0x894c; BYTE $0xcf // movq %r9, %rdi + LONG $0x80fa8148; WORD $0x0000; BYTE $0x00 // cmpq $128, %rdx + JB LBB0_3 + +LBB0_9: + LONG $0x487cd162; WORD $0x2210 // vmovups (%r10), %zmm4 + LONG $0x487cd162; WORD $0x6a10; BYTE $0x01 // vmovups 64(%r10), %zmm5 + LONG $0x487cd162; WORD $0x7210; BYTE $0x02 // vmovups 128(%r10), %zmm6 + LONG $0x487cd162; WORD $0x7a10; BYTE $0x03 // vmovups 192(%r10), %zmm7 + LONG $0x487c7162; WORD $0x0610 // vmovups (%rsi), %zmm8 + LONG $0x487c7162; WORD $0x4e10; BYTE $0x01 // vmovups 64(%rsi), %zmm9 + LONG $0x487c7162; WORD $0x5610; BYTE $0x02 // vmovups 128(%rsi), %zmm10 + LONG $0x487c7162; WORD $0x5e10; BYTE $0x03 // vmovups 192(%rsi), %zmm11 + LONG $0x485dd262; WORD $0xd0b8 // vfmadd231ps %zmm8, %zmm4, %zmm2 # zmm2 = (zmm4 * zmm8) + zmm2 + LONG $0x4855d262; WORD $0xd9b8 // vfmadd231ps %zmm9, %zmm5, %zmm3 # zmm3 = (zmm5 * zmm9) + zmm3 + LONG $0x485df262; WORD $0xccb8 // vfmadd231ps %zmm4, %zmm4, %zmm1 # zmm1 = (zmm4 * zmm4) + zmm1 + LONG $0x4855f262; WORD $0xcdb8 // vfmadd231ps %zmm5, %zmm5, %zmm1 # zmm1 = (zmm5 * zmm5) + zmm1 + LONG $0x483dd262; WORD $0xc0b8 // vfmadd231ps %zmm8, %zmm8, %zmm0 # zmm0 = (zmm8 * zmm8) + zmm0 + LONG $0x4835d262; WORD $0xc1b8 // vfmadd231ps %zmm9, %zmm9, %zmm0 # zmm0 = (zmm9 * zmm9) + zmm0 + LONG $0x484dd262; WORD $0xd2b8 // vfmadd231ps %zmm10, %zmm6, %zmm2 # zmm2 = (zmm6 * zmm10) + zmm2 + LONG $0x4845d262; WORD $0xdbb8 // vfmadd231ps %zmm11, %zmm7, %zmm3 # zmm3 = (zmm7 * zmm11) + zmm3 + LONG $0x484df262; WORD $0xceb8 // vfmadd231ps %zmm6, %zmm6, %zmm1 # zmm1 = (zmm6 * zmm6) + zmm1 + LONG $0x4845f262; WORD $0xcfb8 // vfmadd231ps %zmm7, %zmm7, %zmm1 # zmm1 = (zmm7 * zmm7) + zmm1 + LONG $0x482dd262; WORD $0xc2b8 // vfmadd231ps %zmm10, %zmm10, %zmm0 # zmm0 = (zmm10 * zmm10) + zmm0 + LONG $0x4825d262; WORD $0xc3b8 // vfmadd231ps %zmm11, %zmm11, %zmm0 # zmm0 = (zmm11 * zmm11) + zmm0 + LONG $0x00c28149; WORD $0x0001; BYTE $0x00 // addq $256, %r10 # imm = 0x100 + LONG $0x00c68148; WORD $0x0001; BYTE $0x00 // addq $256, %rsi # imm = 0x100 + WORD $0x394d; BYTE $0xca // cmpq %r9, %r10 + JNE LBB0_9 + WORD $0x894c; BYTE $0xcf // movq %r9, %rdi + +LBB0_3: + LONG $0x40c18349 // addq $64, %r9 + WORD $0x394c; BYTE $0xc9 // cmpq %r9, %rcx + JB LBB0_22 + LONG $0x487cf162; WORD $0x2710 // vmovups (%rdi), %zmm4 + LONG $0x487cf162; WORD $0x2e10 // vmovups (%rsi), %zmm5 + JMP LBB0_21 + +LBB0_12: + LONG $0xc057f8c5 // vxorps %xmm0, %xmm0, %xmm0 + LONG $0xc957f0c5 // vxorps %xmm1, %xmm1, %xmm1 + LONG $0xdb57e0c5 // vxorps %xmm3, %xmm3, %xmm3 + LONG $0xd257e8c5 // vxorps %xmm2, %xmm2, %xmm2 + JMP LBB0_19 + +LBB0_14: + LONG $0xd257e8c5 // vxorps %xmm2, %xmm2, %xmm2 + WORD $0x8949; BYTE $0xfa // movq %rdi, %r10 + LONG $0xdb57e0c5 // vxorps %xmm3, %xmm3, %xmm3 + LONG $0xc957f0c5 // vxorps %xmm1, %xmm1, %xmm1 + LONG $0xc057f8c5 // vxorps %xmm0, %xmm0, %xmm0 + WORD $0x894c; BYTE $0xcf // movq %r9, %rdi + LONG $0x80fa8148; WORD $0x0000; BYTE $0x00 // cmpq $128, %rdx + JB LBB0_19 + +LBB0_17: + LONG $0x487cd162; WORD $0x2228 // vmovaps (%r10), %zmm4 + LONG $0x487cd162; WORD $0x6a28; BYTE $0x01 // vmovaps 64(%r10), %zmm5 + LONG $0x487cd162; WORD $0x7228; BYTE $0x02 // vmovaps 128(%r10), %zmm6 + LONG $0x487cd162; WORD $0x7a28; BYTE $0x03 // vmovaps 192(%r10), %zmm7 + LONG $0x487c7162; WORD $0x0628 // vmovaps (%rsi), %zmm8 + LONG $0x487c7162; WORD $0x4e28; BYTE $0x01 // vmovaps 64(%rsi), %zmm9 + LONG $0x487c7162; WORD $0x5628; BYTE $0x02 // vmovaps 128(%rsi), %zmm10 + LONG $0x487c7162; WORD $0x5e28; BYTE $0x03 // vmovaps 192(%rsi), %zmm11 + LONG $0x485dd262; WORD $0xd0b8 // vfmadd231ps %zmm8, %zmm4, %zmm2 # zmm2 = (zmm4 * zmm8) + zmm2 + LONG $0x4855d262; WORD $0xd9b8 // vfmadd231ps %zmm9, %zmm5, %zmm3 # zmm3 = (zmm5 * zmm9) + zmm3 + LONG $0x485df262; WORD $0xccb8 // vfmadd231ps %zmm4, %zmm4, %zmm1 # zmm1 = (zmm4 * zmm4) + zmm1 + LONG $0x4855f262; WORD $0xcdb8 // vfmadd231ps %zmm5, %zmm5, %zmm1 # zmm1 = (zmm5 * zmm5) + zmm1 + LONG $0x483dd262; WORD $0xc0b8 // vfmadd231ps %zmm8, %zmm8, %zmm0 # zmm0 = (zmm8 * zmm8) + zmm0 + LONG $0x4835d262; WORD $0xc1b8 // vfmadd231ps %zmm9, %zmm9, %zmm0 # zmm0 = (zmm9 * zmm9) + zmm0 + LONG $0x484dd262; WORD $0xd2b8 // vfmadd231ps %zmm10, %zmm6, %zmm2 # zmm2 = (zmm6 * zmm10) + zmm2 + LONG $0x4845d262; WORD $0xdbb8 // vfmadd231ps %zmm11, %zmm7, %zmm3 # zmm3 = (zmm7 * zmm11) + zmm3 + LONG $0x484df262; WORD $0xceb8 // vfmadd231ps %zmm6, %zmm6, %zmm1 # zmm1 = (zmm6 * zmm6) + zmm1 + LONG $0x4845f262; WORD $0xcfb8 // vfmadd231ps %zmm7, %zmm7, %zmm1 # zmm1 = (zmm7 * zmm7) + zmm1 + LONG $0x482dd262; WORD $0xc2b8 // vfmadd231ps %zmm10, %zmm10, %zmm0 # zmm0 = (zmm10 * zmm10) + zmm0 + LONG $0x4825d262; WORD $0xc3b8 // vfmadd231ps %zmm11, %zmm11, %zmm0 # zmm0 = (zmm11 * zmm11) + zmm0 + LONG $0x00c28149; WORD $0x0001; BYTE $0x00 // addq $256, %r10 # imm = 0x100 + LONG $0x00c68148; WORD $0x0001; BYTE $0x00 // addq $256, %rsi # imm = 0x100 + WORD $0x394d; BYTE $0xca // cmpq %r9, %r10 + JNE LBB0_17 + WORD $0x894c; BYTE $0xcf // movq %r9, %rdi + +LBB0_19: + LONG $0x40c18349 // addq $64, %r9 + WORD $0x394c; BYTE $0xc9 // cmpq %r9, %rcx + JB LBB0_22 + LONG $0x487cf162; WORD $0x2728 // vmovaps (%rdi), %zmm4 + LONG $0x487cf162; WORD $0x2e28 // vmovaps (%rsi), %zmm5 + +LBB0_21: + LONG $0x485df262; WORD $0xd5b8 // vfmadd231ps %zmm5, %zmm4, %zmm2 # zmm2 = (zmm4 * zmm5) + zmm2 + LONG $0x485df262; WORD $0xccb8 // vfmadd231ps %zmm4, %zmm4, %zmm1 # zmm1 = (zmm4 * zmm4) + zmm1 + LONG $0x4855f262; WORD $0xc5b8 // vfmadd231ps %zmm5, %zmm5, %zmm0 # zmm0 = (zmm5 * zmm5) + zmm0 + LONG $0x40c78348 // addq $64, %rdi + LONG $0x40c68348 // addq $64, %rsi + +LBB0_22: + LONG $0x4864f162; WORD $0xd258 // vaddps %zmm2, %zmm3, %zmm2 + WORD $0x3948; BYTE $0xcf // cmpq %rcx, %rdi + JE LBB0_24 + WORD $0xf929 // subl %edi, %ecx + WORD $0xe9c1; BYTE $0x02 // shrl $2, %ecx + LONG $0xffffffba; BYTE $0xff // movl $-1, %edx + WORD $0xe2d3 // shll %cl, %edx + WORD $0xd2f7 // notl %edx + LONG $0xca92f8c5 // kmovw %edx, %k1 + LONG $0xc97cf162; WORD $0x1f10 // vmovups (%rdi), %zmm3 {%k1} {z} + LONG $0xc97cf162; WORD $0x2610 // vmovups (%rsi), %zmm4 {%k1} {z} + LONG $0x4965f262; WORD $0xd4b8 // vfmadd231ps %zmm4, %zmm3, %zmm2 {%k1} # zmm2 {%k1} = (zmm3 * zmm4) + zmm2 + LONG $0x4965f262; WORD $0xcbb8 // vfmadd231ps %zmm3, %zmm3, %zmm1 {%k1} # zmm1 {%k1} = (zmm3 * zmm3) + zmm1 + LONG $0x495df262; WORD $0xc4b8 // vfmadd231ps %zmm4, %zmm4, %zmm0 {%k1} # zmm0 {%k1} = (zmm4 * zmm4) + zmm0 + +LBB0_24: + LONG $0x48fdf362; WORD $0xcb1b; BYTE $0x01 // vextractf64x4 $1, %zmm1, %ymm3 + LONG $0xcb58f4c5 // vaddps %ymm3, %ymm1, %ymm1 + LONG $0xc97cf7c5 // vhaddps %ymm1, %ymm1, %ymm1 + LONG $0xc97cf7c5 // vhaddps %ymm1, %ymm1, %ymm1 + LONG $0x48fdf362; WORD $0xc31b; BYTE $0x01 // vextractf64x4 $1, %zmm0, %ymm3 + LONG $0xc358fcc5 // vaddps %ymm3, %ymm0, %ymm0 + LONG $0xc07cffc5 // vhaddps %ymm0, %ymm0, %ymm0 + LONG $0x197de3c4; WORD $0x01cb // vextractf128 $1, %ymm1, %xmm3 + LONG $0xc07cffc5 // vhaddps %ymm0, %ymm0, %ymm0 + LONG $0xcb58f2c5 // vaddss %xmm3, %xmm1, %xmm1 + LONG $0x0811fac5 // vmovss %xmm1, (%rax) + LONG $0x197de3c4; WORD $0x01c1 // vextractf128 $1, %ymm0, %xmm1 + LONG $0x48fdf362; WORD $0xd31b; BYTE $0x01 // vextractf64x4 $1, %zmm2, %ymm3 + LONG $0xd358ecc5 // vaddps %ymm3, %ymm2, %ymm2 + LONG $0xd27cefc5 // vhaddps %ymm2, %ymm2, %ymm2 + LONG $0xc158fac5 // vaddss %xmm1, %xmm0, %xmm0 + LONG $0xca7cefc5 // vhaddps %ymm2, %ymm2, %ymm1 + LONG $0x117ac1c4; BYTE $0x00 // vmovss %xmm0, (%r8) + LONG $0x197de3c4; WORD $0x01c8 // vextractf128 $1, %ymm1, %xmm0 + LONG $0xc058f2c5 // vaddss %xmm0, %xmm1, %xmm0 + WORD $0x8948; BYTE $0xec // movq %rbp, %rsp + BYTE $0x5d // popq %rbp + WORD $0xf8c5; BYTE $0x77 // vzeroupper + MOVSS X0, result+40(FP) + RET diff --git a/internal/ailego/math/mips_euclidean_distance_matrix_fp32_neon.go b/internal/ailego/math/mips_euclidean_distance_matrix_fp32_neon.go new file mode 100644 index 0000000..3b87833 --- /dev/null +++ b/internal/ailego/math/mips_euclidean_distance_matrix_fp32_neon.go @@ -0,0 +1,14 @@ +//go:build !noasm && arm64 +// Code generated by GoAT. DO NOT EDIT. +// versions: +// clang 21.1.8 (6ubuntu1) +// objdump 2.46 +// flags: -O3 +// source: src/mips_euclidean_distance_matrix_fp32_neon.c + +package mathutil + +import "unsafe" + +//go:noescape +func inner_product_and_squared_norm_fp32_neon(lhs, rhs unsafe.Pointer, size int64, dot, lhs_norm, rhs_norm unsafe.Pointer) diff --git a/internal/ailego/math/mips_euclidean_distance_matrix_fp32_neon.s b/internal/ailego/math/mips_euclidean_distance_matrix_fp32_neon.s new file mode 100644 index 0000000..a5420f9 --- /dev/null +++ b/internal/ailego/math/mips_euclidean_distance_matrix_fp32_neon.s @@ -0,0 +1,99 @@ +//go:build !noasm && arm64 +// Code generated by GoAT. DO NOT EDIT. +// versions: +// clang 21.1.8 (6ubuntu1) +// objdump 2.46 +// flags: -O3 +// source: src/mips_euclidean_distance_matrix_fp32_neon.c + +TEXT ·inner_product_and_squared_norm_fp32_neon(SB), $0-48 + MOVD lhs+0(FP), R0 + MOVD rhs+8(FP), R1 + MOVD size+16(FP), R2 + MOVD dot+24(FP), R3 + MOVD lhs_norm+32(FP), R4 + MOVD rhs_norm+40(FP), R5 + WORD $0xd37ef448 // lsl x8, x2, #2 + WORD $0xf27be909 // ands x9, x8, #0xffffffffffffffe0 + WORD $0x8b080008 // add x8, x0, x8 + WORD $0x8b090009 // add x9, x0, x9 + BEQ LBB0_12 + WORD $0x6f00e402 // movi v2.2d, #0000000000000000 + WORD $0x6f00e403 // movi v3.2d, #0000000000000000 + WORD $0x6f00e401 // movi v1.2d, #0000000000000000 + WORD $0x6f00e400 // movi v0.2d, #0000000000000000 + +LBB0_2: + WORD $0xacc11c24 // ldp q4, q7, [x1], #32 + WORD $0xacc11805 // ldp q5, q6, [x0], #32 + WORD $0xeb09001f // cmp x0, x9 + WORD $0x4e24cc80 // fmla v0.4s, v4.4s, v4.4s + WORD $0x4e25cca1 // fmla v1.4s, v5.4s, v5.4s + WORD $0x4e25cc82 // fmla v2.4s, v4.4s, v5.4s + WORD $0x4e26cce3 // fmla v3.4s, v7.4s, v6.4s + WORD $0x4e27cce0 // fmla v0.4s, v7.4s, v7.4s + WORD $0x4e26ccc1 // fmla v1.4s, v6.4s, v6.4s + BNE LBB0_2 + WORD $0xaa0903e0 // mov x0, x9 + WORD $0x91004129 // add x9, x9, #16 + WORD $0xeb09011f // cmp x8, x9 + BLO LBB0_5 + +LBB0_4: + WORD $0x3cc10404 // ldr q4, [x0], #16 + WORD $0x3cc10425 // ldr q5, [x1], #16 + WORD $0x4e24cc81 // fmla v1.4s, v4.4s, v4.4s + WORD $0x4e24cca2 // fmla v2.4s, v5.4s, v4.4s + WORD $0x4e25cca0 // fmla v0.4s, v5.4s, v5.4s + +LBB0_5: + WORD $0x4e22d462 // fadd v2.4s, v3.4s, v2.4s + WORD $0x6e21d421 // faddp v1.4s, v1.4s, v1.4s + WORD $0xcb000108 // sub x8, x8, x0 + WORD $0x6e20d403 // faddp v3.4s, v0.4s, v0.4s + WORD $0x9342fd08 // asr x8, x8, #2 + WORD $0xf100051f // cmp x8, #1 + WORD $0x6e22d442 // faddp v2.4s, v2.4s, v2.4s + WORD $0x7e30d821 // faddp s1, v1.2s + WORD $0x7e30d840 // faddp s0, v2.2s + WORD $0x7e30d862 // faddp s2, v3.2s + BEQ LBB0_10 + WORD $0xf100091f // cmp x8, #2 + BEQ LBB0_9 + WORD $0xf1000d1f // cmp x8, #3 + BNE LBB0_11 + WORD $0xbd400803 // ldr s3, [x0, #8] + WORD $0xbd400824 // ldr s4, [x1, #8] + WORD $0x1f040060 // fmadd s0, s3, s4, s0 + WORD $0x1f030461 // fmadd s1, s3, s3, s1 + WORD $0x1f040882 // fmadd s2, s4, s4, s2 + +LBB0_9: + WORD $0xbd400403 // ldr s3, [x0, #4] + WORD $0xbd400424 // ldr s4, [x1, #4] + WORD $0x1f040060 // fmadd s0, s3, s4, s0 + WORD $0x1f030461 // fmadd s1, s3, s3, s1 + WORD $0x1f040882 // fmadd s2, s4, s4, s2 + +LBB0_10: + WORD $0xbd400003 // ldr s3, [x0] + WORD $0xbd400024 // ldr s4, [x1] + WORD $0x1f040060 // fmadd s0, s3, s4, s0 + WORD $0x1f030461 // fmadd s1, s3, s3, s1 + WORD $0x1f040882 // fmadd s2, s4, s4, s2 + +LBB0_11: + WORD $0xbd000060 // str s0, [x3] + WORD $0xbd000081 // str s1, [x4] + WORD $0xbd0000a2 // str s2, [x5] + RET + +LBB0_12: + WORD $0x6f00e400 // movi v0.2d, #0000000000000000 + WORD $0x6f00e401 // movi v1.2d, #0000000000000000 + WORD $0x6f00e403 // movi v3.2d, #0000000000000000 + WORD $0x6f00e402 // movi v2.2d, #0000000000000000 + WORD $0x91004129 // add x9, x9, #16 + WORD $0xeb09011f // cmp x8, x9 + BHS LBB0_4 + B LBB0_5 diff --git a/internal/ailego/math/mips_euclidean_distance_matrix_fp32_sse.go b/internal/ailego/math/mips_euclidean_distance_matrix_fp32_sse.go new file mode 100644 index 0000000..ce20fc3 --- /dev/null +++ b/internal/ailego/math/mips_euclidean_distance_matrix_fp32_sse.go @@ -0,0 +1,14 @@ +//go:build !noasm && amd64 +// Code generated by GoAT. DO NOT EDIT. +// versions: +// clang 21.1.8 (6ubuntu1) +// objdump 2.46 +// flags: -msse2 -O3 +// source: src/mips_euclidean_distance_matrix_fp32_sse.c + +package mathutil + +import "unsafe" + +//go:noescape +func inner_product_and_squared_norm_fp32_sse(lhs, rhs unsafe.Pointer, size int64, lhs_norm, rhs_norm unsafe.Pointer) (result float32) diff --git a/internal/ailego/math/mips_euclidean_distance_matrix_fp32_sse.s b/internal/ailego/math/mips_euclidean_distance_matrix_fp32_sse.s new file mode 100644 index 0000000..0465b15 --- /dev/null +++ b/internal/ailego/math/mips_euclidean_distance_matrix_fp32_sse.s @@ -0,0 +1,195 @@ +//go:build !noasm && amd64 +// Code generated by GoAT. DO NOT EDIT. +// versions: +// clang 21.1.8 (6ubuntu1) +// objdump 2.46 +// flags: -msse2 -O3 +// source: src/mips_euclidean_distance_matrix_fp32_sse.c + +TEXT ·inner_product_and_squared_norm_fp32_sse(SB), $8-44 + MOVQ lhs+0(FP), DI + MOVQ rhs+8(FP), SI + MOVQ size+16(FP), DX + MOVQ lhs_norm+24(FP), CX + MOVQ rhs_norm+32(FP), R8 + BYTE $0x55 // pushq %rbp + WORD $0x8948; BYTE $0xe5 // movq %rsp, %rbp + LONG $0xf8e48348 // andq $-8, %rsp + LONG $0x97048d48 // leaq (%rdi,%rdx,4), %rax + LONG $0x02e2c148 // shlq $2, %rdx + LONG $0xe0e28348 // andq $-32, %rdx + LONG $0x170c8d4c // leaq (%rdi,%rdx), %r9 + WORD $0x8941; BYTE $0xf2 // movl %esi, %r10d + WORD $0x0941; BYTE $0xfa // orl %edi, %r10d + LONG $0x0fc2f641 // testb $15, %r10b + JE LBB0_3 + WORD $0x8548; BYTE $0xd2 // testq %rdx, %rdx + JE LBB0_2 + WORD $0x570f; BYTE $0xc0 // xorps %xmm0, %xmm0 + WORD $0x570f; BYTE $0xd2 // xorps %xmm2, %xmm2 + WORD $0x570f; BYTE $0xc9 // xorps %xmm1, %xmm1 + +LBB0_11: + WORD $0x100f; BYTE $0x1f // movups (%rdi), %xmm3 + WORD $0x280f; BYTE $0xe3 // movaps %xmm3, %xmm4 + WORD $0x590f; BYTE $0xdb // mulps %xmm3, %xmm3 + WORD $0x580f; BYTE $0xda // addps %xmm2, %xmm3 + LONG $0x1057100f // movups 16(%rdi), %xmm2 + WORD $0x100f; BYTE $0x2e // movups (%rsi), %xmm5 + WORD $0x590f; BYTE $0xe5 // mulps %xmm5, %xmm4 + WORD $0x590f; BYTE $0xed // mulps %xmm5, %xmm5 + WORD $0x580f; BYTE $0xe9 // addps %xmm1, %xmm5 + LONG $0x104e100f // movups 16(%rsi), %xmm1 + WORD $0x580f; BYTE $0xe0 // addps %xmm0, %xmm4 + WORD $0x280f; BYTE $0xc2 // movaps %xmm2, %xmm0 + WORD $0x590f; BYTE $0xc1 // mulps %xmm1, %xmm0 + WORD $0x580f; BYTE $0xc4 // addps %xmm4, %xmm0 + WORD $0x590f; BYTE $0xd2 // mulps %xmm2, %xmm2 + WORD $0x580f; BYTE $0xd3 // addps %xmm3, %xmm2 + WORD $0x590f; BYTE $0xc9 // mulps %xmm1, %xmm1 + WORD $0x580f; BYTE $0xcd // addps %xmm5, %xmm1 + LONG $0x20c78348 // addq $32, %rdi + LONG $0x20c68348 // addq $32, %rsi + WORD $0x394c; BYTE $0xcf // cmpq %r9, %rdi + JNE LBB0_11 + WORD $0x894c; BYTE $0xcf // movq %r9, %rdi + LONG $0x10c18349 // addq $16, %r9 + WORD $0x394c; BYTE $0xc8 // cmpq %r9, %rax + JAE LBB0_14 + JMP LBB0_16 + +LBB0_3: + WORD $0x8548; BYTE $0xd2 // testq %rdx, %rdx + JE LBB0_4 + WORD $0x570f; BYTE $0xc0 // xorps %xmm0, %xmm0 + WORD $0x570f; BYTE $0xd2 // xorps %xmm2, %xmm2 + WORD $0x570f; BYTE $0xc9 // xorps %xmm1, %xmm1 + +LBB0_6: + WORD $0x280f; BYTE $0x1f // movaps (%rdi), %xmm3 + WORD $0x280f; BYTE $0xe3 // movaps %xmm3, %xmm4 + WORD $0x590f; BYTE $0xdb // mulps %xmm3, %xmm3 + WORD $0x580f; BYTE $0xda // addps %xmm2, %xmm3 + LONG $0x1057280f // movaps 16(%rdi), %xmm2 + WORD $0x280f; BYTE $0x2e // movaps (%rsi), %xmm5 + WORD $0x590f; BYTE $0xe5 // mulps %xmm5, %xmm4 + WORD $0x590f; BYTE $0xed // mulps %xmm5, %xmm5 + WORD $0x580f; BYTE $0xe9 // addps %xmm1, %xmm5 + LONG $0x104e280f // movaps 16(%rsi), %xmm1 + WORD $0x580f; BYTE $0xe0 // addps %xmm0, %xmm4 + WORD $0x280f; BYTE $0xc2 // movaps %xmm2, %xmm0 + WORD $0x590f; BYTE $0xc1 // mulps %xmm1, %xmm0 + WORD $0x580f; BYTE $0xc4 // addps %xmm4, %xmm0 + WORD $0x590f; BYTE $0xd2 // mulps %xmm2, %xmm2 + WORD $0x580f; BYTE $0xd3 // addps %xmm3, %xmm2 + WORD $0x590f; BYTE $0xc9 // mulps %xmm1, %xmm1 + WORD $0x580f; BYTE $0xcd // addps %xmm5, %xmm1 + LONG $0x20c78348 // addq $32, %rdi + LONG $0x20c68348 // addq $32, %rsi + WORD $0x394c; BYTE $0xcf // cmpq %r9, %rdi + JNE LBB0_6 + WORD $0x894c; BYTE $0xcf // movq %r9, %rdi + LONG $0x10c18349 // addq $16, %r9 + WORD $0x394c; BYTE $0xc8 // cmpq %r9, %rax + JAE LBB0_9 + JMP LBB0_16 + +LBB0_2: + WORD $0x570f; BYTE $0xc9 // xorps %xmm1, %xmm1 + WORD $0x570f; BYTE $0xd2 // xorps %xmm2, %xmm2 + WORD $0x570f; BYTE $0xc0 // xorps %xmm0, %xmm0 + LONG $0x10c18349 // addq $16, %r9 + WORD $0x394c; BYTE $0xc8 // cmpq %r9, %rax + JB LBB0_16 + +LBB0_14: + WORD $0x100f; BYTE $0x1f // movups (%rdi), %xmm3 + WORD $0x100f; BYTE $0x26 // movups (%rsi), %xmm4 + JMP LBB0_15 + +LBB0_4: + WORD $0x570f; BYTE $0xc9 // xorps %xmm1, %xmm1 + WORD $0x570f; BYTE $0xd2 // xorps %xmm2, %xmm2 + WORD $0x570f; BYTE $0xc0 // xorps %xmm0, %xmm0 + LONG $0x10c18349 // addq $16, %r9 + WORD $0x394c; BYTE $0xc8 // cmpq %r9, %rax + JB LBB0_16 + +LBB0_9: + WORD $0x280f; BYTE $0x1f // movaps (%rdi), %xmm3 + WORD $0x280f; BYTE $0x26 // movaps (%rsi), %xmm4 + +LBB0_15: + WORD $0x280f; BYTE $0xeb // movaps %xmm3, %xmm5 + WORD $0x590f; BYTE $0xec // mulps %xmm4, %xmm5 + WORD $0x580f; BYTE $0xc5 // addps %xmm5, %xmm0 + WORD $0x590f; BYTE $0xdb // mulps %xmm3, %xmm3 + WORD $0x580f; BYTE $0xd3 // addps %xmm3, %xmm2 + WORD $0x590f; BYTE $0xe4 // mulps %xmm4, %xmm4 + WORD $0x580f; BYTE $0xcc // addps %xmm4, %xmm1 + LONG $0x10c78348 // addq $16, %rdi + LONG $0x10c68348 // addq $16, %rsi + +LBB0_16: + WORD $0x280f; BYTE $0xd8 // movaps %xmm0, %xmm3 + LONG $0xd8150f66 // unpckhpd %xmm0, %xmm3 # xmm3 = xmm3[1],xmm0[1] + WORD $0x580f; BYTE $0xd8 // addps %xmm0, %xmm3 + WORD $0x280f; BYTE $0xc3 // movaps %xmm3, %xmm0 + LONG $0x55c3c60f // shufps $85, %xmm3, %xmm0 # xmm0 = xmm0[1,1],xmm3[1,1] + LONG $0xc3580ff3 // addss %xmm3, %xmm0 + WORD $0x280f; BYTE $0xda // movaps %xmm2, %xmm3 + LONG $0xda150f66 // unpckhpd %xmm2, %xmm3 # xmm3 = xmm3[1],xmm2[1] + WORD $0x580f; BYTE $0xda // addps %xmm2, %xmm3 + WORD $0x280f; BYTE $0xd1 // movaps %xmm1, %xmm2 + LONG $0xd1150f66 // unpckhpd %xmm1, %xmm2 # xmm2 = xmm2[1],xmm1[1] + WORD $0x580f; BYTE $0xd1 // addps %xmm1, %xmm2 + WORD $0x280f; BYTE $0xcb // movaps %xmm3, %xmm1 + WORD $0x140f; BYTE $0xca // unpcklps %xmm2, %xmm1 # xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1] + LONG $0x11d3c60f // shufps $17, %xmm3, %xmm2 # xmm2 = xmm2[1,0],xmm3[1,0] + LONG $0xe2d3c60f // shufps $226, %xmm3, %xmm2 # xmm2 = xmm2[2,0],xmm3[2,3] + WORD $0x580f; BYTE $0xd1 // addps %xmm1, %xmm2 + WORD $0x2948; BYTE $0xf8 // subq %rdi, %rax + LONG $0x02f8c148 // sarq $2, %rax + LONG $0x01f88348 // cmpq $1, %rax + JE LBB0_21 + LONG $0x02f88348 // cmpq $2, %rax + JE LBB0_20 + LONG $0x03f88348 // cmpq $3, %rax + JNE LBB0_22 + LONG $0x4f100ff3; BYTE $0x08 // movss 8(%rdi), %xmm1 # xmm1 = mem[0],zero,zero,zero + LONG $0x5e100ff3; BYTE $0x08 // movss 8(%rsi), %xmm3 # xmm3 = mem[0],zero,zero,zero + WORD $0x280f; BYTE $0xe1 // movaps %xmm1, %xmm4 + LONG $0xe3590ff3 // mulss %xmm3, %xmm4 + LONG $0xc4580ff3 // addss %xmm4, %xmm0 + WORD $0x140f; BYTE $0xcb // unpcklps %xmm3, %xmm1 # xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1] + WORD $0x590f; BYTE $0xc9 // mulps %xmm1, %xmm1 + WORD $0x580f; BYTE $0xd1 // addps %xmm1, %xmm2 + +LBB0_20: + LONG $0x4f100ff3; BYTE $0x04 // movss 4(%rdi), %xmm1 # xmm1 = mem[0],zero,zero,zero + LONG $0x5e100ff3; BYTE $0x04 // movss 4(%rsi), %xmm3 # xmm3 = mem[0],zero,zero,zero + WORD $0x280f; BYTE $0xe1 // movaps %xmm1, %xmm4 + LONG $0xe3590ff3 // mulss %xmm3, %xmm4 + LONG $0xc4580ff3 // addss %xmm4, %xmm0 + WORD $0x140f; BYTE $0xcb // unpcklps %xmm3, %xmm1 # xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1] + WORD $0x590f; BYTE $0xc9 // mulps %xmm1, %xmm1 + WORD $0x580f; BYTE $0xd1 // addps %xmm1, %xmm2 + +LBB0_21: + LONG $0x0f100ff3 // movss (%rdi), %xmm1 # xmm1 = mem[0],zero,zero,zero + LONG $0x1e100ff3 // movss (%rsi), %xmm3 # xmm3 = mem[0],zero,zero,zero + WORD $0x280f; BYTE $0xe1 // movaps %xmm1, %xmm4 + LONG $0xe3590ff3 // mulss %xmm3, %xmm4 + LONG $0xc4580ff3 // addss %xmm4, %xmm0 + WORD $0x140f; BYTE $0xcb // unpcklps %xmm3, %xmm1 # xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1] + WORD $0x590f; BYTE $0xc9 // mulps %xmm1, %xmm1 + WORD $0x580f; BYTE $0xd1 // addps %xmm1, %xmm2 + +LBB0_22: + LONG $0x11110ff3 // movss %xmm2, (%rcx) + LONG $0x55d2c60f // shufps $85, %xmm2, %xmm2 # xmm2 = xmm2[1,1,1,1] + LONG $0x110f41f3; BYTE $0x10 // movss %xmm2, (%r8) + WORD $0x8948; BYTE $0xec // movq %rbp, %rsp + BYTE $0x5d // popq %rbp + MOVSS X0, result+40(FP) + RET diff --git a/internal/ailego/math/src/distance_matrix_fp32.h b/internal/ailego/math/src/distance_matrix_fp32.h new file mode 100644 index 0000000..11da030 --- /dev/null +++ b/internal/ailego/math/src/distance_matrix_fp32.h @@ -0,0 +1,60 @@ +// Copyright 2026-present the xvec project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef XVEC_DISTANCE_MATRIX_FP32_H_ +#define XVEC_DISTANCE_MATRIX_FP32_H_ + +#include +#include + +#if !defined(__FMA__) +#define _mm_fmadd_ps(a, b, c) _mm_add_ps(_mm_mul_ps((a), (b)), (c)) +#define _mm256_fmadd_ps(a, b, c) _mm256_add_ps(_mm256_mul_ps((a), (b)), (c)) +#endif + +static inline float horizontal_add_fp32_v128(__m128 value) { +#if defined(__SSE3__) + __m128 x1 = _mm_hadd_ps(value, value); + __m128 x2 = _mm_hadd_ps(x1, x1); + return _mm_cvtss_f32(x2); +#else + __m128 x1 = _mm_movehl_ps(value, value); + __m128 x2 = _mm_add_ps(value, x1); + __m128 x3 = _mm_shuffle_ps(x2, x2, 1); + __m128 x4 = _mm_add_ss(x2, x3); + return _mm_cvtss_f32(x4); +#endif +} + +static inline float horizontal_add_fp32_v256(__m256 value) { + __m256 x1 = _mm256_hadd_ps(value, value); + __m256 x2 = _mm256_hadd_ps(x1, x1); + __m128 x3 = _mm256_extractf128_ps(x2, 1); + __m128 x4 = _mm_add_ss(_mm256_castps256_ps128(x2), x3); + return _mm_cvtss_f32(x4); +} + +#if defined(__AVX512F__) +static inline float horizontal_add_fp32_v512(__m512 value) { + __m256 low = _mm512_castps512_ps256(value); + __m256 high = _mm256_castpd_ps( + _mm512_extractf64x4_pd(_mm512_castps_pd(value), 1)); + return horizontal_add_fp32_v256(_mm256_add_ps(low, high)); +} +#endif + +#define FMA_FP32_AVX512(m, q, sum) sum = _mm512_fmadd_ps((m), (q), (sum)); +#define FMA_MASK_FP32_AVX512(m, q, sum, mask) sum = _mm512_mask3_fmadd_ps((m), (q), (sum), (mask)); + +#endif diff --git a/internal/floats/src/floats_batch_avx.c b/internal/ailego/math/src/distance_utility_batch_avx.c similarity index 100% rename from internal/floats/src/floats_batch_avx.c rename to internal/ailego/math/src/distance_utility_batch_avx.c diff --git a/internal/floats/src/floats_lasx.c b/internal/ailego/math/src/distance_utility_lasx.c similarity index 100% rename from internal/floats/src/floats_lasx.c rename to internal/ailego/math/src/distance_utility_lasx.c diff --git a/internal/floats/src/floats_rvv.c b/internal/ailego/math/src/distance_utility_rvv.c similarity index 100% rename from internal/floats/src/floats_rvv.c rename to internal/ailego/math/src/distance_utility_rvv.c diff --git a/internal/ailego/math/src/euclidean_distance_matrix_fp32_avx.c b/internal/ailego/math/src/euclidean_distance_matrix_fp32_avx.c new file mode 100644 index 0000000..907dd31 --- /dev/null +++ b/internal/ailego/math/src/euclidean_distance_matrix_fp32_avx.c @@ -0,0 +1,104 @@ +// Copyright 2026-present the xvec project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "distance_matrix_fp32.h" + +float squared_euclidean_distance_fp32_avx(const float *lhs, const float *rhs, + int64_t size) { + const float *last = lhs + size; + const float *last_aligned = lhs + ((size >> 4) << 4); + + __m256 ymm_sum_0 = _mm256_setzero_ps(); + __m256 ymm_sum_1 = _mm256_setzero_ps(); + + if (((uintptr_t)lhs & 0x1f) == 0 && ((uintptr_t)rhs & 0x1f) == 0) { + for (; lhs != last_aligned; lhs += 16, rhs += 16) { + __m256 ymm_d_0 = + _mm256_sub_ps(_mm256_load_ps(lhs + 0), _mm256_load_ps(rhs + 0)); + __m256 ymm_d_1 = + _mm256_sub_ps(_mm256_load_ps(lhs + 8), _mm256_load_ps(rhs + 8)); + ymm_sum_0 = _mm256_fmadd_ps(ymm_d_0, ymm_d_0, ymm_sum_0); + ymm_sum_1 = _mm256_fmadd_ps(ymm_d_1, ymm_d_1, ymm_sum_1); + } + + if (last >= last_aligned + 8) { + __m256 ymm_d = _mm256_sub_ps(_mm256_load_ps(lhs), _mm256_load_ps(rhs)); + ymm_sum_0 = _mm256_fmadd_ps(ymm_d, ymm_d, ymm_sum_0); + lhs += 8; + rhs += 8; + } + } else { + for (; lhs != last_aligned; lhs += 16, rhs += 16) { + __m256 ymm_d_0 = + _mm256_sub_ps(_mm256_loadu_ps(lhs + 0), _mm256_loadu_ps(rhs + 0)); + __m256 ymm_d_1 = + _mm256_sub_ps(_mm256_loadu_ps(lhs + 8), _mm256_loadu_ps(rhs + 8)); + ymm_sum_0 = _mm256_fmadd_ps(ymm_d_0, ymm_d_0, ymm_sum_0); + ymm_sum_1 = _mm256_fmadd_ps(ymm_d_1, ymm_d_1, ymm_sum_1); + } + + if (last >= last_aligned + 8) { + __m256 ymm_d = _mm256_sub_ps(_mm256_loadu_ps(lhs), _mm256_loadu_ps(rhs)); + ymm_sum_0 = _mm256_fmadd_ps(ymm_d, ymm_d, ymm_sum_0); + lhs += 8; + rhs += 8; + } + } + float result = horizontal_add_fp32_v256(_mm256_add_ps(ymm_sum_0, ymm_sum_1)); + + switch (last - lhs) { + case 7: + { + float difference = lhs[6] - rhs[6]; + result += difference * difference; + } + /* FALLTHRU */ + case 6: + { + float difference = lhs[5] - rhs[5]; + result += difference * difference; + } + /* FALLTHRU */ + case 5: + { + float difference = lhs[4] - rhs[4]; + result += difference * difference; + } + /* FALLTHRU */ + case 4: + { + float difference = lhs[3] - rhs[3]; + result += difference * difference; + } + /* FALLTHRU */ + case 3: + { + float difference = lhs[2] - rhs[2]; + result += difference * difference; + } + /* FALLTHRU */ + case 2: + { + float difference = lhs[1] - rhs[1]; + result += difference * difference; + } + /* FALLTHRU */ + case 1: + { + float difference = lhs[0] - rhs[0]; + result += difference * difference; + } + } + return result; +} diff --git a/internal/ailego/math/src/euclidean_distance_matrix_fp32_avx512.c b/internal/ailego/math/src/euclidean_distance_matrix_fp32_avx512.c new file mode 100644 index 0000000..2c697dc --- /dev/null +++ b/internal/ailego/math/src/euclidean_distance_matrix_fp32_avx512.c @@ -0,0 +1,69 @@ +// Copyright 2026-present the xvec project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "distance_matrix_fp32.h" + +float squared_euclidean_distance_fp32_avx512(const float *lhs, const float *rhs, + int64_t size) { + const float *last = lhs + size; + const float *last_aligned = lhs + ((size >> 5) << 5); + + __m512 zmm_sum_0 = _mm512_setzero_ps(); + __m512 zmm_sum_1 = _mm512_setzero_ps(); + + if (((uintptr_t)lhs & 0x3f) == 0 && ((uintptr_t)rhs & 0x3f) == 0) { + for (; lhs != last_aligned; lhs += 32, rhs += 32) { + __m512 zmm_d_0 = + _mm512_sub_ps(_mm512_load_ps(lhs + 0), _mm512_load_ps(rhs + 0)); + __m512 zmm_d_1 = + _mm512_sub_ps(_mm512_load_ps(lhs + 16), _mm512_load_ps(rhs + 16)); + zmm_sum_0 = _mm512_fmadd_ps(zmm_d_0, zmm_d_0, zmm_sum_0); + zmm_sum_1 = _mm512_fmadd_ps(zmm_d_1, zmm_d_1, zmm_sum_1); + } + + if (last >= last_aligned + 16) { + __m512 zmm_d = _mm512_sub_ps(_mm512_load_ps(lhs), _mm512_load_ps(rhs)); + zmm_sum_0 = _mm512_fmadd_ps(zmm_d, zmm_d, zmm_sum_0); + lhs += 16; + rhs += 16; + } + } else { + for (; lhs != last_aligned; lhs += 32, rhs += 32) { + __m512 zmm_d_0 = + _mm512_sub_ps(_mm512_loadu_ps(lhs + 0), _mm512_loadu_ps(rhs + 0)); + __m512 zmm_d_1 = + _mm512_sub_ps(_mm512_loadu_ps(lhs + 16), _mm512_loadu_ps(rhs + 16)); + zmm_sum_0 = _mm512_fmadd_ps(zmm_d_0, zmm_d_0, zmm_sum_0); + zmm_sum_1 = _mm512_fmadd_ps(zmm_d_1, zmm_d_1, zmm_sum_1); + } + + if (last >= last_aligned + 16) { + __m512 zmm_d = _mm512_sub_ps(_mm512_loadu_ps(lhs), _mm512_loadu_ps(rhs)); + zmm_sum_0 = _mm512_fmadd_ps(zmm_d, zmm_d, zmm_sum_0); + lhs += 16; + rhs += 16; + } + } + + zmm_sum_0 = _mm512_add_ps(zmm_sum_0, zmm_sum_1); + if (lhs != last) { + __mmask16 mask = (__mmask16)((1 << (last - lhs)) - 1); + __m512 zmm_undefined = _mm512_undefined_ps(); + __m512 zmm_d = _mm512_mask_sub_ps( + zmm_undefined, mask, _mm512_mask_loadu_ps(zmm_undefined, mask, lhs), + _mm512_mask_loadu_ps(zmm_undefined, mask, rhs)); + zmm_sum_0 = _mm512_mask3_fmadd_ps(zmm_d, zmm_d, zmm_sum_0, mask); + } + return horizontal_add_fp32_v512(zmm_sum_0); +} diff --git a/internal/ailego/math/src/euclidean_distance_matrix_fp32_neon.c b/internal/ailego/math/src/euclidean_distance_matrix_fp32_neon.c new file mode 100644 index 0000000..3987986 --- /dev/null +++ b/internal/ailego/math/src/euclidean_distance_matrix_fp32_neon.c @@ -0,0 +1,57 @@ +// Copyright 2026-present the xvec project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include + +float squared_euclidean_distance_fp32_neon(const float *lhs, const float *rhs, + int64_t size) { + const float *last = lhs + size; + const float *last_aligned = lhs + ((size >> 3) << 3); + + float32x4_t v_sum_0 = vdupq_n_f32(0); + float32x4_t v_sum_1 = vdupq_n_f32(0); + + for (; lhs != last_aligned; lhs += 8, rhs += 8) { + float32x4_t v_d_0 = vsubq_f32(vld1q_f32(lhs + 0), vld1q_f32(rhs + 0)); + float32x4_t v_d_1 = vsubq_f32(vld1q_f32(lhs + 4), vld1q_f32(rhs + 4)); + v_sum_0 = vfmaq_f32(v_sum_0, v_d_0, v_d_0); + v_sum_1 = vfmaq_f32(v_sum_1, v_d_1, v_d_1); + } + if (last >= last_aligned + 4) { + float32x4_t v_d = vsubq_f32(vld1q_f32(lhs), vld1q_f32(rhs)); + v_sum_0 = vfmaq_f32(v_sum_0, v_d, v_d); + lhs += 4; + rhs += 4; + } + + float result = vaddvq_f32(vaddq_f32(v_sum_0, v_sum_1)); + switch (last - lhs) { + case 3: { + float difference = lhs[2] - rhs[2]; + result += difference * difference; + } + /* FALLTHRU */ + case 2: { + float difference = lhs[1] - rhs[1]; + result += difference * difference; + } + /* FALLTHRU */ + case 1: { + float difference = lhs[0] - rhs[0]; + result += difference * difference; + } + } + return result; +} diff --git a/internal/ailego/math/src/euclidean_distance_matrix_fp32_sse.c b/internal/ailego/math/src/euclidean_distance_matrix_fp32_sse.c new file mode 100644 index 0000000..7d19153 --- /dev/null +++ b/internal/ailego/math/src/euclidean_distance_matrix_fp32_sse.c @@ -0,0 +1,76 @@ +// Copyright 2026-present the xvec project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "distance_matrix_fp32.h" + +float squared_euclidean_distance_fp32_sse(const float *lhs, const float *rhs, + int64_t size) { + const float *last = lhs + size; + const float *last_aligned = lhs + ((size >> 3) << 3); + + __m128 xmm_sum_0 = _mm_setzero_ps(); + __m128 xmm_sum_1 = _mm_setzero_ps(); + + if (((uintptr_t)lhs & 0xf) == 0 && ((uintptr_t)rhs & 0xf) == 0) { + for (; lhs != last_aligned; lhs += 8, rhs += 8) { + __m128 xmm_d_0 = _mm_sub_ps(_mm_load_ps(lhs + 0), _mm_load_ps(rhs + 0)); + __m128 xmm_d_1 = _mm_sub_ps(_mm_load_ps(lhs + 4), _mm_load_ps(rhs + 4)); + xmm_sum_0 = _mm_fmadd_ps(xmm_d_0, xmm_d_0, xmm_sum_0); + xmm_sum_1 = _mm_fmadd_ps(xmm_d_1, xmm_d_1, xmm_sum_1); + } + + if (last >= last_aligned + 4) { + __m128 xmm_d = _mm_sub_ps(_mm_load_ps(lhs), _mm_load_ps(rhs)); + xmm_sum_0 = _mm_fmadd_ps(xmm_d, xmm_d, xmm_sum_0); + lhs += 4; + rhs += 4; + } + } else { + for (; lhs != last_aligned; lhs += 8, rhs += 8) { + __m128 xmm_d_0 = _mm_sub_ps(_mm_loadu_ps(lhs + 0), _mm_loadu_ps(rhs + 0)); + __m128 xmm_d_1 = _mm_sub_ps(_mm_loadu_ps(lhs + 4), _mm_loadu_ps(rhs + 4)); + xmm_sum_0 = _mm_fmadd_ps(xmm_d_0, xmm_d_0, xmm_sum_0); + xmm_sum_1 = _mm_fmadd_ps(xmm_d_1, xmm_d_1, xmm_sum_1); + } + + if (last >= last_aligned + 4) { + __m128 xmm_d = _mm_sub_ps(_mm_loadu_ps(lhs), _mm_loadu_ps(rhs)); + xmm_sum_0 = _mm_fmadd_ps(xmm_d, xmm_d, xmm_sum_0); + lhs += 4; + rhs += 4; + } + } + float result = horizontal_add_fp32_v128(_mm_add_ps(xmm_sum_0, xmm_sum_1)); + + switch (last - lhs) { + case 3: + { + float difference = lhs[2] - rhs[2]; + result += difference * difference; + } + /* FALLTHRU */ + case 2: + { + float difference = lhs[1] - rhs[1]; + result += difference * difference; + } + /* FALLTHRU */ + case 1: + { + float difference = lhs[0] - rhs[0]; + result += difference * difference; + } + } + return result; +} diff --git a/internal/ailego/math/src/inner_product_matrix_fp32_avx.c b/internal/ailego/math/src/inner_product_matrix_fp32_avx.c new file mode 100644 index 0000000..08701fc --- /dev/null +++ b/internal/ailego/math/src/inner_product_matrix_fp32_avx.c @@ -0,0 +1,83 @@ +// Copyright 2026-present the xvec project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "distance_matrix_fp32.h" + +float inner_product_fp32_avx(const float *lhs, const float *rhs, + int64_t size) { + const float *last = lhs + size; + const float *last_aligned = lhs + ((size >> 4) << 4); + + __m256 ymm_sum_0 = _mm256_setzero_ps(); + __m256 ymm_sum_1 = _mm256_setzero_ps(); + + if (((uintptr_t)lhs & 0x1f) == 0 && ((uintptr_t)rhs & 0x1f) == 0) { + for (; lhs != last_aligned; lhs += 16, rhs += 16) { + __m256 ymm_lhs_0 = _mm256_load_ps(lhs + 0); + __m256 ymm_lhs_1 = _mm256_load_ps(lhs + 8); + __m256 ymm_rhs_0 = _mm256_load_ps(rhs + 0); + __m256 ymm_rhs_1 = _mm256_load_ps(rhs + 8); + ymm_sum_0 = _mm256_fmadd_ps(ymm_lhs_0, ymm_rhs_0, ymm_sum_0); + ymm_sum_1 = _mm256_fmadd_ps(ymm_lhs_1, ymm_rhs_1, ymm_sum_1); + } + + if (last >= last_aligned + 8) { + ymm_sum_0 = + _mm256_fmadd_ps(_mm256_load_ps(lhs), _mm256_load_ps(rhs), ymm_sum_0); + lhs += 8; + rhs += 8; + } + } else { + for (; lhs != last_aligned; lhs += 16, rhs += 16) { + __m256 ymm_lhs_0 = _mm256_loadu_ps(lhs + 0); + __m256 ymm_lhs_1 = _mm256_loadu_ps(lhs + 8); + __m256 ymm_rhs_0 = _mm256_loadu_ps(rhs + 0); + __m256 ymm_rhs_1 = _mm256_loadu_ps(rhs + 8); + ymm_sum_0 = _mm256_fmadd_ps(ymm_lhs_0, ymm_rhs_0, ymm_sum_0); + ymm_sum_1 = _mm256_fmadd_ps(ymm_lhs_1, ymm_rhs_1, ymm_sum_1); + } + + if (last >= last_aligned + 8) { + ymm_sum_0 = _mm256_fmadd_ps(_mm256_loadu_ps(lhs), _mm256_loadu_ps(rhs), + ymm_sum_0); + lhs += 8; + rhs += 8; + } + } + float result = horizontal_add_fp32_v256(_mm256_add_ps(ymm_sum_0, ymm_sum_1)); + + switch (last - lhs) { + case 7: + result += lhs[6] * rhs[6]; + /* FALLTHRU */ + case 6: + result += lhs[5] * rhs[5]; + /* FALLTHRU */ + case 5: + result += lhs[4] * rhs[4]; + /* FALLTHRU */ + case 4: + result += lhs[3] * rhs[3]; + /* FALLTHRU */ + case 3: + result += lhs[2] * rhs[2]; + /* FALLTHRU */ + case 2: + result += lhs[1] * rhs[1]; + /* FALLTHRU */ + case 1: + result += lhs[0] * rhs[0]; + } + return result; +} diff --git a/internal/ailego/math/src/inner_product_matrix_fp32_avx512.c b/internal/ailego/math/src/inner_product_matrix_fp32_avx512.c new file mode 100644 index 0000000..a28eefc --- /dev/null +++ b/internal/ailego/math/src/inner_product_matrix_fp32_avx512.c @@ -0,0 +1,64 @@ +// Copyright 2026-present the xvec project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "distance_matrix_fp32.h" + +float inner_product_fp32_avx512(const float *lhs, const float *rhs, + int64_t size) { + const float *last = lhs + size; + const float *last_aligned = lhs + ((size >> 5) << 5); + + __m512 zmm_sum_0 = _mm512_setzero_ps(); + __m512 zmm_sum_1 = _mm512_setzero_ps(); + + if (((uintptr_t)lhs & 0x3f) == 0 && ((uintptr_t)rhs & 0x3f) == 0) { + for (; lhs != last_aligned; lhs += 32, rhs += 32) { + FMA_FP32_AVX512(_mm512_load_ps(lhs + 0), _mm512_load_ps(rhs + 0), + zmm_sum_0) + + FMA_FP32_AVX512(_mm512_load_ps(lhs + 16), _mm512_load_ps(rhs + 16), + zmm_sum_1) + } + + if (last >= last_aligned + 16) { + FMA_FP32_AVX512(_mm512_load_ps(lhs), _mm512_load_ps(rhs), zmm_sum_0) + lhs += 16; + rhs += 16; + } + } else { + for (; lhs != last_aligned; lhs += 32, rhs += 32) { + FMA_FP32_AVX512(_mm512_loadu_ps(lhs + 0), _mm512_loadu_ps(rhs + 0), + zmm_sum_0) + + FMA_FP32_AVX512(_mm512_loadu_ps(lhs + 16), _mm512_loadu_ps(rhs + 16), + zmm_sum_1) + } + + if (last >= last_aligned + 16) { + FMA_FP32_AVX512(_mm512_loadu_ps(lhs), _mm512_loadu_ps(rhs), zmm_sum_0) + lhs += 16; + rhs += 16; + } + } + + zmm_sum_0 = _mm512_add_ps(zmm_sum_0, zmm_sum_1); + if (lhs != last) { + __mmask16 mask = (__mmask16)((1 << (last - lhs)) - 1); + __m512 zmm_undefined = _mm512_undefined_ps(); + zmm_sum_0 = _mm512_mask3_fmadd_ps( + _mm512_mask_loadu_ps(zmm_undefined, mask, lhs), + _mm512_mask_loadu_ps(zmm_undefined, mask, rhs), zmm_sum_0, mask); + } + return horizontal_add_fp32_v512(zmm_sum_0); +} diff --git a/internal/ailego/math/src/inner_product_matrix_fp32_neon.c b/internal/ailego/math/src/inner_product_matrix_fp32_neon.c new file mode 100644 index 0000000..8b106bf --- /dev/null +++ b/internal/ailego/math/src/inner_product_matrix_fp32_neon.c @@ -0,0 +1,47 @@ +// Copyright 2026-present the xvec project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include + +float inner_product_fp32_neon(const float *lhs, const float *rhs, int64_t size) { + const float *last = lhs + size; + const float *last_aligned = lhs + ((size >> 3) << 3); + + float32x4_t v_sum_0 = vdupq_n_f32(0); + float32x4_t v_sum_1 = vdupq_n_f32(0); + + for (; lhs != last_aligned; lhs += 8, rhs += 8) { + v_sum_0 = vfmaq_f32(v_sum_0, vld1q_f32(lhs + 0), vld1q_f32(rhs + 0)); + v_sum_1 = vfmaq_f32(v_sum_1, vld1q_f32(lhs + 4), vld1q_f32(rhs + 4)); + } + if (last >= last_aligned + 4) { + v_sum_0 = vfmaq_f32(v_sum_0, vld1q_f32(lhs), vld1q_f32(rhs)); + lhs += 4; + rhs += 4; + } + + float result = vaddvq_f32(vaddq_f32(v_sum_0, v_sum_1)); + switch (last - lhs) { + case 3: + result += lhs[2] * rhs[2]; + /* FALLTHRU */ + case 2: + result += lhs[1] * rhs[1]; + /* FALLTHRU */ + case 1: + result += lhs[0] * rhs[0]; + } + return result; +} diff --git a/internal/ailego/math/src/inner_product_matrix_fp32_sse.c b/internal/ailego/math/src/inner_product_matrix_fp32_sse.c new file mode 100644 index 0000000..053af7e --- /dev/null +++ b/internal/ailego/math/src/inner_product_matrix_fp32_sse.c @@ -0,0 +1,69 @@ +// Copyright 2026-present the xvec project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "distance_matrix_fp32.h" + +float inner_product_fp32_sse(const float *lhs, const float *rhs, + int64_t size) { + const float *last = lhs + size; + const float *last_aligned = lhs + ((size >> 3) << 3); + + __m128 xmm_sum_0 = _mm_setzero_ps(); + __m128 xmm_sum_1 = _mm_setzero_ps(); + + if (((uintptr_t)lhs & 0xf) == 0 && ((uintptr_t)rhs & 0xf) == 0) { + for (; lhs != last_aligned; lhs += 8, rhs += 8) { + __m128 xmm_lhs_0 = _mm_load_ps(lhs + 0); + __m128 xmm_lhs_1 = _mm_load_ps(lhs + 4); + __m128 xmm_rhs_0 = _mm_load_ps(rhs + 0); + __m128 xmm_rhs_1 = _mm_load_ps(rhs + 4); + xmm_sum_0 = _mm_fmadd_ps(xmm_lhs_0, xmm_rhs_0, xmm_sum_0); + xmm_sum_1 = _mm_fmadd_ps(xmm_lhs_1, xmm_rhs_1, xmm_sum_1); + } + + if (last >= last_aligned + 4) { + xmm_sum_0 = _mm_fmadd_ps(_mm_load_ps(lhs), _mm_load_ps(rhs), xmm_sum_0); + lhs += 4; + rhs += 4; + } + } else { + for (; lhs != last_aligned; lhs += 8, rhs += 8) { + __m128 xmm_lhs_0 = _mm_loadu_ps(lhs + 0); + __m128 xmm_lhs_1 = _mm_loadu_ps(lhs + 4); + __m128 xmm_rhs_0 = _mm_loadu_ps(rhs + 0); + __m128 xmm_rhs_1 = _mm_loadu_ps(rhs + 4); + xmm_sum_0 = _mm_fmadd_ps(xmm_lhs_0, xmm_rhs_0, xmm_sum_0); + xmm_sum_1 = _mm_fmadd_ps(xmm_lhs_1, xmm_rhs_1, xmm_sum_1); + } + + if (last >= last_aligned + 4) { + xmm_sum_0 = _mm_fmadd_ps(_mm_loadu_ps(lhs), _mm_loadu_ps(rhs), xmm_sum_0); + lhs += 4; + rhs += 4; + } + } + float result = horizontal_add_fp32_v128(_mm_add_ps(xmm_sum_0, xmm_sum_1)); + + switch (last - lhs) { + case 3: + result += lhs[2] * rhs[2]; + /* FALLTHRU */ + case 2: + result += lhs[1] * rhs[1]; + /* FALLTHRU */ + case 1: + result += lhs[0] * rhs[0]; + } + return result; +} diff --git a/internal/ailego/math/src/mips_euclidean_distance_matrix_fp32_avx.c b/internal/ailego/math/src/mips_euclidean_distance_matrix_fp32_avx.c new file mode 100644 index 0000000..471eb6d --- /dev/null +++ b/internal/ailego/math/src/mips_euclidean_distance_matrix_fp32_avx.c @@ -0,0 +1,118 @@ +// Copyright 2026-present the xvec project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "distance_matrix_fp32.h" + +float inner_product_and_squared_norm_fp32_avx( + const float *lhs, const float *rhs, int64_t size, float *lhs_norm, + float *rhs_norm) { + const float *last = lhs + size; + const float *last_aligned = lhs + ((size >> 4) << 4); + + __m256 ymm_sum_0 = _mm256_setzero_ps(); + __m256 ymm_sum_1 = _mm256_setzero_ps(); + __m256 ymm_sum_norm1 = _mm256_setzero_ps(); + __m256 ymm_sum_norm2 = _mm256_setzero_ps(); + + if (((uintptr_t)lhs & 0x1f) == 0 && ((uintptr_t)rhs & 0x1f) == 0) { + for (; lhs != last_aligned; lhs += 16, rhs += 16) { + __m256 ymm_lhs_0 = _mm256_load_ps(lhs + 0); + __m256 ymm_lhs_1 = _mm256_load_ps(lhs + 8); + __m256 ymm_rhs_0 = _mm256_load_ps(rhs + 0); + __m256 ymm_rhs_1 = _mm256_load_ps(rhs + 8); + ymm_sum_0 = _mm256_fmadd_ps(ymm_lhs_0, ymm_rhs_0, ymm_sum_0); + ymm_sum_1 = _mm256_fmadd_ps(ymm_lhs_1, ymm_rhs_1, ymm_sum_1); + ymm_sum_norm1 = _mm256_fmadd_ps(ymm_lhs_0, ymm_lhs_0, ymm_sum_norm1); + ymm_sum_norm1 = _mm256_fmadd_ps(ymm_lhs_1, ymm_lhs_1, ymm_sum_norm1); + ymm_sum_norm2 = _mm256_fmadd_ps(ymm_rhs_0, ymm_rhs_0, ymm_sum_norm2); + ymm_sum_norm2 = _mm256_fmadd_ps(ymm_rhs_1, ymm_rhs_1, ymm_sum_norm2); + } + + if (last >= last_aligned + 8) { + __m256 ymm_lhs_0 = _mm256_load_ps(lhs); + __m256 ymm_rhs_0 = _mm256_load_ps(rhs); + ymm_sum_0 = _mm256_fmadd_ps(ymm_lhs_0, ymm_rhs_0, ymm_sum_0); + ymm_sum_norm1 = _mm256_fmadd_ps(ymm_lhs_0, ymm_lhs_0, ymm_sum_norm1); + ymm_sum_norm2 = _mm256_fmadd_ps(ymm_rhs_0, ymm_rhs_0, ymm_sum_norm2); + lhs += 8; + rhs += 8; + } + } else { + for (; lhs != last_aligned; lhs += 16, rhs += 16) { + __m256 ymm_lhs_0 = _mm256_loadu_ps(lhs + 0); + __m256 ymm_lhs_1 = _mm256_loadu_ps(lhs + 8); + __m256 ymm_rhs_0 = _mm256_loadu_ps(rhs + 0); + __m256 ymm_rhs_1 = _mm256_loadu_ps(rhs + 8); + ymm_sum_0 = _mm256_fmadd_ps(ymm_lhs_0, ymm_rhs_0, ymm_sum_0); + ymm_sum_1 = _mm256_fmadd_ps(ymm_lhs_1, ymm_rhs_1, ymm_sum_1); + ymm_sum_norm1 = _mm256_fmadd_ps(ymm_lhs_0, ymm_lhs_0, ymm_sum_norm1); + ymm_sum_norm1 = _mm256_fmadd_ps(ymm_lhs_1, ymm_lhs_1, ymm_sum_norm1); + ymm_sum_norm2 = _mm256_fmadd_ps(ymm_rhs_0, ymm_rhs_0, ymm_sum_norm2); + ymm_sum_norm2 = _mm256_fmadd_ps(ymm_rhs_1, ymm_rhs_1, ymm_sum_norm2); + } + + if (last >= last_aligned + 8) { + __m256 ymm_lhs_0 = _mm256_loadu_ps(lhs); + __m256 ymm_rhs_0 = _mm256_loadu_ps(rhs); + ymm_sum_0 = _mm256_fmadd_ps(ymm_lhs_0, ymm_rhs_0, ymm_sum_0); + ymm_sum_norm1 = _mm256_fmadd_ps(ymm_lhs_0, ymm_lhs_0, ymm_sum_norm1); + ymm_sum_norm2 = _mm256_fmadd_ps(ymm_rhs_0, ymm_rhs_0, ymm_sum_norm2); + lhs += 8; + rhs += 8; + } + } + float result = horizontal_add_fp32_v256(_mm256_add_ps(ymm_sum_0, ymm_sum_1)); + float norm1 = horizontal_add_fp32_v256(ymm_sum_norm1); + float norm2 = horizontal_add_fp32_v256(ymm_sum_norm2); + + switch (last - lhs) { + case 7: + result += lhs[6] * rhs[6]; + norm1 += lhs[6] * lhs[6]; + norm2 += rhs[6] * rhs[6]; + /* FALLTHRU */ + case 6: + result += lhs[5] * rhs[5]; + norm1 += lhs[5] * lhs[5]; + norm2 += rhs[5] * rhs[5]; + /* FALLTHRU */ + case 5: + result += lhs[4] * rhs[4]; + norm1 += lhs[4] * lhs[4]; + norm2 += rhs[4] * rhs[4]; + /* FALLTHRU */ + case 4: + result += lhs[3] * rhs[3]; + norm1 += lhs[3] * lhs[3]; + norm2 += rhs[3] * rhs[3]; + /* FALLTHRU */ + case 3: + result += lhs[2] * rhs[2]; + norm1 += lhs[2] * lhs[2]; + norm2 += rhs[2] * rhs[2]; + /* FALLTHRU */ + case 2: + result += lhs[1] * rhs[1]; + norm1 += lhs[1] * lhs[1]; + norm2 += rhs[1] * rhs[1]; + /* FALLTHRU */ + case 1: + result += lhs[0] * rhs[0]; + norm1 += lhs[0] * lhs[0]; + norm2 += rhs[0] * rhs[0]; + } + *lhs_norm = norm1; + *rhs_norm = norm2; + return result; +} diff --git a/internal/ailego/math/src/mips_euclidean_distance_matrix_fp32_avx512.c b/internal/ailego/math/src/mips_euclidean_distance_matrix_fp32_avx512.c new file mode 100644 index 0000000..93ec35c --- /dev/null +++ b/internal/ailego/math/src/mips_euclidean_distance_matrix_fp32_avx512.c @@ -0,0 +1,90 @@ +// Copyright 2026-present the xvec project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "distance_matrix_fp32.h" + +float inner_product_and_squared_norm_fp32_avx512( + const float *lhs, const float *rhs, int64_t size, float *lhs_norm, + float *rhs_norm) { + const float *last = lhs + size; + const float *last_aligned = lhs + ((size >> 5) << 5); + + __m512 zmm_sum_0 = _mm512_setzero_ps(); + __m512 zmm_sum_1 = _mm512_setzero_ps(); + __m512 zmm_sum_norm1 = _mm512_setzero_ps(); + __m512 zmm_sum_norm2 = _mm512_setzero_ps(); + + if (((uintptr_t)lhs & 0x3f) == 0 && ((uintptr_t)rhs & 0x3f) == 0) { + for (; lhs != last_aligned; lhs += 32, rhs += 32) { + __m512 zmm_lhs_0 = _mm512_load_ps(lhs + 0); + __m512 zmm_lhs_1 = _mm512_load_ps(lhs + 16); + __m512 zmm_rhs_0 = _mm512_load_ps(rhs + 0); + __m512 zmm_rhs_1 = _mm512_load_ps(rhs + 16); + FMA_FP32_AVX512(zmm_lhs_0, zmm_rhs_0, zmm_sum_0) + FMA_FP32_AVX512(zmm_lhs_1, zmm_rhs_1, zmm_sum_1) + FMA_FP32_AVX512(zmm_lhs_0, zmm_lhs_0, zmm_sum_norm1) + FMA_FP32_AVX512(zmm_lhs_1, zmm_lhs_1, zmm_sum_norm1) + FMA_FP32_AVX512(zmm_rhs_0, zmm_rhs_0, zmm_sum_norm2) + FMA_FP32_AVX512(zmm_rhs_1, zmm_rhs_1, zmm_sum_norm2) + } + + if (last >= last_aligned + 16) { + __m512 zmm_lhs_0 = _mm512_load_ps(lhs); + __m512 zmm_rhs_0 = _mm512_load_ps(rhs); + FMA_FP32_AVX512(zmm_lhs_0, zmm_rhs_0, zmm_sum_0) + FMA_FP32_AVX512(zmm_lhs_0, zmm_lhs_0, zmm_sum_norm1) + FMA_FP32_AVX512(zmm_rhs_0, zmm_rhs_0, zmm_sum_norm2) + lhs += 16; + rhs += 16; + } + } else { + for (; lhs != last_aligned; lhs += 32, rhs += 32) { + __m512 zmm_lhs_0 = _mm512_loadu_ps(lhs + 0); + __m512 zmm_lhs_1 = _mm512_loadu_ps(lhs + 16); + __m512 zmm_rhs_0 = _mm512_loadu_ps(rhs + 0); + __m512 zmm_rhs_1 = _mm512_loadu_ps(rhs + 16); + FMA_FP32_AVX512(zmm_lhs_0, zmm_rhs_0, zmm_sum_0) + FMA_FP32_AVX512(zmm_lhs_1, zmm_rhs_1, zmm_sum_1) + FMA_FP32_AVX512(zmm_lhs_0, zmm_lhs_0, zmm_sum_norm1) + FMA_FP32_AVX512(zmm_lhs_1, zmm_lhs_1, zmm_sum_norm1) + FMA_FP32_AVX512(zmm_rhs_0, zmm_rhs_0, zmm_sum_norm2) + FMA_FP32_AVX512(zmm_rhs_1, zmm_rhs_1, zmm_sum_norm2) + } + + if (last >= last_aligned + 16) { + __m512 zmm_lhs_0 = _mm512_loadu_ps(lhs); + __m512 zmm_rhs_0 = _mm512_loadu_ps(rhs); + FMA_FP32_AVX512(zmm_lhs_0, zmm_rhs_0, zmm_sum_0) + FMA_FP32_AVX512(zmm_lhs_0, zmm_lhs_0, zmm_sum_norm1) + FMA_FP32_AVX512(zmm_rhs_0, zmm_rhs_0, zmm_sum_norm2) + lhs += 16; + rhs += 16; + } + } + + zmm_sum_0 = _mm512_add_ps(zmm_sum_0, zmm_sum_1); + if (lhs != last) { + __mmask16 mask = (__mmask16)((1 << (last - lhs)) - 1); + __m512 zmm_undefined = _mm512_undefined_ps(); + __m512 zmm_lhs_0 = _mm512_mask_loadu_ps(zmm_undefined, mask, lhs); + __m512 zmm_rhs_0 = _mm512_mask_loadu_ps(zmm_undefined, mask, rhs); + FMA_MASK_FP32_AVX512(zmm_lhs_0, zmm_rhs_0, zmm_sum_0, mask); + FMA_MASK_FP32_AVX512(zmm_lhs_0, zmm_lhs_0, zmm_sum_norm1, mask); + FMA_MASK_FP32_AVX512(zmm_rhs_0, zmm_rhs_0, zmm_sum_norm2, mask); + } + + *lhs_norm = horizontal_add_fp32_v512(zmm_sum_norm1); + *rhs_norm = horizontal_add_fp32_v512(zmm_sum_norm2); + return horizontal_add_fp32_v512(zmm_sum_0); +} diff --git a/internal/ailego/math/src/mips_euclidean_distance_matrix_fp32_neon.c b/internal/ailego/math/src/mips_euclidean_distance_matrix_fp32_neon.c new file mode 100644 index 0000000..8cca0b9 --- /dev/null +++ b/internal/ailego/math/src/mips_euclidean_distance_matrix_fp32_neon.c @@ -0,0 +1,73 @@ +// Copyright 2026-present the xvec project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include + +void inner_product_and_squared_norm_fp32_neon(const float *lhs, const float *rhs, + int64_t size, float *dot, + float *lhs_norm, float *rhs_norm) { + const float *last = lhs + size; + const float *last_aligned = lhs + ((size >> 3) << 3); + + float32x4_t v_sum_0 = vdupq_n_f32(0); + float32x4_t v_sum_1 = vdupq_n_f32(0); + float32x4_t v_sum_norm1 = vdupq_n_f32(0); + float32x4_t v_sum_norm2 = vdupq_n_f32(0); + + for (; lhs != last_aligned; lhs += 8, rhs += 8) { + float32x4_t v_lhs_0 = vld1q_f32(lhs + 0); + float32x4_t v_lhs_1 = vld1q_f32(lhs + 4); + float32x4_t v_rhs_0 = vld1q_f32(rhs + 0); + float32x4_t v_rhs_1 = vld1q_f32(rhs + 4); + v_sum_0 = vfmaq_f32(v_sum_0, v_lhs_0, v_rhs_0); + v_sum_1 = vfmaq_f32(v_sum_1, v_lhs_1, v_rhs_1); + v_sum_norm1 = vfmaq_f32(v_sum_norm1, v_lhs_0, v_lhs_0); + v_sum_norm1 = vfmaq_f32(v_sum_norm1, v_lhs_1, v_lhs_1); + v_sum_norm2 = vfmaq_f32(v_sum_norm2, v_rhs_0, v_rhs_0); + v_sum_norm2 = vfmaq_f32(v_sum_norm2, v_rhs_1, v_rhs_1); + } + if (last >= last_aligned + 4) { + float32x4_t v_lhs_0 = vld1q_f32(lhs); + float32x4_t v_rhs_0 = vld1q_f32(rhs); + v_sum_0 = vfmaq_f32(v_sum_0, v_lhs_0, v_rhs_0); + v_sum_norm1 = vfmaq_f32(v_sum_norm1, v_lhs_0, v_lhs_0); + v_sum_norm2 = vfmaq_f32(v_sum_norm2, v_rhs_0, v_rhs_0); + lhs += 4; + rhs += 4; + } + + float result = vaddvq_f32(vaddq_f32(v_sum_0, v_sum_1)); + float norm1 = vaddvq_f32(v_sum_norm1); + float norm2 = vaddvq_f32(v_sum_norm2); + switch (last - lhs) { + case 3: + result += lhs[2] * rhs[2]; + norm1 += lhs[2] * lhs[2]; + norm2 += rhs[2] * rhs[2]; + /* FALLTHRU */ + case 2: + result += lhs[1] * rhs[1]; + norm1 += lhs[1] * lhs[1]; + norm2 += rhs[1] * rhs[1]; + /* FALLTHRU */ + case 1: + result += lhs[0] * rhs[0]; + norm1 += lhs[0] * lhs[0]; + norm2 += rhs[0] * rhs[0]; + } + *dot = result; + *lhs_norm = norm1; + *rhs_norm = norm2; +} diff --git a/internal/ailego/math/src/mips_euclidean_distance_matrix_fp32_sse.c b/internal/ailego/math/src/mips_euclidean_distance_matrix_fp32_sse.c new file mode 100644 index 0000000..ed5441c --- /dev/null +++ b/internal/ailego/math/src/mips_euclidean_distance_matrix_fp32_sse.c @@ -0,0 +1,97 @@ +// Copyright 2026-present the xvec project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "distance_matrix_fp32.h" + +float inner_product_and_squared_norm_fp32_sse( + const float *lhs, const float *rhs, int64_t size, float *lhs_norm, + float *rhs_norm) { + const float *last = lhs + size; + const float *last_aligned = lhs + ((size >> 3) << 3); + + __m128 xmm_sum = _mm_setzero_ps(); + __m128 xmm_sum_norm1 = _mm_setzero_ps(); + __m128 xmm_sum_norm2 = _mm_setzero_ps(); + + if (((uintptr_t)lhs & 0xf) == 0 && ((uintptr_t)rhs & 0xf) == 0) { + for (; lhs != last_aligned; lhs += 8, rhs += 8) { + __m128 xmm_lhs_0 = _mm_load_ps(lhs + 0); + __m128 xmm_lhs_1 = _mm_load_ps(lhs + 4); + __m128 xmm_rhs_0 = _mm_load_ps(rhs + 0); + __m128 xmm_rhs_1 = _mm_load_ps(rhs + 4); + xmm_sum = _mm_fmadd_ps(xmm_lhs_0, xmm_rhs_0, xmm_sum); + xmm_sum = _mm_fmadd_ps(xmm_lhs_1, xmm_rhs_1, xmm_sum); + xmm_sum_norm1 = _mm_fmadd_ps(xmm_lhs_0, xmm_lhs_0, xmm_sum_norm1); + xmm_sum_norm1 = _mm_fmadd_ps(xmm_lhs_1, xmm_lhs_1, xmm_sum_norm1); + xmm_sum_norm2 = _mm_fmadd_ps(xmm_rhs_0, xmm_rhs_0, xmm_sum_norm2); + xmm_sum_norm2 = _mm_fmadd_ps(xmm_rhs_1, xmm_rhs_1, xmm_sum_norm2); + } + + if (last >= last_aligned + 4) { + __m128 xmm_lhs_0 = _mm_load_ps(lhs); + __m128 xmm_rhs_0 = _mm_load_ps(rhs); + xmm_sum = _mm_fmadd_ps(xmm_lhs_0, xmm_rhs_0, xmm_sum); + xmm_sum_norm1 = _mm_fmadd_ps(xmm_lhs_0, xmm_lhs_0, xmm_sum_norm1); + xmm_sum_norm2 = _mm_fmadd_ps(xmm_rhs_0, xmm_rhs_0, xmm_sum_norm2); + lhs += 4; + rhs += 4; + } + } else { + for (; lhs != last_aligned; lhs += 8, rhs += 8) { + __m128 xmm_lhs_0 = _mm_loadu_ps(lhs + 0); + __m128 xmm_lhs_1 = _mm_loadu_ps(lhs + 4); + __m128 xmm_rhs_0 = _mm_loadu_ps(rhs + 0); + __m128 xmm_rhs_1 = _mm_loadu_ps(rhs + 4); + xmm_sum = _mm_fmadd_ps(xmm_lhs_0, xmm_rhs_0, xmm_sum); + xmm_sum = _mm_fmadd_ps(xmm_lhs_1, xmm_rhs_1, xmm_sum); + xmm_sum_norm1 = _mm_fmadd_ps(xmm_lhs_0, xmm_lhs_0, xmm_sum_norm1); + xmm_sum_norm1 = _mm_fmadd_ps(xmm_lhs_1, xmm_lhs_1, xmm_sum_norm1); + xmm_sum_norm2 = _mm_fmadd_ps(xmm_rhs_0, xmm_rhs_0, xmm_sum_norm2); + xmm_sum_norm2 = _mm_fmadd_ps(xmm_rhs_1, xmm_rhs_1, xmm_sum_norm2); + } + + if (last >= last_aligned + 4) { + __m128 xmm_lhs_0 = _mm_loadu_ps(lhs); + __m128 xmm_rhs_0 = _mm_loadu_ps(rhs); + xmm_sum = _mm_fmadd_ps(xmm_lhs_0, xmm_rhs_0, xmm_sum); + xmm_sum_norm1 = _mm_fmadd_ps(xmm_lhs_0, xmm_lhs_0, xmm_sum_norm1); + xmm_sum_norm2 = _mm_fmadd_ps(xmm_rhs_0, xmm_rhs_0, xmm_sum_norm2); + lhs += 4; + rhs += 4; + } + } + float result = horizontal_add_fp32_v128(xmm_sum); + float norm1 = horizontal_add_fp32_v128(xmm_sum_norm1); + float norm2 = horizontal_add_fp32_v128(xmm_sum_norm2); + + switch (last - lhs) { + case 3: + result += lhs[2] * rhs[2]; + norm1 += lhs[2] * lhs[2]; + norm2 += rhs[2] * rhs[2]; + /* FALLTHRU */ + case 2: + result += lhs[1] * rhs[1]; + norm1 += lhs[1] * lhs[1]; + norm2 += rhs[1] * rhs[1]; + /* FALLTHRU */ + case 1: + result += lhs[0] * rhs[0]; + norm1 += lhs[0] * lhs[0]; + norm2 += rhs[0] * rhs[0]; + } + *lhs_norm = norm1; + *rhs_norm = norm2; + return result; +} diff --git a/internal/floats/Makefile b/internal/floats/Makefile deleted file mode 100644 index 937b20c..0000000 --- a/internal/floats/Makefile +++ /dev/null @@ -1,49 +0,0 @@ -GO ?= go -GOAT ?= $(GO) tool goat -GOAT_TARGET_AVX ?= amd64 -GOAT_TARGET_NEON ?= arm64 -GOAT_TARGET_RVV ?= riscv64 -GOAT_TARGET_LASX ?= loong64 -CLANG_RESOURCE_DIR ?= $(shell clang -print-resource-dir) -GOAT_RVV_INCLUDE ?= $(CLANG_RESOURCE_DIR)/include -LASX_OBJDUMP ?= loongarch64-linux-gnu-objdump -RISCV64_QEMU ?= qemu-riscv64 -RISCV64_QEMU_FLAGS ?= -cpu max,v=true -LOONG64_QEMU ?= qemu-loongarch64 -LOONG64_QEMU_FLAGS ?= -cpu max -TEST_RVV_BINARY ?= .test-rvv.riscv64.test -TEST_LASX_BINARY ?= .test-lasx.loong64.test - -.PHONY: generate avx avx512 neon rvv lasx clean test-rvv test-lasx - -generate: avx avx512 neon rvv lasx clean - -avx: - $(GOAT) src/floats_avx.c --target $(GOAT_TARGET_AVX) -O3 -mavx - -avx512: - $(GOAT) src/floats_avx512.c --target $(GOAT_TARGET_AVX) -O3 -mavx -mfma -mavx512f - -neon: - $(GOAT) src/floats_neon.c --target $(GOAT_TARGET_NEON) -O3 - -rvv: - $(GOAT) src/floats_rvv.c --target $(GOAT_TARGET_RVV) -O3 -march=rv64imafdv -e=-isystem -e=$(GOAT_RVV_INCLUDE) - -lasx: - OBJDUMP=$(LASX_OBJDUMP) $(GOAT) src/floats_lasx.c --target $(GOAT_TARGET_LASX) -O3 -mlasx - -clean: - $(RM) src/*.o src/*.s - -test-rvv: - @set -e; \ - trap '$(RM) ./$(TEST_RVV_BINARY)' EXIT; \ - GOOS=linux GOARCH=riscv64 CGO_ENABLED=0 $(GO) test -c -o $(TEST_RVV_BINARY) .; \ - $(RISCV64_QEMU) $(RISCV64_QEMU_FLAGS) ./$(TEST_RVV_BINARY) - -test-lasx: - @set -e; \ - trap '$(RM) ./$(TEST_LASX_BINARY)' EXIT; \ - GOOS=linux GOARCH=loong64 CGO_ENABLED=0 $(GO) test -c -o $(TEST_LASX_BINARY) .; \ - $(LOONG64_QEMU) $(LOONG64_QEMU_FLAGS) ./$(TEST_LASX_BINARY) diff --git a/internal/floats/floats_avx.go b/internal/floats/floats_avx.go deleted file mode 100644 index 42b33d0..0000000 --- a/internal/floats/floats_avx.go +++ /dev/null @@ -1,20 +0,0 @@ -//go:build !noasm && amd64 -// Code generated by GoAT. DO NOT EDIT. -// versions: -// clang 21.1.8 (6ubuntu1) -// objdump 2.46 -// flags: -mavx -O3 -// source: src/floats_avx.c - -package floats - -import "unsafe" - -//go:noescape -func xvec_avx_l2_squared(left, right unsafe.Pointer, size int64, output unsafe.Pointer) - -//go:noescape -func xvec_avx_inner_product(left, right unsafe.Pointer, size int64, output unsafe.Pointer) - -//go:noescape -func xvec_avx_dot_norms(left, right unsafe.Pointer, size int64, dot, left_norm, right_norm unsafe.Pointer) diff --git a/internal/floats/floats_avx.s b/internal/floats/floats_avx.s deleted file mode 100644 index 229e4f9..0000000 --- a/internal/floats/floats_avx.s +++ /dev/null @@ -1,517 +0,0 @@ -//go:build !noasm && amd64 -// Code generated by GoAT. DO NOT EDIT. -// versions: -// clang 21.1.8 (6ubuntu1) -// objdump 2.46 -// flags: -mavx -O3 -// source: src/floats_avx.c - -TEXT ·xvec_avx_l2_squared(SB), $0-32 - MOVQ left+0(FP), DI - MOVQ right+8(FP), SI - MOVQ size+16(FP), DX - MOVQ output+24(FP), CX - BYTE $0x55 // pushq %rbp - WORD $0x8948; BYTE $0xe5 // movq %rsp, %rbp - LONG $0xf8e48348 // andq $-8, %rsp - LONG $0x07428d4c // leaq 7(%rdx), %r8 - WORD $0x8548; BYTE $0xd2 // testq %rdx, %rdx - LONG $0xc2490f4c // cmovnsq %rdx, %r8 - WORD $0x894d; BYTE $0xc1 // movq %r8, %r9 - LONG $0xf8e18349 // andq $-8, %r9 - WORD $0x8948; BYTE $0xd0 // movq %rdx, %rax - WORD $0x294c; BYTE $0xc8 // subq %r9, %rax - LONG $0xc057f8c5 // vxorps %xmm0, %xmm0, %xmm0 - LONG $0x08fa8348 // cmpq $8, %rdx - JL LBB0_7 - LONG $0x03f8c149 // sarq $3, %r8 - LONG $0xff488d4d // leaq -1(%r8), %r9 - WORD $0x8944; BYTE $0xc2 // movl %r8d, %edx - WORD $0xe283; BYTE $0x03 // andl $3, %edx - LONG $0x03f98349 // cmpq $3, %r9 - JAE LBB0_16 - WORD $0x8949; BYTE $0xf9 // movq %rdi, %r9 - WORD $0x8949; BYTE $0xf2 // movq %rsi, %r10 - JMP LBB0_3 - -LBB0_16: - QUAD $0xfffffffffffcbb49; WORD $0x0fff // movabsq $1152921504606846972, %r11 # imm = 0xFFFFFFFFFFFFFFC - WORD $0x214d; BYTE $0xc3 // andq %r8, %r11 - WORD $0x8949; BYTE $0xf9 // movq %rdi, %r9 - WORD $0x8949; BYTE $0xf2 // movq %rsi, %r10 - -LBB0_17: - LONG $0x107cc1c4; BYTE $0x09 // vmovups (%r9), %ymm1 - LONG $0x107cc1c4; WORD $0x2051 // vmovups 32(%r9), %ymm2 - LONG $0x107cc1c4; WORD $0x4059 // vmovups 64(%r9), %ymm3 - LONG $0x107cc1c4; WORD $0x6061 // vmovups 96(%r9), %ymm4 - LONG $0x5c74c1c4; BYTE $0x0a // vsubps (%r10), %ymm1, %ymm1 - LONG $0xc959f4c5 // vmulps %ymm1, %ymm1, %ymm1 - LONG $0xc158fcc5 // vaddps %ymm1, %ymm0, %ymm0 - LONG $0x5c6cc1c4; WORD $0x204a // vsubps 32(%r10), %ymm2, %ymm1 - LONG $0xc959f4c5 // vmulps %ymm1, %ymm1, %ymm1 - LONG $0xc158fcc5 // vaddps %ymm1, %ymm0, %ymm0 - LONG $0x5c64c1c4; WORD $0x404a // vsubps 64(%r10), %ymm3, %ymm1 - LONG $0xc959f4c5 // vmulps %ymm1, %ymm1, %ymm1 - LONG $0xc158fcc5 // vaddps %ymm1, %ymm0, %ymm0 - LONG $0x5c5cc1c4; WORD $0x604a // vsubps 96(%r10), %ymm4, %ymm1 - LONG $0xc959f4c5 // vmulps %ymm1, %ymm1, %ymm1 - LONG $0xc158fcc5 // vaddps %ymm1, %ymm0, %ymm0 - LONG $0x80e98349 // subq $-128, %r9 - LONG $0x80ea8349 // subq $-128, %r10 - LONG $0xfcc38349 // addq $-4, %r11 - JNE LBB0_17 - -LBB0_3: - WORD $0x8548; BYTE $0xd2 // testq %rdx, %rdx - JE LBB0_6 - WORD $0xe2c1; BYTE $0x05 // shll $5, %edx - WORD $0x3145; BYTE $0xdb // xorl %r11d, %r11d - -LBB0_5: - LONG $0x107c81c4; WORD $0x190c // vmovups (%r9,%r11), %ymm1 - LONG $0x5c7481c4; WORD $0x1a0c // vsubps (%r10,%r11), %ymm1, %ymm1 - LONG $0xc959f4c5 // vmulps %ymm1, %ymm1, %ymm1 - LONG $0xc158fcc5 // vaddps %ymm1, %ymm0, %ymm0 - LONG $0x20c38349 // addq $32, %r11 - WORD $0x394c; BYTE $0xda // cmpq %r11, %rdx - JNE LBB0_5 - -LBB0_6: - LONG $0x05e0c149 // shlq $5, %r8 - WORD $0x014c; BYTE $0xc7 // addq %r8, %rdi - WORD $0x014c; BYTE $0xc6 // addq %r8, %rsi - -LBB0_7: - LONG $0xc816fac5 // vmovshdup %xmm0, %xmm1 # xmm1 = xmm0[1,1,3,3] - LONG $0xc958fac5 // vaddss %xmm1, %xmm0, %xmm1 - LONG $0xd0c6f9c5; BYTE $0x01 // vshufpd $1, %xmm0, %xmm0, %xmm2 # xmm2 = xmm0[1,0] - LONG $0xc958eac5 // vaddss %xmm1, %xmm2, %xmm1 - LONG $0xd0c6f8c5; BYTE $0xff // vshufps $255, %xmm0, %xmm0, %xmm2 # xmm2 = xmm0[3,3,3,3] - LONG $0xc958eac5 // vaddss %xmm1, %xmm2, %xmm1 - LONG $0x197de3c4; WORD $0x01c0 // vextractf128 $1, %ymm0, %xmm0 - LONG $0xc958fac5 // vaddss %xmm1, %xmm0, %xmm1 - LONG $0xd016fac5 // vmovshdup %xmm0, %xmm2 # xmm2 = xmm0[1,1,3,3] - LONG $0xc958eac5 // vaddss %xmm1, %xmm2, %xmm1 - LONG $0xd0c6f9c5; BYTE $0x01 // vshufpd $1, %xmm0, %xmm0, %xmm2 # xmm2 = xmm0[1,0] - LONG $0xc958eac5 // vaddss %xmm1, %xmm2, %xmm1 - LONG $0xc0c6f8c5; BYTE $0xff // vshufps $255, %xmm0, %xmm0, %xmm0 # xmm0 = xmm0[3,3,3,3] - LONG $0xc158fac5 // vaddss %xmm1, %xmm0, %xmm0 - WORD $0x8548; BYTE $0xc0 // testq %rax, %rax - JLE LBB0_15 - LONG $0x0f10fac5 // vmovss (%rdi), %xmm1 # xmm1 = mem[0],zero,zero,zero - LONG $0x0e5cf2c5 // vsubss (%rsi), %xmm1, %xmm1 - LONG $0xc959f2c5 // vmulss %xmm1, %xmm1, %xmm1 - LONG $0xc058f2c5 // vaddss %xmm0, %xmm1, %xmm0 - LONG $0x01f88348 // cmpq $1, %rax - JE LBB0_15 - LONG $0x4f10fac5; BYTE $0x04 // vmovss 4(%rdi), %xmm1 # xmm1 = mem[0],zero,zero,zero - LONG $0x4e5cf2c5; BYTE $0x04 // vsubss 4(%rsi), %xmm1, %xmm1 - LONG $0xc959f2c5 // vmulss %xmm1, %xmm1, %xmm1 - LONG $0xc058f2c5 // vaddss %xmm0, %xmm1, %xmm0 - LONG $0x02f88348 // cmpq $2, %rax - JE LBB0_15 - LONG $0x4f10fac5; BYTE $0x08 // vmovss 8(%rdi), %xmm1 # xmm1 = mem[0],zero,zero,zero - LONG $0x4e5cf2c5; BYTE $0x08 // vsubss 8(%rsi), %xmm1, %xmm1 - LONG $0xc959f2c5 // vmulss %xmm1, %xmm1, %xmm1 - LONG $0xc058f2c5 // vaddss %xmm0, %xmm1, %xmm0 - LONG $0x03f88348 // cmpq $3, %rax - JE LBB0_15 - LONG $0x4f10fac5; BYTE $0x0c // vmovss 12(%rdi), %xmm1 # xmm1 = mem[0],zero,zero,zero - LONG $0x4e5cf2c5; BYTE $0x0c // vsubss 12(%rsi), %xmm1, %xmm1 - LONG $0xc959f2c5 // vmulss %xmm1, %xmm1, %xmm1 - LONG $0xc058f2c5 // vaddss %xmm0, %xmm1, %xmm0 - LONG $0x04f88348 // cmpq $4, %rax - JE LBB0_15 - LONG $0x4f10fac5; BYTE $0x10 // vmovss 16(%rdi), %xmm1 # xmm1 = mem[0],zero,zero,zero - LONG $0x4e5cf2c5; BYTE $0x10 // vsubss 16(%rsi), %xmm1, %xmm1 - LONG $0xc959f2c5 // vmulss %xmm1, %xmm1, %xmm1 - LONG $0xc058f2c5 // vaddss %xmm0, %xmm1, %xmm0 - LONG $0x05f88348 // cmpq $5, %rax - JE LBB0_15 - LONG $0x4f10fac5; BYTE $0x14 // vmovss 20(%rdi), %xmm1 # xmm1 = mem[0],zero,zero,zero - LONG $0x4e5cf2c5; BYTE $0x14 // vsubss 20(%rsi), %xmm1, %xmm1 - LONG $0xc959f2c5 // vmulss %xmm1, %xmm1, %xmm1 - LONG $0xc058f2c5 // vaddss %xmm0, %xmm1, %xmm0 - LONG $0x06f88348 // cmpq $6, %rax - JE LBB0_15 - LONG $0x4f10fac5; BYTE $0x18 // vmovss 24(%rdi), %xmm1 # xmm1 = mem[0],zero,zero,zero - LONG $0x4e5cf2c5; BYTE $0x18 // vsubss 24(%rsi), %xmm1, %xmm1 - LONG $0xc959f2c5 // vmulss %xmm1, %xmm1, %xmm1 - LONG $0xc058f2c5 // vaddss %xmm0, %xmm1, %xmm0 - -LBB0_15: - LONG $0x0111fac5 // vmovss %xmm0, (%rcx) - WORD $0x8948; BYTE $0xec // movq %rbp, %rsp - BYTE $0x5d // popq %rbp - WORD $0xf8c5; BYTE $0x77 // vzeroupper - RET - -TEXT ·xvec_avx_inner_product(SB), $0-32 - MOVQ left+0(FP), DI - MOVQ right+8(FP), SI - MOVQ size+16(FP), DX - MOVQ output+24(FP), CX - BYTE $0x55 // pushq %rbp - WORD $0x8948; BYTE $0xe5 // movq %rsp, %rbp - LONG $0xf8e48348 // andq $-8, %rsp - LONG $0x07428d4c // leaq 7(%rdx), %r8 - WORD $0x8548; BYTE $0xd2 // testq %rdx, %rdx - LONG $0xc2490f4c // cmovnsq %rdx, %r8 - WORD $0x894d; BYTE $0xc1 // movq %r8, %r9 - LONG $0xf8e18349 // andq $-8, %r9 - WORD $0x8948; BYTE $0xd0 // movq %rdx, %rax - WORD $0x294c; BYTE $0xc8 // subq %r9, %rax - LONG $0xc057f8c5 // vxorps %xmm0, %xmm0, %xmm0 - LONG $0x08fa8348 // cmpq $8, %rdx - JL LBB1_7 - LONG $0x03f8c149 // sarq $3, %r8 - LONG $0xff488d4d // leaq -1(%r8), %r9 - WORD $0x8944; BYTE $0xc2 // movl %r8d, %edx - WORD $0xe283; BYTE $0x03 // andl $3, %edx - LONG $0x03f98349 // cmpq $3, %r9 - JAE LBB1_16 - WORD $0x8949; BYTE $0xf9 // movq %rdi, %r9 - WORD $0x8949; BYTE $0xf2 // movq %rsi, %r10 - JMP LBB1_3 - -LBB1_16: - QUAD $0xfffffffffffcbb49; WORD $0x0fff // movabsq $1152921504606846972, %r11 # imm = 0xFFFFFFFFFFFFFFC - WORD $0x214d; BYTE $0xc3 // andq %r8, %r11 - WORD $0x8949; BYTE $0xf9 // movq %rdi, %r9 - WORD $0x8949; BYTE $0xf2 // movq %rsi, %r10 - -LBB1_17: - LONG $0x107cc1c4; BYTE $0x09 // vmovups (%r9), %ymm1 - LONG $0x107cc1c4; WORD $0x2051 // vmovups 32(%r9), %ymm2 - LONG $0x107cc1c4; WORD $0x4059 // vmovups 64(%r9), %ymm3 - LONG $0x107cc1c4; WORD $0x6061 // vmovups 96(%r9), %ymm4 - LONG $0x5974c1c4; BYTE $0x0a // vmulps (%r10), %ymm1, %ymm1 - LONG $0xc158fcc5 // vaddps %ymm1, %ymm0, %ymm0 - LONG $0x596cc1c4; WORD $0x204a // vmulps 32(%r10), %ymm2, %ymm1 - LONG $0x5964c1c4; WORD $0x4052 // vmulps 64(%r10), %ymm3, %ymm2 - LONG $0xc158fcc5 // vaddps %ymm1, %ymm0, %ymm0 - LONG $0xc258fcc5 // vaddps %ymm2, %ymm0, %ymm0 - LONG $0x595cc1c4; WORD $0x604a // vmulps 96(%r10), %ymm4, %ymm1 - LONG $0xc158fcc5 // vaddps %ymm1, %ymm0, %ymm0 - LONG $0x80e98349 // subq $-128, %r9 - LONG $0x80ea8349 // subq $-128, %r10 - LONG $0xfcc38349 // addq $-4, %r11 - JNE LBB1_17 - -LBB1_3: - WORD $0x8548; BYTE $0xd2 // testq %rdx, %rdx - JE LBB1_6 - WORD $0xe2c1; BYTE $0x05 // shll $5, %edx - WORD $0x3145; BYTE $0xdb // xorl %r11d, %r11d - -LBB1_5: - LONG $0x107c81c4; WORD $0x190c // vmovups (%r9,%r11), %ymm1 - LONG $0x597481c4; WORD $0x1a0c // vmulps (%r10,%r11), %ymm1, %ymm1 - LONG $0xc158fcc5 // vaddps %ymm1, %ymm0, %ymm0 - LONG $0x20c38349 // addq $32, %r11 - WORD $0x394c; BYTE $0xda // cmpq %r11, %rdx - JNE LBB1_5 - -LBB1_6: - LONG $0x05e0c149 // shlq $5, %r8 - WORD $0x014c; BYTE $0xc7 // addq %r8, %rdi - WORD $0x014c; BYTE $0xc6 // addq %r8, %rsi - -LBB1_7: - LONG $0xc957f0c5 // vxorps %xmm1, %xmm1, %xmm1 - LONG $0xc958fac5 // vaddss %xmm1, %xmm0, %xmm1 - LONG $0xd016fac5 // vmovshdup %xmm0, %xmm2 # xmm2 = xmm0[1,1,3,3] - LONG $0xc958eac5 // vaddss %xmm1, %xmm2, %xmm1 - LONG $0xd0c6f9c5; BYTE $0x01 // vshufpd $1, %xmm0, %xmm0, %xmm2 # xmm2 = xmm0[1,0] - LONG $0xc958eac5 // vaddss %xmm1, %xmm2, %xmm1 - LONG $0xd0c6f8c5; BYTE $0xff // vshufps $255, %xmm0, %xmm0, %xmm2 # xmm2 = xmm0[3,3,3,3] - LONG $0xc958eac5 // vaddss %xmm1, %xmm2, %xmm1 - LONG $0x197de3c4; WORD $0x01c0 // vextractf128 $1, %ymm0, %xmm0 - LONG $0xc958fac5 // vaddss %xmm1, %xmm0, %xmm1 - LONG $0xd016fac5 // vmovshdup %xmm0, %xmm2 # xmm2 = xmm0[1,1,3,3] - LONG $0xc958eac5 // vaddss %xmm1, %xmm2, %xmm1 - LONG $0xd0c6f9c5; BYTE $0x01 // vshufpd $1, %xmm0, %xmm0, %xmm2 # xmm2 = xmm0[1,0] - LONG $0xc958eac5 // vaddss %xmm1, %xmm2, %xmm1 - LONG $0xc0c6f8c5; BYTE $0xff // vshufps $255, %xmm0, %xmm0, %xmm0 # xmm0 = xmm0[3,3,3,3] - LONG $0xc158fac5 // vaddss %xmm1, %xmm0, %xmm0 - WORD $0x8548; BYTE $0xc0 // testq %rax, %rax - JLE LBB1_15 - LONG $0x0f10fac5 // vmovss (%rdi), %xmm1 # xmm1 = mem[0],zero,zero,zero - LONG $0x0e59f2c5 // vmulss (%rsi), %xmm1, %xmm1 - LONG $0xc058f2c5 // vaddss %xmm0, %xmm1, %xmm0 - LONG $0x01f88348 // cmpq $1, %rax - JE LBB1_15 - LONG $0x4f10fac5; BYTE $0x04 // vmovss 4(%rdi), %xmm1 # xmm1 = mem[0],zero,zero,zero - LONG $0x4e59f2c5; BYTE $0x04 // vmulss 4(%rsi), %xmm1, %xmm1 - LONG $0xc058f2c5 // vaddss %xmm0, %xmm1, %xmm0 - LONG $0x02f88348 // cmpq $2, %rax - JE LBB1_15 - LONG $0x4f10fac5; BYTE $0x08 // vmovss 8(%rdi), %xmm1 # xmm1 = mem[0],zero,zero,zero - LONG $0x4e59f2c5; BYTE $0x08 // vmulss 8(%rsi), %xmm1, %xmm1 - LONG $0xc058f2c5 // vaddss %xmm0, %xmm1, %xmm0 - LONG $0x03f88348 // cmpq $3, %rax - JE LBB1_15 - LONG $0x4f10fac5; BYTE $0x0c // vmovss 12(%rdi), %xmm1 # xmm1 = mem[0],zero,zero,zero - LONG $0x4e59f2c5; BYTE $0x0c // vmulss 12(%rsi), %xmm1, %xmm1 - LONG $0xc058f2c5 // vaddss %xmm0, %xmm1, %xmm0 - LONG $0x04f88348 // cmpq $4, %rax - JE LBB1_15 - LONG $0x4f10fac5; BYTE $0x10 // vmovss 16(%rdi), %xmm1 # xmm1 = mem[0],zero,zero,zero - LONG $0x4e59f2c5; BYTE $0x10 // vmulss 16(%rsi), %xmm1, %xmm1 - LONG $0xc058f2c5 // vaddss %xmm0, %xmm1, %xmm0 - LONG $0x05f88348 // cmpq $5, %rax - JE LBB1_15 - LONG $0x4f10fac5; BYTE $0x14 // vmovss 20(%rdi), %xmm1 # xmm1 = mem[0],zero,zero,zero - LONG $0x4e59f2c5; BYTE $0x14 // vmulss 20(%rsi), %xmm1, %xmm1 - LONG $0xc058f2c5 // vaddss %xmm0, %xmm1, %xmm0 - LONG $0x06f88348 // cmpq $6, %rax - JE LBB1_15 - LONG $0x4f10fac5; BYTE $0x18 // vmovss 24(%rdi), %xmm1 # xmm1 = mem[0],zero,zero,zero - LONG $0x4e59f2c5; BYTE $0x18 // vmulss 24(%rsi), %xmm1, %xmm1 - LONG $0xc058f2c5 // vaddss %xmm0, %xmm1, %xmm0 - -LBB1_15: - LONG $0x0111fac5 // vmovss %xmm0, (%rcx) - WORD $0x8948; BYTE $0xec // movq %rbp, %rsp - BYTE $0x5d // popq %rbp - WORD $0xf8c5; BYTE $0x77 // vzeroupper - RET - -TEXT ·xvec_avx_dot_norms(SB), $0-48 - MOVQ left+0(FP), DI - MOVQ right+8(FP), SI - MOVQ size+16(FP), DX - MOVQ dot+24(FP), CX - MOVQ left_norm+32(FP), R8 - MOVQ right_norm+40(FP), R9 - BYTE $0x55 // pushq %rbp - WORD $0x8948; BYTE $0xe5 // movq %rsp, %rbp - BYTE $0x53 // pushq %rbx - LONG $0xf8e48348 // andq $-8, %rsp - LONG $0x07528d4c // leaq 7(%rdx), %r10 - WORD $0x8548; BYTE $0xd2 // testq %rdx, %rdx - LONG $0xd2490f4c // cmovnsq %rdx, %r10 - WORD $0x894d; BYTE $0xd3 // movq %r10, %r11 - LONG $0xf8e38349 // andq $-8, %r11 - WORD $0x8948; BYTE $0xd0 // movq %rdx, %rax - WORD $0x294c; BYTE $0xd8 // subq %r11, %rax - LONG $0xd257e8c5 // vxorps %xmm2, %xmm2, %xmm2 - LONG $0x08fa8348 // cmpq $8, %rdx - JL LBB2_1 - LONG $0x03fac149 // sarq $3, %r10 - QUAD $0xfffffffffff8bb49; WORD $0x7fff // movabsq $9223372036854775800, %r11 # imm = 0x7FFFFFFFFFFFFFF8 - WORD $0x214c; BYTE $0xda // andq %r11, %rdx - LONG $0x08fa8348 // cmpq $8, %rdx - JNE LBB2_9 - LONG $0xc057f8c5 // vxorps %xmm0, %xmm0, %xmm0 - WORD $0x8948; BYTE $0xfa // movq %rdi, %rdx - WORD $0x8949; BYTE $0xf3 // movq %rsi, %r11 - LONG $0xc957f0c5 // vxorps %xmm1, %xmm1, %xmm1 - JMP LBB2_4 - -LBB2_1: - LONG $0xc957f0c5 // vxorps %xmm1, %xmm1, %xmm1 - LONG $0xc057f8c5 // vxorps %xmm0, %xmm0, %xmm0 - JMP LBB2_7 - -LBB2_9: - QUAD $0xfffffffffffebb48; WORD $0x0fff // movabsq $1152921504606846974, %rbx # imm = 0xFFFFFFFFFFFFFFE - WORD $0x214c; BYTE $0xd3 // andq %r10, %rbx - LONG $0xc057f8c5 // vxorps %xmm0, %xmm0, %xmm0 - WORD $0x8948; BYTE $0xfa // movq %rdi, %rdx - WORD $0x8949; BYTE $0xf3 // movq %rsi, %r11 - LONG $0xc957f0c5 // vxorps %xmm1, %xmm1, %xmm1 - -LBB2_10: - LONG $0x1a10fcc5 // vmovups (%rdx), %ymm3 - LONG $0x6210fcc5; BYTE $0x20 // vmovups 32(%rdx), %ymm4 - LONG $0x107cc1c4; BYTE $0x2b // vmovups (%r11), %ymm5 - LONG $0x107cc1c4; WORD $0x2073 // vmovups 32(%r11), %ymm6 - LONG $0xfd59e4c5 // vmulps %ymm5, %ymm3, %ymm7 - LONG $0xd758ecc5 // vaddps %ymm7, %ymm2, %ymm2 - LONG $0xdb59e4c5 // vmulps %ymm3, %ymm3, %ymm3 - LONG $0xcb58f4c5 // vaddps %ymm3, %ymm1, %ymm1 - LONG $0xdd59d4c5 // vmulps %ymm5, %ymm5, %ymm3 - LONG $0xc358fcc5 // vaddps %ymm3, %ymm0, %ymm0 - LONG $0xde59dcc5 // vmulps %ymm6, %ymm4, %ymm3 - LONG $0xd358ecc5 // vaddps %ymm3, %ymm2, %ymm2 - LONG $0xdc59dcc5 // vmulps %ymm4, %ymm4, %ymm3 - LONG $0xcb58f4c5 // vaddps %ymm3, %ymm1, %ymm1 - LONG $0xde59ccc5 // vmulps %ymm6, %ymm6, %ymm3 - LONG $0xc358fcc5 // vaddps %ymm3, %ymm0, %ymm0 - LONG $0x40c28348 // addq $64, %rdx - LONG $0x40c38349 // addq $64, %r11 - LONG $0xfec38348 // addq $-2, %rbx - JNE LBB2_10 - -LBB2_4: - LONG $0x01c2f641 // testb $1, %r10b - JE LBB2_6 - LONG $0x1a10fcc5 // vmovups (%rdx), %ymm3 - LONG $0x107cc1c4; BYTE $0x23 // vmovups (%r11), %ymm4 - LONG $0xec59e4c5 // vmulps %ymm4, %ymm3, %ymm5 - LONG $0xd558ecc5 // vaddps %ymm5, %ymm2, %ymm2 - LONG $0xdb59e4c5 // vmulps %ymm3, %ymm3, %ymm3 - LONG $0xcb58f4c5 // vaddps %ymm3, %ymm1, %ymm1 - LONG $0xdc59dcc5 // vmulps %ymm4, %ymm4, %ymm3 - LONG $0xc358fcc5 // vaddps %ymm3, %ymm0, %ymm0 - -LBB2_6: - LONG $0x05e2c149 // shlq $5, %r10 - WORD $0x014c; BYTE $0xd7 // addq %r10, %rdi - WORD $0x014c; BYTE $0xd6 // addq %r10, %rsi - -LBB2_7: - LONG $0xdb57e0c5 // vxorps %xmm3, %xmm3, %xmm3 - LONG $0xdb58eac5 // vaddss %xmm3, %xmm2, %xmm3 - LONG $0xe216fac5 // vmovshdup %xmm2, %xmm4 # xmm4 = xmm2[1,1,3,3] - LONG $0xdb58dac5 // vaddss %xmm3, %xmm4, %xmm3 - LONG $0xe2c6e9c5; BYTE $0x01 // vshufpd $1, %xmm2, %xmm2, %xmm4 # xmm4 = xmm2[1,0] - LONG $0xdb58dac5 // vaddss %xmm3, %xmm4, %xmm3 - LONG $0xe2c6e8c5; BYTE $0xff // vshufps $255, %xmm2, %xmm2, %xmm4 # xmm4 = xmm2[3,3,3,3] - LONG $0xdb58dac5 // vaddss %xmm3, %xmm4, %xmm3 - LONG $0x197de3c4; WORD $0x01d2 // vextractf128 $1, %ymm2, %xmm2 - LONG $0xdb58eac5 // vaddss %xmm3, %xmm2, %xmm3 - LONG $0xe216fac5 // vmovshdup %xmm2, %xmm4 # xmm4 = xmm2[1,1,3,3] - LONG $0xdb58dac5 // vaddss %xmm3, %xmm4, %xmm3 - LONG $0xe2c6e9c5; BYTE $0x01 // vshufpd $1, %xmm2, %xmm2, %xmm4 # xmm4 = xmm2[1,0] - LONG $0xdb58dac5 // vaddss %xmm3, %xmm4, %xmm3 - LONG $0xd2c6e8c5; BYTE $0xff // vshufps $255, %xmm2, %xmm2, %xmm2 # xmm2 = xmm2[3,3,3,3] - LONG $0xd358eac5 // vaddss %xmm3, %xmm2, %xmm2 - LONG $0x1111fac5 // vmovss %xmm2, (%rcx) - LONG $0xd116fac5 // vmovshdup %xmm1, %xmm2 # xmm2 = xmm1[1,1,3,3] - LONG $0xd258f2c5 // vaddss %xmm2, %xmm1, %xmm2 - LONG $0xd9c6f1c5; BYTE $0x01 // vshufpd $1, %xmm1, %xmm1, %xmm3 # xmm3 = xmm1[1,0] - LONG $0xd258e2c5 // vaddss %xmm2, %xmm3, %xmm2 - LONG $0xd9c6f0c5; BYTE $0xff // vshufps $255, %xmm1, %xmm1, %xmm3 # xmm3 = xmm1[3,3,3,3] - LONG $0xd258e2c5 // vaddss %xmm2, %xmm3, %xmm2 - LONG $0x197de3c4; WORD $0x01c9 // vextractf128 $1, %ymm1, %xmm1 - LONG $0xd258f2c5 // vaddss %xmm2, %xmm1, %xmm2 - LONG $0xd916fac5 // vmovshdup %xmm1, %xmm3 # xmm3 = xmm1[1,1,3,3] - LONG $0xd258e2c5 // vaddss %xmm2, %xmm3, %xmm2 - LONG $0xd9c6f1c5; BYTE $0x01 // vshufpd $1, %xmm1, %xmm1, %xmm3 # xmm3 = xmm1[1,0] - LONG $0xd258e2c5 // vaddss %xmm2, %xmm3, %xmm2 - LONG $0xc9c6f0c5; BYTE $0xff // vshufps $255, %xmm1, %xmm1, %xmm1 # xmm1 = xmm1[3,3,3,3] - LONG $0xca58f2c5 // vaddss %xmm2, %xmm1, %xmm1 - LONG $0x117ac1c4; BYTE $0x08 // vmovss %xmm1, (%r8) - LONG $0xc816fac5 // vmovshdup %xmm0, %xmm1 # xmm1 = xmm0[1,1,3,3] - LONG $0xc958fac5 // vaddss %xmm1, %xmm0, %xmm1 - LONG $0xd0c6f9c5; BYTE $0x01 // vshufpd $1, %xmm0, %xmm0, %xmm2 # xmm2 = xmm0[1,0] - LONG $0xc958eac5 // vaddss %xmm1, %xmm2, %xmm1 - LONG $0xd0c6f8c5; BYTE $0xff // vshufps $255, %xmm0, %xmm0, %xmm2 # xmm2 = xmm0[3,3,3,3] - LONG $0xc958eac5 // vaddss %xmm1, %xmm2, %xmm1 - LONG $0x197de3c4; WORD $0x01c0 // vextractf128 $1, %ymm0, %xmm0 - LONG $0xc958fac5 // vaddss %xmm1, %xmm0, %xmm1 - LONG $0xd016fac5 // vmovshdup %xmm0, %xmm2 # xmm2 = xmm0[1,1,3,3] - LONG $0xc958eac5 // vaddss %xmm1, %xmm2, %xmm1 - LONG $0xd0c6f9c5; BYTE $0x01 // vshufpd $1, %xmm0, %xmm0, %xmm2 # xmm2 = xmm0[1,0] - LONG $0xc958eac5 // vaddss %xmm1, %xmm2, %xmm1 - LONG $0xc0c6f8c5; BYTE $0xff // vshufps $255, %xmm0, %xmm0, %xmm0 # xmm0 = xmm0[3,3,3,3] - LONG $0xc158fac5 // vaddss %xmm1, %xmm0, %xmm0 - LONG $0x117ac1c4; BYTE $0x01 // vmovss %xmm0, (%r9) - WORD $0x8548; BYTE $0xc0 // testq %rax, %rax - JLE LBB2_8 - LONG $0x0710fac5 // vmovss (%rdi), %xmm0 # xmm0 = mem[0],zero,zero,zero - LONG $0x0659fac5 // vmulss (%rsi), %xmm0, %xmm0 - LONG $0x0158fac5 // vaddss (%rcx), %xmm0, %xmm0 - LONG $0x0111fac5 // vmovss %xmm0, (%rcx) - LONG $0x0710fac5 // vmovss (%rdi), %xmm0 # xmm0 = mem[0],zero,zero,zero - LONG $0xc059fac5 // vmulss %xmm0, %xmm0, %xmm0 - LONG $0x587ac1c4; BYTE $0x00 // vaddss (%r8), %xmm0, %xmm0 - LONG $0x117ac1c4; BYTE $0x00 // vmovss %xmm0, (%r8) - LONG $0x0610fac5 // vmovss (%rsi), %xmm0 # xmm0 = mem[0],zero,zero,zero - LONG $0xc059fac5 // vmulss %xmm0, %xmm0, %xmm0 - LONG $0x587ac1c4; BYTE $0x01 // vaddss (%r9), %xmm0, %xmm0 - LONG $0x117ac1c4; BYTE $0x01 // vmovss %xmm0, (%r9) - LONG $0x01f88348 // cmpq $1, %rax - JE LBB2_8 - LONG $0x4710fac5; BYTE $0x04 // vmovss 4(%rdi), %xmm0 # xmm0 = mem[0],zero,zero,zero - LONG $0x4659fac5; BYTE $0x04 // vmulss 4(%rsi), %xmm0, %xmm0 - LONG $0x0158fac5 // vaddss (%rcx), %xmm0, %xmm0 - LONG $0x0111fac5 // vmovss %xmm0, (%rcx) - LONG $0x4710fac5; BYTE $0x04 // vmovss 4(%rdi), %xmm0 # xmm0 = mem[0],zero,zero,zero - LONG $0xc059fac5 // vmulss %xmm0, %xmm0, %xmm0 - LONG $0x587ac1c4; BYTE $0x00 // vaddss (%r8), %xmm0, %xmm0 - LONG $0x117ac1c4; BYTE $0x00 // vmovss %xmm0, (%r8) - LONG $0x4610fac5; BYTE $0x04 // vmovss 4(%rsi), %xmm0 # xmm0 = mem[0],zero,zero,zero - LONG $0xc059fac5 // vmulss %xmm0, %xmm0, %xmm0 - LONG $0x587ac1c4; BYTE $0x01 // vaddss (%r9), %xmm0, %xmm0 - LONG $0x117ac1c4; BYTE $0x01 // vmovss %xmm0, (%r9) - LONG $0x02f88348 // cmpq $2, %rax - JE LBB2_8 - LONG $0x4710fac5; BYTE $0x08 // vmovss 8(%rdi), %xmm0 # xmm0 = mem[0],zero,zero,zero - LONG $0x4659fac5; BYTE $0x08 // vmulss 8(%rsi), %xmm0, %xmm0 - LONG $0x0158fac5 // vaddss (%rcx), %xmm0, %xmm0 - LONG $0x0111fac5 // vmovss %xmm0, (%rcx) - LONG $0x4710fac5; BYTE $0x08 // vmovss 8(%rdi), %xmm0 # xmm0 = mem[0],zero,zero,zero - LONG $0xc059fac5 // vmulss %xmm0, %xmm0, %xmm0 - LONG $0x587ac1c4; BYTE $0x00 // vaddss (%r8), %xmm0, %xmm0 - LONG $0x117ac1c4; BYTE $0x00 // vmovss %xmm0, (%r8) - LONG $0x4610fac5; BYTE $0x08 // vmovss 8(%rsi), %xmm0 # xmm0 = mem[0],zero,zero,zero - LONG $0xc059fac5 // vmulss %xmm0, %xmm0, %xmm0 - LONG $0x587ac1c4; BYTE $0x01 // vaddss (%r9), %xmm0, %xmm0 - LONG $0x117ac1c4; BYTE $0x01 // vmovss %xmm0, (%r9) - LONG $0x03f88348 // cmpq $3, %rax - JE LBB2_8 - LONG $0x4710fac5; BYTE $0x0c // vmovss 12(%rdi), %xmm0 # xmm0 = mem[0],zero,zero,zero - LONG $0x4659fac5; BYTE $0x0c // vmulss 12(%rsi), %xmm0, %xmm0 - LONG $0x0158fac5 // vaddss (%rcx), %xmm0, %xmm0 - LONG $0x0111fac5 // vmovss %xmm0, (%rcx) - LONG $0x4710fac5; BYTE $0x0c // vmovss 12(%rdi), %xmm0 # xmm0 = mem[0],zero,zero,zero - LONG $0xc059fac5 // vmulss %xmm0, %xmm0, %xmm0 - LONG $0x587ac1c4; BYTE $0x00 // vaddss (%r8), %xmm0, %xmm0 - LONG $0x117ac1c4; BYTE $0x00 // vmovss %xmm0, (%r8) - LONG $0x4610fac5; BYTE $0x0c // vmovss 12(%rsi), %xmm0 # xmm0 = mem[0],zero,zero,zero - LONG $0xc059fac5 // vmulss %xmm0, %xmm0, %xmm0 - LONG $0x587ac1c4; BYTE $0x01 // vaddss (%r9), %xmm0, %xmm0 - LONG $0x117ac1c4; BYTE $0x01 // vmovss %xmm0, (%r9) - LONG $0x04f88348 // cmpq $4, %rax - JE LBB2_8 - LONG $0x4710fac5; BYTE $0x10 // vmovss 16(%rdi), %xmm0 # xmm0 = mem[0],zero,zero,zero - LONG $0x4659fac5; BYTE $0x10 // vmulss 16(%rsi), %xmm0, %xmm0 - LONG $0x0158fac5 // vaddss (%rcx), %xmm0, %xmm0 - LONG $0x0111fac5 // vmovss %xmm0, (%rcx) - LONG $0x4710fac5; BYTE $0x10 // vmovss 16(%rdi), %xmm0 # xmm0 = mem[0],zero,zero,zero - LONG $0xc059fac5 // vmulss %xmm0, %xmm0, %xmm0 - LONG $0x587ac1c4; BYTE $0x00 // vaddss (%r8), %xmm0, %xmm0 - LONG $0x117ac1c4; BYTE $0x00 // vmovss %xmm0, (%r8) - LONG $0x4610fac5; BYTE $0x10 // vmovss 16(%rsi), %xmm0 # xmm0 = mem[0],zero,zero,zero - LONG $0xc059fac5 // vmulss %xmm0, %xmm0, %xmm0 - LONG $0x587ac1c4; BYTE $0x01 // vaddss (%r9), %xmm0, %xmm0 - LONG $0x117ac1c4; BYTE $0x01 // vmovss %xmm0, (%r9) - LONG $0x05f88348 // cmpq $5, %rax - JE LBB2_8 - LONG $0x4710fac5; BYTE $0x14 // vmovss 20(%rdi), %xmm0 # xmm0 = mem[0],zero,zero,zero - LONG $0x4659fac5; BYTE $0x14 // vmulss 20(%rsi), %xmm0, %xmm0 - LONG $0x0158fac5 // vaddss (%rcx), %xmm0, %xmm0 - LONG $0x0111fac5 // vmovss %xmm0, (%rcx) - LONG $0x4710fac5; BYTE $0x14 // vmovss 20(%rdi), %xmm0 # xmm0 = mem[0],zero,zero,zero - LONG $0xc059fac5 // vmulss %xmm0, %xmm0, %xmm0 - LONG $0x587ac1c4; BYTE $0x00 // vaddss (%r8), %xmm0, %xmm0 - LONG $0x117ac1c4; BYTE $0x00 // vmovss %xmm0, (%r8) - LONG $0x4610fac5; BYTE $0x14 // vmovss 20(%rsi), %xmm0 # xmm0 = mem[0],zero,zero,zero - LONG $0xc059fac5 // vmulss %xmm0, %xmm0, %xmm0 - LONG $0x587ac1c4; BYTE $0x01 // vaddss (%r9), %xmm0, %xmm0 - LONG $0x117ac1c4; BYTE $0x01 // vmovss %xmm0, (%r9) - LONG $0x06f88348 // cmpq $6, %rax - JE LBB2_8 - LONG $0x4710fac5; BYTE $0x18 // vmovss 24(%rdi), %xmm0 # xmm0 = mem[0],zero,zero,zero - LONG $0x4659fac5; BYTE $0x18 // vmulss 24(%rsi), %xmm0, %xmm0 - LONG $0x0158fac5 // vaddss (%rcx), %xmm0, %xmm0 - LONG $0x0111fac5 // vmovss %xmm0, (%rcx) - LONG $0x4710fac5; BYTE $0x18 // vmovss 24(%rdi), %xmm0 # xmm0 = mem[0],zero,zero,zero - LONG $0xc059fac5 // vmulss %xmm0, %xmm0, %xmm0 - LONG $0x587ac1c4; BYTE $0x00 // vaddss (%r8), %xmm0, %xmm0 - LONG $0x117ac1c4; BYTE $0x00 // vmovss %xmm0, (%r8) - LONG $0x4610fac5; BYTE $0x18 // vmovss 24(%rsi), %xmm0 # xmm0 = mem[0],zero,zero,zero - LONG $0xc059fac5 // vmulss %xmm0, %xmm0, %xmm0 - LONG $0x587ac1c4; BYTE $0x01 // vaddss (%r9), %xmm0, %xmm0 - LONG $0x117ac1c4; BYTE $0x01 // vmovss %xmm0, (%r9) - -LBB2_8: - LONG $0xf8658d48 // leaq -8(%rbp), %rsp - BYTE $0x5b // popq %rbx - BYTE $0x5d // popq %rbp - WORD $0xf8c5; BYTE $0x77 // vzeroupper - RET diff --git a/internal/floats/floats_avx512.go b/internal/floats/floats_avx512.go deleted file mode 100644 index 85d7fe6..0000000 --- a/internal/floats/floats_avx512.go +++ /dev/null @@ -1,20 +0,0 @@ -//go:build !noasm && amd64 -// Code generated by GoAT. DO NOT EDIT. -// versions: -// clang 21.1.8 (6ubuntu1) -// objdump 2.46 -// flags: -mavx -mfma -mavx512f -O3 -// source: src/floats_avx512.c - -package floats - -import "unsafe" - -//go:noescape -func xvec_avx512_l2_squared(left, right unsafe.Pointer, size int64, output unsafe.Pointer) - -//go:noescape -func xvec_avx512_inner_product(left, right unsafe.Pointer, size int64, output unsafe.Pointer) - -//go:noescape -func xvec_avx512_dot_norms(left, right unsafe.Pointer, size int64, dot, left_norm, right_norm unsafe.Pointer) diff --git a/internal/floats/floats_avx512.s b/internal/floats/floats_avx512.s deleted file mode 100644 index aff450b..0000000 --- a/internal/floats/floats_avx512.s +++ /dev/null @@ -1,528 +0,0 @@ -//go:build !noasm && amd64 -// Code generated by GoAT. DO NOT EDIT. -// versions: -// clang 21.1.8 (6ubuntu1) -// objdump 2.46 -// flags: -mavx -mfma -mavx512f -O3 -// source: src/floats_avx512.c - -TEXT ·xvec_avx512_l2_squared(SB), $0-32 - MOVQ left+0(FP), DI - MOVQ right+8(FP), SI - MOVQ size+16(FP), DX - MOVQ output+24(FP), CX - BYTE $0x55 // pushq %rbp - WORD $0x8948; BYTE $0xe5 // movq %rsp, %rbp - LONG $0xf8e48348 // andq $-8, %rsp - LONG $0x0f428d4c // leaq 15(%rdx), %r8 - WORD $0x8548; BYTE $0xd2 // testq %rdx, %rdx - LONG $0xc2490f4c // cmovnsq %rdx, %r8 - WORD $0x894d; BYTE $0xc1 // movq %r8, %r9 - LONG $0xf0e18349 // andq $-16, %r9 - WORD $0x8948; BYTE $0xd0 // movq %rdx, %rax - WORD $0x294c; BYTE $0xc8 // subq %r9, %rax - LONG $0xc057f8c5 // vxorps %xmm0, %xmm0, %xmm0 - LONG $0x10fa8348 // cmpq $16, %rdx - JL LBB0_7 - LONG $0x04f8c149 // sarq $4, %r8 - LONG $0xff488d4d // leaq -1(%r8), %r9 - WORD $0x8944; BYTE $0xc2 // movl %r8d, %edx - WORD $0xe283; BYTE $0x03 // andl $3, %edx - LONG $0x03f98349 // cmpq $3, %r9 - JAE LBB0_14 - WORD $0x8949; BYTE $0xf9 // movq %rdi, %r9 - WORD $0x8949; BYTE $0xf2 // movq %rsi, %r10 - JMP LBB0_3 - -LBB0_14: - QUAD $0xfffffffffffcbb49; WORD $0x07ff // movabsq $576460752303423484, %r11 # imm = 0x7FFFFFFFFFFFFFC - WORD $0x214d; BYTE $0xc3 // andq %r8, %r11 - WORD $0x8949; BYTE $0xf9 // movq %rdi, %r9 - WORD $0x8949; BYTE $0xf2 // movq %rsi, %r10 - -LBB0_15: - LONG $0x487cd162; WORD $0x0910 // vmovups (%r9), %zmm1 - LONG $0x487cd162; WORD $0x5110; BYTE $0x01 // vmovups 64(%r9), %zmm2 - LONG $0x487cd162; WORD $0x5910; BYTE $0x02 // vmovups 128(%r9), %zmm3 - LONG $0x487cd162; WORD $0x6110; BYTE $0x03 // vmovups 192(%r9), %zmm4 - LONG $0x4874d162; WORD $0x0a5c // vsubps (%r10), %zmm1, %zmm1 - LONG $0x4875f262; WORD $0xc8a8 // vfmadd213ps %zmm0, %zmm1, %zmm1 # zmm1 = (zmm1 * zmm1) + zmm0 - LONG $0x486cd162; WORD $0x425c; BYTE $0x01 // vsubps 64(%r10), %zmm2, %zmm0 - LONG $0x4864d162; WORD $0x525c; BYTE $0x02 // vsubps 128(%r10), %zmm3, %zmm2 - LONG $0x487df262; WORD $0xc1a8 // vfmadd213ps %zmm1, %zmm0, %zmm0 # zmm0 = (zmm0 * zmm0) + zmm1 - LONG $0x486df262; WORD $0xd0a8 // vfmadd213ps %zmm0, %zmm2, %zmm2 # zmm2 = (zmm2 * zmm2) + zmm0 - LONG $0x485cd162; WORD $0x425c; BYTE $0x03 // vsubps 192(%r10), %zmm4, %zmm0 - LONG $0x487df262; WORD $0xc2a8 // vfmadd213ps %zmm2, %zmm0, %zmm0 # zmm0 = (zmm0 * zmm0) + zmm2 - LONG $0x00c18149; WORD $0x0001; BYTE $0x00 // addq $256, %r9 # imm = 0x100 - LONG $0x00c28149; WORD $0x0001; BYTE $0x00 // addq $256, %r10 # imm = 0x100 - LONG $0xfcc38349 // addq $-4, %r11 - JNE LBB0_15 - -LBB0_3: - WORD $0x8548; BYTE $0xd2 // testq %rdx, %rdx - JE LBB0_6 - WORD $0xe2c1; BYTE $0x06 // shll $6, %edx - WORD $0x3145; BYTE $0xdb // xorl %r11d, %r11d - -LBB0_5: - LONG $0x487c9162; WORD $0x0c10; BYTE $0x19 // vmovups (%r9,%r11), %zmm1 - LONG $0x48749162; WORD $0x0c5c; BYTE $0x1a // vsubps (%r10,%r11), %zmm1, %zmm1 - LONG $0x4875f262; WORD $0xc1b8 // vfmadd231ps %zmm1, %zmm1, %zmm0 # zmm0 = (zmm1 * zmm1) + zmm0 - LONG $0x40c38349 // addq $64, %r11 - WORD $0x394c; BYTE $0xda // cmpq %r11, %rdx - JNE LBB0_5 - -LBB0_6: - LONG $0x06e0c149 // shlq $6, %r8 - WORD $0x014c; BYTE $0xc7 // addq %r8, %rdi - WORD $0x014c; BYTE $0xc6 // addq %r8, %rsi - -LBB0_7: - LONG $0xc816fac5 // vmovshdup %xmm0, %xmm1 # xmm1 = xmm0[1,1,3,3] - LONG $0xc958fac5 // vaddss %xmm1, %xmm0, %xmm1 - LONG $0xd0c6f9c5; BYTE $0x01 // vshufpd $1, %xmm0, %xmm0, %xmm2 # xmm2 = xmm0[1,0] - LONG $0xc958eac5 // vaddss %xmm1, %xmm2, %xmm1 - LONG $0xd0c6f8c5; BYTE $0xff // vshufps $255, %xmm0, %xmm0, %xmm2 # xmm2 = xmm0[3,3,3,3] - LONG $0xc958eac5 // vaddss %xmm1, %xmm2, %xmm1 - LONG $0x197de3c4; WORD $0x01c2 // vextractf128 $1, %ymm0, %xmm2 - LONG $0xc958eac5 // vaddss %xmm1, %xmm2, %xmm1 - LONG $0xda16fac5 // vmovshdup %xmm2, %xmm3 # xmm3 = xmm2[1,1,3,3] - LONG $0xc958e2c5 // vaddss %xmm1, %xmm3, %xmm1 - LONG $0xdac6e9c5; BYTE $0x01 // vshufpd $1, %xmm2, %xmm2, %xmm3 # xmm3 = xmm2[1,0] - LONG $0xc958e2c5 // vaddss %xmm1, %xmm3, %xmm1 - LONG $0xd2c6e8c5; BYTE $0xff // vshufps $255, %xmm2, %xmm2, %xmm2 # xmm2 = xmm2[3,3,3,3] - LONG $0xc958eac5 // vaddss %xmm1, %xmm2, %xmm1 - LONG $0x487df362; WORD $0xc219; BYTE $0x02 // vextractf32x4 $2, %zmm0, %xmm2 - LONG $0xc958eac5 // vaddss %xmm1, %xmm2, %xmm1 - LONG $0xda16fac5 // vmovshdup %xmm2, %xmm3 # xmm3 = xmm2[1,1,3,3] - LONG $0xc958e2c5 // vaddss %xmm1, %xmm3, %xmm1 - LONG $0xdac6e9c5; BYTE $0x01 // vshufpd $1, %xmm2, %xmm2, %xmm3 # xmm3 = xmm2[1,0] - LONG $0xc958e2c5 // vaddss %xmm1, %xmm3, %xmm1 - LONG $0xd2c6e8c5; BYTE $0xff // vshufps $255, %xmm2, %xmm2, %xmm2 # xmm2 = xmm2[3,3,3,3] - LONG $0xc958eac5 // vaddss %xmm1, %xmm2, %xmm1 - LONG $0x487df362; WORD $0xc019; BYTE $0x03 // vextractf32x4 $3, %zmm0, %xmm0 - LONG $0xc958fac5 // vaddss %xmm1, %xmm0, %xmm1 - LONG $0xd016fac5 // vmovshdup %xmm0, %xmm2 # xmm2 = xmm0[1,1,3,3] - LONG $0xc958eac5 // vaddss %xmm1, %xmm2, %xmm1 - LONG $0xd0c6f9c5; BYTE $0x01 // vshufpd $1, %xmm0, %xmm0, %xmm2 # xmm2 = xmm0[1,0] - LONG $0xc958eac5 // vaddss %xmm1, %xmm2, %xmm1 - LONG $0xc0c6f8c5; BYTE $0xff // vshufps $255, %xmm0, %xmm0, %xmm0 # xmm0 = xmm0[3,3,3,3] - LONG $0xc158fac5 // vaddss %xmm1, %xmm0, %xmm0 - WORD $0x8548; BYTE $0xc0 // testq %rax, %rax - JLE LBB0_13 - WORD $0xc289 // movl %eax, %edx - WORD $0xe283; BYTE $0x03 // andl $3, %edx - LONG $0x04f88348 // cmpq $4, %rax - JAE LBB0_16 - WORD $0x3145; BYTE $0xc0 // xorl %r8d, %r8d - JMP LBB0_10 - -LBB0_16: - QUAD $0xfffffffffffcb849; WORD $0x7fff // movabsq $9223372036854775804, %r8 # imm = 0x7FFFFFFFFFFFFFFC - WORD $0x214c; BYTE $0xc0 // andq %r8, %rax - WORD $0x3145; BYTE $0xc0 // xorl %r8d, %r8d - -LBB0_17: - LONG $0x107aa1c4; WORD $0x870c // vmovss (%rdi,%r8,4), %xmm1 # xmm1 = mem[0],zero,zero,zero - LONG $0x107aa1c4; WORD $0x8754; BYTE $0x04 // vmovss 4(%rdi,%r8,4), %xmm2 # xmm2 = mem[0],zero,zero,zero - LONG $0x5c72a1c4; WORD $0x860c // vsubss (%rsi,%r8,4), %xmm1, %xmm1 - LONG $0x5c6aa1c4; WORD $0x8654; BYTE $0x04 // vsubss 4(%rsi,%r8,4), %xmm2, %xmm2 - LONG $0xa971e2c4; BYTE $0xc8 // vfmadd213ss %xmm0, %xmm1, %xmm1 # xmm1 = (xmm1 * xmm1) + xmm0 - LONG $0x107aa1c4; WORD $0x8744; BYTE $0x08 // vmovss 8(%rdi,%r8,4), %xmm0 # xmm0 = mem[0],zero,zero,zero - LONG $0x5c7aa1c4; WORD $0x865c; BYTE $0x08 // vsubss 8(%rsi,%r8,4), %xmm0, %xmm3 - LONG $0xa969e2c4; BYTE $0xd1 // vfmadd213ss %xmm1, %xmm2, %xmm2 # xmm2 = (xmm2 * xmm2) + xmm1 - LONG $0x107aa1c4; WORD $0x8744; BYTE $0x0c // vmovss 12(%rdi,%r8,4), %xmm0 # xmm0 = mem[0],zero,zero,zero - LONG $0x5c7aa1c4; WORD $0x8644; BYTE $0x0c // vsubss 12(%rsi,%r8,4), %xmm0, %xmm0 - LONG $0xa961e2c4; BYTE $0xda // vfmadd213ss %xmm2, %xmm3, %xmm3 # xmm3 = (xmm3 * xmm3) + xmm2 - LONG $0xa979e2c4; BYTE $0xc3 // vfmadd213ss %xmm3, %xmm0, %xmm0 # xmm0 = (xmm0 * xmm0) + xmm3 - LONG $0x04c08349 // addq $4, %r8 - WORD $0x394c; BYTE $0xc0 // cmpq %r8, %rax - JNE LBB0_17 - -LBB0_10: - WORD $0x8548; BYTE $0xd2 // testq %rdx, %rdx - JE LBB0_13 - LONG $0x86048d4a // leaq (%rsi,%r8,4), %rax - LONG $0x87348d4a // leaq (%rdi,%r8,4), %rsi - WORD $0xff31 // xorl %edi, %edi - -LBB0_12: - LONG $0x0c10fac5; BYTE $0xbe // vmovss (%rsi,%rdi,4), %xmm1 # xmm1 = mem[0],zero,zero,zero - LONG $0x0c5cf2c5; BYTE $0xb8 // vsubss (%rax,%rdi,4), %xmm1, %xmm1 - LONG $0xb971e2c4; BYTE $0xc1 // vfmadd231ss %xmm1, %xmm1, %xmm0 # xmm0 = (xmm1 * xmm1) + xmm0 - WORD $0xff48; BYTE $0xc7 // incq %rdi - WORD $0x3948; BYTE $0xfa // cmpq %rdi, %rdx - JNE LBB0_12 - -LBB0_13: - LONG $0x0111fac5 // vmovss %xmm0, (%rcx) - WORD $0x8948; BYTE $0xec // movq %rbp, %rsp - BYTE $0x5d // popq %rbp - WORD $0xf8c5; BYTE $0x77 // vzeroupper - RET - -TEXT ·xvec_avx512_inner_product(SB), $0-32 - MOVQ left+0(FP), DI - MOVQ right+8(FP), SI - MOVQ size+16(FP), DX - MOVQ output+24(FP), CX - BYTE $0x55 // pushq %rbp - WORD $0x8948; BYTE $0xe5 // movq %rsp, %rbp - LONG $0xf8e48348 // andq $-8, %rsp - LONG $0x0f428d4c // leaq 15(%rdx), %r8 - WORD $0x8548; BYTE $0xd2 // testq %rdx, %rdx - LONG $0xc2490f4c // cmovnsq %rdx, %r8 - WORD $0x894d; BYTE $0xc1 // movq %r8, %r9 - LONG $0xf0e18349 // andq $-16, %r9 - WORD $0x8948; BYTE $0xd0 // movq %rdx, %rax - WORD $0x294c; BYTE $0xc8 // subq %r9, %rax - LONG $0xc057f8c5 // vxorps %xmm0, %xmm0, %xmm0 - LONG $0x10fa8348 // cmpq $16, %rdx - JL LBB1_7 - LONG $0x04f8c149 // sarq $4, %r8 - LONG $0xff488d4d // leaq -1(%r8), %r9 - WORD $0x8944; BYTE $0xc2 // movl %r8d, %edx - WORD $0xe283; BYTE $0x03 // andl $3, %edx - LONG $0x03f98349 // cmpq $3, %r9 - JAE LBB1_14 - WORD $0x8949; BYTE $0xf9 // movq %rdi, %r9 - WORD $0x8949; BYTE $0xf2 // movq %rsi, %r10 - JMP LBB1_3 - -LBB1_14: - QUAD $0xfffffffffffcbb49; WORD $0x07ff // movabsq $576460752303423484, %r11 # imm = 0x7FFFFFFFFFFFFFC - WORD $0x214d; BYTE $0xc3 // andq %r8, %r11 - LONG $0xc957f0c5 // vxorps %xmm1, %xmm1, %xmm1 - WORD $0x8949; BYTE $0xf9 // movq %rdi, %r9 - WORD $0x8949; BYTE $0xf2 // movq %rsi, %r10 - -LBB1_15: - LONG $0x487cd162; WORD $0x0110 // vmovups (%r9), %zmm0 - LONG $0x487cd162; WORD $0x5110; BYTE $0x01 // vmovups 64(%r9), %zmm2 - LONG $0x487cd162; WORD $0x5910; BYTE $0x02 // vmovups 128(%r9), %zmm3 - LONG $0x487cd162; WORD $0x6110; BYTE $0x03 // vmovups 192(%r9), %zmm4 - LONG $0x4875d262; WORD $0x0298 // vfmadd132ps (%r10), %zmm1, %zmm0 # zmm0 = (zmm0 * mem) + zmm1 - LONG $0x486dd262; WORD $0x42b8; BYTE $0x01 // vfmadd231ps 64(%r10), %zmm2, %zmm0 # zmm0 = (zmm2 * mem) + zmm0 - LONG $0x4865d262; WORD $0x42b8; BYTE $0x02 // vfmadd231ps 128(%r10), %zmm3, %zmm0 # zmm0 = (zmm3 * mem) + zmm0 - LONG $0x485dd262; WORD $0x42b8; BYTE $0x03 // vfmadd231ps 192(%r10), %zmm4, %zmm0 # zmm0 = (zmm4 * mem) + zmm0 - LONG $0x00c18149; WORD $0x0001; BYTE $0x00 // addq $256, %r9 # imm = 0x100 - LONG $0x00c28149; WORD $0x0001; BYTE $0x00 // addq $256, %r10 # imm = 0x100 - LONG $0x487cf162; WORD $0xc828 // vmovaps %zmm0, %zmm1 - LONG $0xfcc38349 // addq $-4, %r11 - JNE LBB1_15 - -LBB1_3: - WORD $0x8548; BYTE $0xd2 // testq %rdx, %rdx - JE LBB1_6 - WORD $0xe2c1; BYTE $0x06 // shll $6, %edx - WORD $0x3145; BYTE $0xdb // xorl %r11d, %r11d - -LBB1_5: - LONG $0x487c9162; WORD $0x0c10; BYTE $0x19 // vmovups (%r9,%r11), %zmm1 - LONG $0x48759262; WORD $0x04b8; BYTE $0x1a // vfmadd231ps (%r10,%r11), %zmm1, %zmm0 # zmm0 = (zmm1 * mem) + zmm0 - LONG $0x40c38349 // addq $64, %r11 - WORD $0x394c; BYTE $0xda // cmpq %r11, %rdx - JNE LBB1_5 - -LBB1_6: - LONG $0x06e0c149 // shlq $6, %r8 - WORD $0x014c; BYTE $0xc7 // addq %r8, %rdi - WORD $0x014c; BYTE $0xc6 // addq %r8, %rsi - -LBB1_7: - LONG $0xc957f0c5 // vxorps %xmm1, %xmm1, %xmm1 - LONG $0xc958fac5 // vaddss %xmm1, %xmm0, %xmm1 - LONG $0xd016fac5 // vmovshdup %xmm0, %xmm2 # xmm2 = xmm0[1,1,3,3] - LONG $0xc958eac5 // vaddss %xmm1, %xmm2, %xmm1 - LONG $0xd0c6f9c5; BYTE $0x01 // vshufpd $1, %xmm0, %xmm0, %xmm2 # xmm2 = xmm0[1,0] - LONG $0xc958eac5 // vaddss %xmm1, %xmm2, %xmm1 - LONG $0xd0c6f8c5; BYTE $0xff // vshufps $255, %xmm0, %xmm0, %xmm2 # xmm2 = xmm0[3,3,3,3] - LONG $0xc958eac5 // vaddss %xmm1, %xmm2, %xmm1 - LONG $0x197de3c4; WORD $0x01c2 // vextractf128 $1, %ymm0, %xmm2 - LONG $0xc958eac5 // vaddss %xmm1, %xmm2, %xmm1 - LONG $0xda16fac5 // vmovshdup %xmm2, %xmm3 # xmm3 = xmm2[1,1,3,3] - LONG $0xc958e2c5 // vaddss %xmm1, %xmm3, %xmm1 - LONG $0xdac6e9c5; BYTE $0x01 // vshufpd $1, %xmm2, %xmm2, %xmm3 # xmm3 = xmm2[1,0] - LONG $0xc958e2c5 // vaddss %xmm1, %xmm3, %xmm1 - LONG $0xd2c6e8c5; BYTE $0xff // vshufps $255, %xmm2, %xmm2, %xmm2 # xmm2 = xmm2[3,3,3,3] - LONG $0xc958eac5 // vaddss %xmm1, %xmm2, %xmm1 - LONG $0x487df362; WORD $0xc219; BYTE $0x02 // vextractf32x4 $2, %zmm0, %xmm2 - LONG $0xc958eac5 // vaddss %xmm1, %xmm2, %xmm1 - LONG $0xda16fac5 // vmovshdup %xmm2, %xmm3 # xmm3 = xmm2[1,1,3,3] - LONG $0xc958e2c5 // vaddss %xmm1, %xmm3, %xmm1 - LONG $0xdac6e9c5; BYTE $0x01 // vshufpd $1, %xmm2, %xmm2, %xmm3 # xmm3 = xmm2[1,0] - LONG $0xc958e2c5 // vaddss %xmm1, %xmm3, %xmm1 - LONG $0xd2c6e8c5; BYTE $0xff // vshufps $255, %xmm2, %xmm2, %xmm2 # xmm2 = xmm2[3,3,3,3] - LONG $0xc958eac5 // vaddss %xmm1, %xmm2, %xmm1 - LONG $0x487df362; WORD $0xc019; BYTE $0x03 // vextractf32x4 $3, %zmm0, %xmm0 - LONG $0xc958fac5 // vaddss %xmm1, %xmm0, %xmm1 - LONG $0xd016fac5 // vmovshdup %xmm0, %xmm2 # xmm2 = xmm0[1,1,3,3] - LONG $0xc958eac5 // vaddss %xmm1, %xmm2, %xmm1 - LONG $0xd0c6f9c5; BYTE $0x01 // vshufpd $1, %xmm0, %xmm0, %xmm2 # xmm2 = xmm0[1,0] - LONG $0xc958eac5 // vaddss %xmm1, %xmm2, %xmm1 - LONG $0xc0c6f8c5; BYTE $0xff // vshufps $255, %xmm0, %xmm0, %xmm0 # xmm0 = xmm0[3,3,3,3] - LONG $0xc158fac5 // vaddss %xmm1, %xmm0, %xmm0 - WORD $0x8548; BYTE $0xc0 // testq %rax, %rax - JLE LBB1_13 - WORD $0xc289 // movl %eax, %edx - WORD $0xe283; BYTE $0x03 // andl $3, %edx - LONG $0x04f88348 // cmpq $4, %rax - JAE LBB1_16 - WORD $0x3145; BYTE $0xc0 // xorl %r8d, %r8d - JMP LBB1_10 - -LBB1_16: - QUAD $0xfffffffffffcb849; WORD $0x7fff // movabsq $9223372036854775804, %r8 # imm = 0x7FFFFFFFFFFFFFFC - WORD $0x214c; BYTE $0xc0 // andq %r8, %rax - WORD $0x3145; BYTE $0xc0 // xorl %r8d, %r8d - -LBB1_17: - LONG $0x107aa1c4; WORD $0x870c // vmovss (%rdi,%r8,4), %xmm1 # xmm1 = mem[0],zero,zero,zero - LONG $0x107aa1c4; WORD $0x8754; BYTE $0x04 // vmovss 4(%rdi,%r8,4), %xmm2 # xmm2 = mem[0],zero,zero,zero - LONG $0x9979a2c4; WORD $0x860c // vfmadd132ss (%rsi,%r8,4), %xmm0, %xmm1 # xmm1 = (xmm1 * mem) + xmm0 - LONG $0xb969a2c4; WORD $0x864c; BYTE $0x04 // vfmadd231ss 4(%rsi,%r8,4), %xmm2, %xmm1 # xmm1 = (xmm2 * mem) + xmm1 - LONG $0x107aa1c4; WORD $0x8754; BYTE $0x08 // vmovss 8(%rdi,%r8,4), %xmm2 # xmm2 = mem[0],zero,zero,zero - LONG $0x9971a2c4; WORD $0x8654; BYTE $0x08 // vfmadd132ss 8(%rsi,%r8,4), %xmm1, %xmm2 # xmm2 = (xmm2 * mem) + xmm1 - LONG $0x107aa1c4; WORD $0x8744; BYTE $0x0c // vmovss 12(%rdi,%r8,4), %xmm0 # xmm0 = mem[0],zero,zero,zero - LONG $0x9969a2c4; WORD $0x8644; BYTE $0x0c // vfmadd132ss 12(%rsi,%r8,4), %xmm2, %xmm0 # xmm0 = (xmm0 * mem) + xmm2 - LONG $0x04c08349 // addq $4, %r8 - WORD $0x394c; BYTE $0xc0 // cmpq %r8, %rax - JNE LBB1_17 - -LBB1_10: - WORD $0x8548; BYTE $0xd2 // testq %rdx, %rdx - JE LBB1_13 - LONG $0x86048d4a // leaq (%rsi,%r8,4), %rax - LONG $0x87348d4a // leaq (%rdi,%r8,4), %rsi - WORD $0xff31 // xorl %edi, %edi - -LBB1_12: - LONG $0x0c10fac5; BYTE $0xbe // vmovss (%rsi,%rdi,4), %xmm1 # xmm1 = mem[0],zero,zero,zero - LONG $0xb971e2c4; WORD $0xb804 // vfmadd231ss (%rax,%rdi,4), %xmm1, %xmm0 # xmm0 = (xmm1 * mem) + xmm0 - WORD $0xff48; BYTE $0xc7 // incq %rdi - WORD $0x3948; BYTE $0xfa // cmpq %rdi, %rdx - JNE LBB1_12 - -LBB1_13: - LONG $0x0111fac5 // vmovss %xmm0, (%rcx) - WORD $0x8948; BYTE $0xec // movq %rbp, %rsp - BYTE $0x5d // popq %rbp - WORD $0xf8c5; BYTE $0x77 // vzeroupper - RET - -TEXT ·xvec_avx512_dot_norms(SB), $0-48 - MOVQ left+0(FP), DI - MOVQ right+8(FP), SI - MOVQ size+16(FP), DX - MOVQ dot+24(FP), CX - MOVQ left_norm+32(FP), R8 - MOVQ right_norm+40(FP), R9 - BYTE $0x55 // pushq %rbp - WORD $0x8948; BYTE $0xe5 // movq %rsp, %rbp - WORD $0x5641 // pushq %r14 - BYTE $0x53 // pushq %rbx - LONG $0xf8e48348 // andq $-8, %rsp - LONG $0x0f528d4c // leaq 15(%rdx), %r10 - WORD $0x8548; BYTE $0xd2 // testq %rdx, %rdx - LONG $0xd2490f4c // cmovnsq %rdx, %r10 - WORD $0x894d; BYTE $0xd3 // movq %r10, %r11 - LONG $0xf0e38349 // andq $-16, %r11 - WORD $0x8948; BYTE $0xd0 // movq %rdx, %rax - WORD $0x294c; BYTE $0xd8 // subq %r11, %rax - LONG $0xd257e8c5 // vxorps %xmm2, %xmm2, %xmm2 - LONG $0x10fa8348 // cmpq $16, %rdx - JL LBB2_1 - LONG $0x04fac149 // sarq $4, %r10 - LONG $0xff5a8d4d // leaq -1(%r10), %r11 - WORD $0x8944; BYTE $0xd2 // movl %r10d, %edx - WORD $0xe283; BYTE $0x03 // andl $3, %edx - LONG $0x03fb8349 // cmpq $3, %r11 - JAE LBB2_12 - LONG $0xc057f8c5 // vxorps %xmm0, %xmm0, %xmm0 - WORD $0x8949; BYTE $0xfb // movq %rdi, %r11 - WORD $0x8948; BYTE $0xf3 // movq %rsi, %rbx - LONG $0xc957f0c5 // vxorps %xmm1, %xmm1, %xmm1 - JMP LBB2_4 - -LBB2_1: - LONG $0xc957f0c5 // vxorps %xmm1, %xmm1, %xmm1 - LONG $0xc057f8c5 // vxorps %xmm0, %xmm0, %xmm0 - JMP LBB2_8 - -LBB2_12: - QUAD $0xfffffffffffcbe49; WORD $0x07ff // movabsq $576460752303423484, %r14 # imm = 0x7FFFFFFFFFFFFFC - WORD $0x214d; BYTE $0xd6 // andq %r10, %r14 - LONG $0xc057f8c5 // vxorps %xmm0, %xmm0, %xmm0 - WORD $0x8949; BYTE $0xfb // movq %rdi, %r11 - WORD $0x8948; BYTE $0xf3 // movq %rsi, %rbx - LONG $0xc957f0c5 // vxorps %xmm1, %xmm1, %xmm1 - -LBB2_13: - LONG $0x487cd162; WORD $0x1b10 // vmovups (%r11), %zmm3 - LONG $0x487cd162; WORD $0x6310; BYTE $0x01 // vmovups 64(%r11), %zmm4 - LONG $0x487cd162; WORD $0x6b10; BYTE $0x02 // vmovups 128(%r11), %zmm5 - LONG $0x487cd162; WORD $0x7310; BYTE $0x03 // vmovups 192(%r11), %zmm6 - LONG $0x487cf162; WORD $0x3b10 // vmovups (%rbx), %zmm7 - LONG $0x487c7162; WORD $0x4310; BYTE $0x01 // vmovups 64(%rbx), %zmm8 - LONG $0x487c7162; WORD $0x4b10; BYTE $0x02 // vmovups 128(%rbx), %zmm9 - LONG $0x487c7162; WORD $0x5310; BYTE $0x03 // vmovups 192(%rbx), %zmm10 - LONG $0x4865f262; WORD $0xd7b8 // vfmadd231ps %zmm7, %zmm3, %zmm2 # zmm2 = (zmm3 * zmm7) + zmm2 - LONG $0x4865f262; WORD $0xcbb8 // vfmadd231ps %zmm3, %zmm3, %zmm1 # zmm1 = (zmm3 * zmm3) + zmm1 - LONG $0x4845f262; WORD $0xc7b8 // vfmadd231ps %zmm7, %zmm7, %zmm0 # zmm0 = (zmm7 * zmm7) + zmm0 - LONG $0x485dd262; WORD $0xd0b8 // vfmadd231ps %zmm8, %zmm4, %zmm2 # zmm2 = (zmm4 * zmm8) + zmm2 - LONG $0x485df262; WORD $0xccb8 // vfmadd231ps %zmm4, %zmm4, %zmm1 # zmm1 = (zmm4 * zmm4) + zmm1 - LONG $0x483dd262; WORD $0xc0b8 // vfmadd231ps %zmm8, %zmm8, %zmm0 # zmm0 = (zmm8 * zmm8) + zmm0 - LONG $0x4855d262; WORD $0xd1b8 // vfmadd231ps %zmm9, %zmm5, %zmm2 # zmm2 = (zmm5 * zmm9) + zmm2 - LONG $0x4855f262; WORD $0xcdb8 // vfmadd231ps %zmm5, %zmm5, %zmm1 # zmm1 = (zmm5 * zmm5) + zmm1 - LONG $0x4835d262; WORD $0xc1b8 // vfmadd231ps %zmm9, %zmm9, %zmm0 # zmm0 = (zmm9 * zmm9) + zmm0 - LONG $0x484dd262; WORD $0xd2b8 // vfmadd231ps %zmm10, %zmm6, %zmm2 # zmm2 = (zmm6 * zmm10) + zmm2 - LONG $0x484df262; WORD $0xceb8 // vfmadd231ps %zmm6, %zmm6, %zmm1 # zmm1 = (zmm6 * zmm6) + zmm1 - LONG $0x482dd262; WORD $0xc2b8 // vfmadd231ps %zmm10, %zmm10, %zmm0 # zmm0 = (zmm10 * zmm10) + zmm0 - LONG $0x00c38149; WORD $0x0001; BYTE $0x00 // addq $256, %r11 # imm = 0x100 - LONG $0x00c38148; WORD $0x0001; BYTE $0x00 // addq $256, %rbx # imm = 0x100 - LONG $0xfcc68349 // addq $-4, %r14 - JNE LBB2_13 - -LBB2_4: - WORD $0x8548; BYTE $0xd2 // testq %rdx, %rdx - JE LBB2_7 - WORD $0xe2c1; BYTE $0x06 // shll $6, %edx - WORD $0x3145; BYTE $0xf6 // xorl %r14d, %r14d - -LBB2_6: - LONG $0x487c9162; WORD $0x1c10; BYTE $0x33 // vmovups (%r11,%r14), %zmm3 - LONG $0x487cb162; WORD $0x2410; BYTE $0x33 // vmovups (%rbx,%r14), %zmm4 - LONG $0x4865f262; WORD $0xd4b8 // vfmadd231ps %zmm4, %zmm3, %zmm2 # zmm2 = (zmm3 * zmm4) + zmm2 - LONG $0x4865f262; WORD $0xcbb8 // vfmadd231ps %zmm3, %zmm3, %zmm1 # zmm1 = (zmm3 * zmm3) + zmm1 - LONG $0x485df262; WORD $0xc4b8 // vfmadd231ps %zmm4, %zmm4, %zmm0 # zmm0 = (zmm4 * zmm4) + zmm0 - LONG $0x40c68349 // addq $64, %r14 - WORD $0x394c; BYTE $0xf2 // cmpq %r14, %rdx - JNE LBB2_6 - -LBB2_7: - LONG $0x06e2c149 // shlq $6, %r10 - WORD $0x014c; BYTE $0xd7 // addq %r10, %rdi - WORD $0x014c; BYTE $0xd6 // addq %r10, %rsi - -LBB2_8: - LONG $0xdb57e0c5 // vxorps %xmm3, %xmm3, %xmm3 - LONG $0xdb58eac5 // vaddss %xmm3, %xmm2, %xmm3 - LONG $0xe216fac5 // vmovshdup %xmm2, %xmm4 # xmm4 = xmm2[1,1,3,3] - LONG $0xdb58dac5 // vaddss %xmm3, %xmm4, %xmm3 - LONG $0xe2c6e9c5; BYTE $0x01 // vshufpd $1, %xmm2, %xmm2, %xmm4 # xmm4 = xmm2[1,0] - LONG $0xdb58dac5 // vaddss %xmm3, %xmm4, %xmm3 - LONG $0xe2c6e8c5; BYTE $0xff // vshufps $255, %xmm2, %xmm2, %xmm4 # xmm4 = xmm2[3,3,3,3] - LONG $0xdb58dac5 // vaddss %xmm3, %xmm4, %xmm3 - LONG $0x197de3c4; WORD $0x01d4 // vextractf128 $1, %ymm2, %xmm4 - LONG $0xdb58dac5 // vaddss %xmm3, %xmm4, %xmm3 - LONG $0xec16fac5 // vmovshdup %xmm4, %xmm5 # xmm5 = xmm4[1,1,3,3] - LONG $0xdb58d2c5 // vaddss %xmm3, %xmm5, %xmm3 - LONG $0xecc6d9c5; BYTE $0x01 // vshufpd $1, %xmm4, %xmm4, %xmm5 # xmm5 = xmm4[1,0] - LONG $0xdb58d2c5 // vaddss %xmm3, %xmm5, %xmm3 - LONG $0xe4c6d8c5; BYTE $0xff // vshufps $255, %xmm4, %xmm4, %xmm4 # xmm4 = xmm4[3,3,3,3] - LONG $0xdb58dac5 // vaddss %xmm3, %xmm4, %xmm3 - LONG $0x487df362; WORD $0xd419; BYTE $0x02 // vextractf32x4 $2, %zmm2, %xmm4 - LONG $0xdb58dac5 // vaddss %xmm3, %xmm4, %xmm3 - LONG $0xec16fac5 // vmovshdup %xmm4, %xmm5 # xmm5 = xmm4[1,1,3,3] - LONG $0xdb58d2c5 // vaddss %xmm3, %xmm5, %xmm3 - LONG $0xecc6d9c5; BYTE $0x01 // vshufpd $1, %xmm4, %xmm4, %xmm5 # xmm5 = xmm4[1,0] - LONG $0xdb58d2c5 // vaddss %xmm3, %xmm5, %xmm3 - LONG $0xe4c6d8c5; BYTE $0xff // vshufps $255, %xmm4, %xmm4, %xmm4 # xmm4 = xmm4[3,3,3,3] - LONG $0xdb58dac5 // vaddss %xmm3, %xmm4, %xmm3 - LONG $0x487df362; WORD $0xd219; BYTE $0x03 // vextractf32x4 $3, %zmm2, %xmm2 - LONG $0xdb58eac5 // vaddss %xmm3, %xmm2, %xmm3 - LONG $0xe216fac5 // vmovshdup %xmm2, %xmm4 # xmm4 = xmm2[1,1,3,3] - LONG $0xdb58dac5 // vaddss %xmm3, %xmm4, %xmm3 - LONG $0xe2c6e9c5; BYTE $0x01 // vshufpd $1, %xmm2, %xmm2, %xmm4 # xmm4 = xmm2[1,0] - LONG $0xdb58dac5 // vaddss %xmm3, %xmm4, %xmm3 - LONG $0xd2c6e8c5; BYTE $0xff // vshufps $255, %xmm2, %xmm2, %xmm2 # xmm2 = xmm2[3,3,3,3] - LONG $0xd358eac5 // vaddss %xmm3, %xmm2, %xmm2 - LONG $0x1111fac5 // vmovss %xmm2, (%rcx) - LONG $0xd116fac5 // vmovshdup %xmm1, %xmm2 # xmm2 = xmm1[1,1,3,3] - LONG $0xd258f2c5 // vaddss %xmm2, %xmm1, %xmm2 - LONG $0xd9c6f1c5; BYTE $0x01 // vshufpd $1, %xmm1, %xmm1, %xmm3 # xmm3 = xmm1[1,0] - LONG $0xd258e2c5 // vaddss %xmm2, %xmm3, %xmm2 - LONG $0xd9c6f0c5; BYTE $0xff // vshufps $255, %xmm1, %xmm1, %xmm3 # xmm3 = xmm1[3,3,3,3] - LONG $0xd258e2c5 // vaddss %xmm2, %xmm3, %xmm2 - LONG $0x197de3c4; WORD $0x01cb // vextractf128 $1, %ymm1, %xmm3 - LONG $0xd258e2c5 // vaddss %xmm2, %xmm3, %xmm2 - LONG $0xe316fac5 // vmovshdup %xmm3, %xmm4 # xmm4 = xmm3[1,1,3,3] - LONG $0xd258dac5 // vaddss %xmm2, %xmm4, %xmm2 - LONG $0xe3c6e1c5; BYTE $0x01 // vshufpd $1, %xmm3, %xmm3, %xmm4 # xmm4 = xmm3[1,0] - LONG $0xd258dac5 // vaddss %xmm2, %xmm4, %xmm2 - LONG $0xdbc6e0c5; BYTE $0xff // vshufps $255, %xmm3, %xmm3, %xmm3 # xmm3 = xmm3[3,3,3,3] - LONG $0xd258e2c5 // vaddss %xmm2, %xmm3, %xmm2 - LONG $0x487df362; WORD $0xcb19; BYTE $0x02 // vextractf32x4 $2, %zmm1, %xmm3 - LONG $0xd258e2c5 // vaddss %xmm2, %xmm3, %xmm2 - LONG $0xe316fac5 // vmovshdup %xmm3, %xmm4 # xmm4 = xmm3[1,1,3,3] - LONG $0xd258dac5 // vaddss %xmm2, %xmm4, %xmm2 - LONG $0xe3c6e1c5; BYTE $0x01 // vshufpd $1, %xmm3, %xmm3, %xmm4 # xmm4 = xmm3[1,0] - LONG $0xd258dac5 // vaddss %xmm2, %xmm4, %xmm2 - LONG $0xdbc6e0c5; BYTE $0xff // vshufps $255, %xmm3, %xmm3, %xmm3 # xmm3 = xmm3[3,3,3,3] - LONG $0xd258e2c5 // vaddss %xmm2, %xmm3, %xmm2 - LONG $0x487df362; WORD $0xc919; BYTE $0x03 // vextractf32x4 $3, %zmm1, %xmm1 - LONG $0xd258f2c5 // vaddss %xmm2, %xmm1, %xmm2 - LONG $0xd916fac5 // vmovshdup %xmm1, %xmm3 # xmm3 = xmm1[1,1,3,3] - LONG $0xd258e2c5 // vaddss %xmm2, %xmm3, %xmm2 - LONG $0xd9c6f1c5; BYTE $0x01 // vshufpd $1, %xmm1, %xmm1, %xmm3 # xmm3 = xmm1[1,0] - LONG $0xd258e2c5 // vaddss %xmm2, %xmm3, %xmm2 - LONG $0xc9c6f0c5; BYTE $0xff // vshufps $255, %xmm1, %xmm1, %xmm1 # xmm1 = xmm1[3,3,3,3] - LONG $0xca58f2c5 // vaddss %xmm2, %xmm1, %xmm1 - LONG $0x117ac1c4; BYTE $0x08 // vmovss %xmm1, (%r8) - LONG $0xc816fac5 // vmovshdup %xmm0, %xmm1 # xmm1 = xmm0[1,1,3,3] - LONG $0xc958fac5 // vaddss %xmm1, %xmm0, %xmm1 - LONG $0xd0c6f9c5; BYTE $0x01 // vshufpd $1, %xmm0, %xmm0, %xmm2 # xmm2 = xmm0[1,0] - LONG $0xc958eac5 // vaddss %xmm1, %xmm2, %xmm1 - LONG $0xd0c6f8c5; BYTE $0xff // vshufps $255, %xmm0, %xmm0, %xmm2 # xmm2 = xmm0[3,3,3,3] - LONG $0xc958eac5 // vaddss %xmm1, %xmm2, %xmm1 - LONG $0x197de3c4; WORD $0x01c2 // vextractf128 $1, %ymm0, %xmm2 - LONG $0xc958eac5 // vaddss %xmm1, %xmm2, %xmm1 - LONG $0xda16fac5 // vmovshdup %xmm2, %xmm3 # xmm3 = xmm2[1,1,3,3] - LONG $0xc958e2c5 // vaddss %xmm1, %xmm3, %xmm1 - LONG $0xdac6e9c5; BYTE $0x01 // vshufpd $1, %xmm2, %xmm2, %xmm3 # xmm3 = xmm2[1,0] - LONG $0xc958e2c5 // vaddss %xmm1, %xmm3, %xmm1 - LONG $0xd2c6e8c5; BYTE $0xff // vshufps $255, %xmm2, %xmm2, %xmm2 # xmm2 = xmm2[3,3,3,3] - LONG $0xc958eac5 // vaddss %xmm1, %xmm2, %xmm1 - LONG $0x487df362; WORD $0xc219; BYTE $0x02 // vextractf32x4 $2, %zmm0, %xmm2 - LONG $0xc958eac5 // vaddss %xmm1, %xmm2, %xmm1 - LONG $0xda16fac5 // vmovshdup %xmm2, %xmm3 # xmm3 = xmm2[1,1,3,3] - LONG $0xc958e2c5 // vaddss %xmm1, %xmm3, %xmm1 - LONG $0xdac6e9c5; BYTE $0x01 // vshufpd $1, %xmm2, %xmm2, %xmm3 # xmm3 = xmm2[1,0] - LONG $0xc958e2c5 // vaddss %xmm1, %xmm3, %xmm1 - LONG $0xd2c6e8c5; BYTE $0xff // vshufps $255, %xmm2, %xmm2, %xmm2 # xmm2 = xmm2[3,3,3,3] - LONG $0xc958eac5 // vaddss %xmm1, %xmm2, %xmm1 - LONG $0x487df362; WORD $0xc019; BYTE $0x03 // vextractf32x4 $3, %zmm0, %xmm0 - LONG $0xc958fac5 // vaddss %xmm1, %xmm0, %xmm1 - LONG $0xd016fac5 // vmovshdup %xmm0, %xmm2 # xmm2 = xmm0[1,1,3,3] - LONG $0xc958eac5 // vaddss %xmm1, %xmm2, %xmm1 - LONG $0xd0c6f9c5; BYTE $0x01 // vshufpd $1, %xmm0, %xmm0, %xmm2 # xmm2 = xmm0[1,0] - LONG $0xc958eac5 // vaddss %xmm1, %xmm2, %xmm1 - LONG $0xc0c6f8c5; BYTE $0xff // vshufps $255, %xmm0, %xmm0, %xmm0 # xmm0 = xmm0[3,3,3,3] - LONG $0xc158fac5 // vaddss %xmm1, %xmm0, %xmm0 - LONG $0x117ac1c4; BYTE $0x01 // vmovss %xmm0, (%r9) - WORD $0x8548; BYTE $0xc0 // testq %rax, %rax - JLE LBB2_11 - WORD $0xd231 // xorl %edx, %edx - -LBB2_10: - LONG $0x0410fac5; BYTE $0x97 // vmovss (%rdi,%rdx,4), %xmm0 # xmm0 = mem[0],zero,zero,zero - LONG $0x0c10fac5; BYTE $0x96 // vmovss (%rsi,%rdx,4), %xmm1 # xmm1 = mem[0],zero,zero,zero - LONG $0xa979e2c4; BYTE $0x09 // vfmadd213ss (%rcx), %xmm0, %xmm1 # xmm1 = (xmm0 * xmm1) + mem - LONG $0x0911fac5 // vmovss %xmm1, (%rcx) - LONG $0x0410fac5; BYTE $0x97 // vmovss (%rdi,%rdx,4), %xmm0 # xmm0 = mem[0],zero,zero,zero - LONG $0xa979c2c4; BYTE $0x00 // vfmadd213ss (%r8), %xmm0, %xmm0 # xmm0 = (xmm0 * xmm0) + mem - LONG $0x117ac1c4; BYTE $0x00 // vmovss %xmm0, (%r8) - LONG $0x0410fac5; BYTE $0x96 // vmovss (%rsi,%rdx,4), %xmm0 # xmm0 = mem[0],zero,zero,zero - LONG $0xa979c2c4; BYTE $0x01 // vfmadd213ss (%r9), %xmm0, %xmm0 # xmm0 = (xmm0 * xmm0) + mem - LONG $0x117ac1c4; BYTE $0x01 // vmovss %xmm0, (%r9) - WORD $0xff48; BYTE $0xc2 // incq %rdx - WORD $0x3948; BYTE $0xd0 // cmpq %rdx, %rax - JNE LBB2_10 - -LBB2_11: - LONG $0xf0658d48 // leaq -16(%rbp), %rsp - BYTE $0x5b // popq %rbx - WORD $0x5e41 // popq %r14 - BYTE $0x5d // popq %rbp - WORD $0xf8c5; BYTE $0x77 // vzeroupper - RET diff --git a/internal/floats/floats_batch_avx.go b/internal/floats/floats_batch_avx.go deleted file mode 100644 index 226fe24..0000000 --- a/internal/floats/floats_batch_avx.go +++ /dev/null @@ -1,25 +0,0 @@ -//go:build !noasm && amd64 - -// Copyright 2026-present the xvec project -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package floats - -import "unsafe" - -//go:noescape -func xvec_avx_batch_inner_products2(query, first, second unsafe.Pointer, size int64, firstOutput, secondOutput unsafe.Pointer) - -//go:noescape -func xvec_avx_batch_inner_products4(query, first, second, third, fourth unsafe.Pointer, size int64, firstOutput, secondOutput, thirdOutput, fourthOutput unsafe.Pointer) diff --git a/internal/floats/floats_batch_avx.s b/internal/floats/floats_batch_avx.s deleted file mode 100644 index bce04aa..0000000 --- a/internal/floats/floats_batch_avx.s +++ /dev/null @@ -1,199 +0,0 @@ -//go:build !noasm && amd64 - -// Copyright 2026-present the xvec project -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "textflag.h" - -// xvec_avx_batch_inner_products2 computes two query/candidate inner products -// in one pass, loading every query cache line once for both candidates. -TEXT ·xvec_avx_batch_inner_products2(SB), NOSPLIT, $64-48 - MOVQ query+0(FP), AX - MOVQ first+8(FP), BX - MOVQ second+16(FP), CX - MOVQ size+24(FP), DX - MOVQ firstOutput+32(FP), SI - MOVQ secondOutput+40(FP), DI - - VXORPS Y0, Y0, Y0 - VXORPS Y1, Y1, Y1 - MOVQ DX, R8 - SHRQ $3, R8 - JE reduce - -vector_loop: - VMOVUPS (AX), Y2 - VMULPS (BX), Y2, Y3 - VMULPS (CX), Y2, Y4 - VADDPS Y3, Y0, Y0 - VADDPS Y4, Y1, Y1 - ADDQ $32, AX - ADDQ $32, BX - ADDQ $32, CX - DECQ R8 - JNE vector_loop - -reduce: - VMOVUPS Y0, 0(SP) - VMOVUPS Y1, 32(SP) - VXORPS X2, X2, X2 - VXORPS X3, X3, X3 - VADDSS 0(SP), X2, X2 - VADDSS 4(SP), X2, X2 - VADDSS 8(SP), X2, X2 - VADDSS 12(SP), X2, X2 - VADDSS 16(SP), X2, X2 - VADDSS 20(SP), X2, X2 - VADDSS 24(SP), X2, X2 - VADDSS 28(SP), X2, X2 - VADDSS 32(SP), X3, X3 - VADDSS 36(SP), X3, X3 - VADDSS 40(SP), X3, X3 - VADDSS 44(SP), X3, X3 - VADDSS 48(SP), X3, X3 - VADDSS 52(SP), X3, X3 - VADDSS 56(SP), X3, X3 - VADDSS 60(SP), X3, X3 - - ANDQ $7, DX - JE done - -tail_loop: - VMOVSS (AX), X4 - VMULSS (BX), X4, X5 - VMULSS (CX), X4, X6 - VADDSS X5, X2, X2 - VADDSS X6, X3, X3 - ADDQ $4, AX - ADDQ $4, BX - ADDQ $4, CX - DECQ DX - JNE tail_loop - -done: - VMOVSS X2, (SI) - VMOVSS X3, (DI) - VZEROUPPER - RET - -// xvec_avx_batch_inner_products4 computes four query/candidate inner products -// in one pass, loading every query cache line once for all four candidates. -TEXT ·xvec_avx_batch_inner_products4(SB), NOSPLIT, $128-80 - MOVQ query+0(FP), AX - MOVQ first+8(FP), BX - MOVQ second+16(FP), CX - MOVQ third+24(FP), DX - MOVQ fourth+32(FP), SI - MOVQ size+40(FP), R8 - MOVQ firstOutput+48(FP), R9 - MOVQ secondOutput+56(FP), R10 - MOVQ thirdOutput+64(FP), R11 - MOVQ fourthOutput+72(FP), R12 - - VXORPS Y0, Y0, Y0 - VXORPS Y1, Y1, Y1 - VXORPS Y2, Y2, Y2 - VXORPS Y3, Y3, Y3 - MOVQ R8, R13 - SHRQ $3, R13 - JE reduce4 - -vector_loop4: - VMOVUPS (AX), Y4 - VMULPS (BX), Y4, Y5 - VMULPS (CX), Y4, Y6 - VMULPS (DX), Y4, Y7 - VMULPS (SI), Y4, Y8 - VADDPS Y5, Y0, Y0 - VADDPS Y6, Y1, Y1 - VADDPS Y7, Y2, Y2 - VADDPS Y8, Y3, Y3 - ADDQ $32, AX - ADDQ $32, BX - ADDQ $32, CX - ADDQ $32, DX - ADDQ $32, SI - DECQ R13 - JNE vector_loop4 - -reduce4: - VMOVUPS Y0, 0(SP) - VMOVUPS Y1, 32(SP) - VMOVUPS Y2, 64(SP) - VMOVUPS Y3, 96(SP) - VXORPS X4, X4, X4 - VXORPS X5, X5, X5 - VXORPS X6, X6, X6 - VXORPS X7, X7, X7 - VADDSS 0(SP), X4, X4 - VADDSS 4(SP), X4, X4 - VADDSS 8(SP), X4, X4 - VADDSS 12(SP), X4, X4 - VADDSS 16(SP), X4, X4 - VADDSS 20(SP), X4, X4 - VADDSS 24(SP), X4, X4 - VADDSS 28(SP), X4, X4 - VADDSS 32(SP), X5, X5 - VADDSS 36(SP), X5, X5 - VADDSS 40(SP), X5, X5 - VADDSS 44(SP), X5, X5 - VADDSS 48(SP), X5, X5 - VADDSS 52(SP), X5, X5 - VADDSS 56(SP), X5, X5 - VADDSS 60(SP), X5, X5 - VADDSS 64(SP), X6, X6 - VADDSS 68(SP), X6, X6 - VADDSS 72(SP), X6, X6 - VADDSS 76(SP), X6, X6 - VADDSS 80(SP), X6, X6 - VADDSS 84(SP), X6, X6 - VADDSS 88(SP), X6, X6 - VADDSS 92(SP), X6, X6 - VADDSS 96(SP), X7, X7 - VADDSS 100(SP), X7, X7 - VADDSS 104(SP), X7, X7 - VADDSS 108(SP), X7, X7 - VADDSS 112(SP), X7, X7 - VADDSS 116(SP), X7, X7 - VADDSS 120(SP), X7, X7 - VADDSS 124(SP), X7, X7 - - ANDQ $7, R8 - JE done4 - -tail_loop4: - VMOVSS (AX), X8 - VMULSS (BX), X8, X9 - VMULSS (CX), X8, X10 - VMULSS (DX), X8, X11 - VMULSS (SI), X8, X12 - VADDSS X9, X4, X4 - VADDSS X10, X5, X5 - VADDSS X11, X6, X6 - VADDSS X12, X7, X7 - ADDQ $4, AX - ADDQ $4, BX - ADDQ $4, CX - ADDQ $4, DX - ADDQ $4, SI - DECQ R8 - JNE tail_loop4 - -done4: - VMOVSS X4, (R9) - VMOVSS X5, (R10) - VMOVSS X6, (R11) - VMOVSS X7, (R12) - VZEROUPPER - RET diff --git a/internal/floats/floats_neon.go b/internal/floats/floats_neon.go deleted file mode 100644 index 4e7a1ce..0000000 --- a/internal/floats/floats_neon.go +++ /dev/null @@ -1,20 +0,0 @@ -//go:build !noasm && arm64 -// Code generated by GoAT. DO NOT EDIT. -// versions: -// clang 21.1.8 (6ubuntu1) -// objdump 2.46 -// flags: -O3 -// source: src/floats_neon.c - -package floats - -import "unsafe" - -//go:noescape -func xvec_neon_l2_squared(left, right unsafe.Pointer, size int64, output unsafe.Pointer) - -//go:noescape -func xvec_neon_inner_product(left, right unsafe.Pointer, size int64, output unsafe.Pointer) - -//go:noescape -func xvec_neon_dot_norms(left, right unsafe.Pointer, size int64, dot, left_norm, right_norm unsafe.Pointer) diff --git a/internal/floats/floats_neon.s b/internal/floats/floats_neon.s deleted file mode 100644 index b99617b..0000000 --- a/internal/floats/floats_neon.s +++ /dev/null @@ -1,234 +0,0 @@ -//go:build !noasm && arm64 -// Code generated by GoAT. DO NOT EDIT. -// versions: -// clang 21.1.8 (6ubuntu1) -// objdump 2.46 -// flags: -O3 -// source: src/floats_neon.c - -TEXT ·xvec_neon_l2_squared(SB), $0-32 - MOVD left+0(FP), R0 - MOVD right+8(FP), R1 - MOVD size+16(FP), R2 - MOVD output+24(FP), R3 - WORD $0x91000c48 // add x8, x2, #3 - WORD $0xf100005f // cmp x2, #0 - WORD $0x6f00e400 // movi v0.2d, #0000000000000000 - WORD $0x9a82b109 // csel x9, x8, x2, lt - WORD $0xf100105f // cmp x2, #4 - WORD $0x927ef528 // and x8, x9, #0xfffffffffffffffc - WORD $0xcb080048 // sub x8, x2, x8 - BLT LBB0_4 - WORD $0x9342fd29 // asr x9, x9, #2 - WORD $0xaa0003eb // mov x11, x0 - WORD $0xaa0103ec // mov x12, x1 - WORD $0xaa0903ea // mov x10, x9 - -LBB0_2: - WORD $0x3cc10561 // ldr q1, [x11], #16 - WORD $0xf100054a // subs x10, x10, #1 - WORD $0x3cc10582 // ldr q2, [x12], #16 - WORD $0x4ea2d421 // fsub v1.4s, v1.4s, v2.4s - WORD $0x4e21cc20 // fmla v0.4s, v1.4s, v1.4s - BNE LBB0_2 - WORD $0xd37ced29 // lsl x9, x9, #4 - WORD $0x8b090000 // add x0, x0, x9 - WORD $0x8b090021 // add x1, x1, x9 - -LBB0_4: - WORD $0x4e0c0401 // dup v1.4s, v0.s[1] - WORD $0x4e140402 // dup v2.4s, v0.s[2] - WORD $0xf100011f // cmp x8, #0 - WORD $0x4e21d401 // fadd v1.4s, v0.4s, v1.4s - WORD $0x4e1c0400 // dup v0.4s, v0.s[3] - WORD $0x4e21d441 // fadd v1.4s, v2.4s, v1.4s - WORD $0x4e21d400 // fadd v0.4s, v0.4s, v1.4s - BLE LBB0_8 - WORD $0xbd400001 // ldr s1, [x0] - WORD $0xbd400022 // ldr s2, [x1] - WORD $0xf100051f // cmp x8, #1 - WORD $0x1e223821 // fsub s1, s1, s2 - WORD $0x1f010020 // fmadd s0, s1, s1, s0 - BEQ LBB0_8 - WORD $0xbd400401 // ldr s1, [x0, #4] - WORD $0xbd400422 // ldr s2, [x1, #4] - WORD $0xf100091f // cmp x8, #2 - WORD $0x1e223821 // fsub s1, s1, s2 - WORD $0x1f010020 // fmadd s0, s1, s1, s0 - BEQ LBB0_8 - WORD $0xbd400801 // ldr s1, [x0, #8] - WORD $0xbd400822 // ldr s2, [x1, #8] - WORD $0x1e223821 // fsub s1, s1, s2 - WORD $0x1f010020 // fmadd s0, s1, s1, s0 - -LBB0_8: - WORD $0xbd000060 // str s0, [x3] - RET - -TEXT ·xvec_neon_inner_product(SB), $0-32 - MOVD left+0(FP), R0 - MOVD right+8(FP), R1 - MOVD size+16(FP), R2 - MOVD output+24(FP), R3 - WORD $0x91000c48 // add x8, x2, #3 - WORD $0xf100005f // cmp x2, #0 - WORD $0x6f00e400 // movi v0.2d, #0000000000000000 - WORD $0x9a82b109 // csel x9, x8, x2, lt - WORD $0xf100105f // cmp x2, #4 - WORD $0x927ef528 // and x8, x9, #0xfffffffffffffffc - WORD $0xcb080048 // sub x8, x2, x8 - BLT LBB1_4 - WORD $0x9342fd29 // asr x9, x9, #2 - WORD $0xaa0003eb // mov x11, x0 - WORD $0xaa0103ec // mov x12, x1 - WORD $0xaa0903ea // mov x10, x9 - -LBB1_2: - WORD $0x3cc10561 // ldr q1, [x11], #16 - WORD $0xf100054a // subs x10, x10, #1 - WORD $0x3cc10582 // ldr q2, [x12], #16 - WORD $0x4e21cc40 // fmla v0.4s, v2.4s, v1.4s - BNE LBB1_2 - WORD $0xd37ced29 // lsl x9, x9, #4 - WORD $0x8b090000 // add x0, x0, x9 - WORD $0x8b090021 // add x1, x1, x9 - -LBB1_4: - WORD $0x4e0c0401 // dup v1.4s, v0.s[1] - WORD $0x4e140402 // dup v2.4s, v0.s[2] - WORD $0xf100011f // cmp x8, #0 - WORD $0x4e21d401 // fadd v1.4s, v0.4s, v1.4s - WORD $0x4e1c0400 // dup v0.4s, v0.s[3] - WORD $0x4e21d441 // fadd v1.4s, v2.4s, v1.4s - WORD $0x4e21d400 // fadd v0.4s, v0.4s, v1.4s - BLE LBB1_8 - WORD $0xbd400001 // ldr s1, [x0] - WORD $0xbd400022 // ldr s2, [x1] - WORD $0xf100051f // cmp x8, #1 - WORD $0x1f020020 // fmadd s0, s1, s2, s0 - BEQ LBB1_8 - WORD $0xbd400401 // ldr s1, [x0, #4] - WORD $0xbd400422 // ldr s2, [x1, #4] - WORD $0xf100091f // cmp x8, #2 - WORD $0x1f020020 // fmadd s0, s1, s2, s0 - BEQ LBB1_8 - WORD $0xbd400801 // ldr s1, [x0, #8] - WORD $0xbd400822 // ldr s2, [x1, #8] - WORD $0x1f020020 // fmadd s0, s1, s2, s0 - -LBB1_8: - WORD $0xbd000060 // str s0, [x3] - RET - -TEXT ·xvec_neon_dot_norms(SB), $0-48 - MOVD left+0(FP), R0 - MOVD right+8(FP), R1 - MOVD size+16(FP), R2 - MOVD dot+24(FP), R3 - MOVD left_norm+32(FP), R4 - MOVD right_norm+40(FP), R5 - WORD $0x91000c48 // add x8, x2, #3 - WORD $0xf100005f // cmp x2, #0 - WORD $0x6f00e400 // movi v0.2d, #0000000000000000 - WORD $0x9a82b109 // csel x9, x8, x2, lt - WORD $0xf100105f // cmp x2, #4 - WORD $0x927ef528 // and x8, x9, #0xfffffffffffffffc - WORD $0xcb080048 // sub x8, x2, x8 - BLT LBB2_4 - WORD $0x6f00e401 // movi v1.2d, #0000000000000000 - WORD $0x6f00e402 // movi v2.2d, #0000000000000000 - WORD $0x9342fd29 // asr x9, x9, #2 - WORD $0xaa0003eb // mov x11, x0 - WORD $0xaa0103ec // mov x12, x1 - WORD $0xaa0903ea // mov x10, x9 - -LBB2_2: - WORD $0x3cc10563 // ldr q3, [x11], #16 - WORD $0xf100054a // subs x10, x10, #1 - WORD $0x3cc10584 // ldr q4, [x12], #16 - WORD $0x4e23cc62 // fmla v2.4s, v3.4s, v3.4s - WORD $0x4e23cc81 // fmla v1.4s, v4.4s, v3.4s - WORD $0x4e24cc80 // fmla v0.4s, v4.4s, v4.4s - BNE LBB2_2 - WORD $0xd37ced29 // lsl x9, x9, #4 - WORD $0x8b090000 // add x0, x0, x9 - WORD $0x8b090021 // add x1, x1, x9 - B LBB2_5 - -LBB2_4: - WORD $0x6f00e402 // movi v2.2d, #0000000000000000 - WORD $0x6f00e401 // movi v1.2d, #0000000000000000 - -LBB2_5: - WORD $0x4e0c0423 // dup v3.4s, v1.s[1] - WORD $0x4e0c0444 // dup v4.4s, v2.s[1] - WORD $0xf100011f // cmp x8, #0 - WORD $0x4e0c0405 // dup v5.4s, v0.s[1] - WORD $0x4e140426 // dup v6.4s, v1.s[2] - WORD $0x4e140447 // dup v7.4s, v2.s[2] - WORD $0x4e140410 // dup v16.4s, v0.s[2] - WORD $0x4e23d423 // fadd v3.4s, v1.4s, v3.4s - WORD $0x4e24d444 // fadd v4.4s, v2.4s, v4.4s - WORD $0x4e1c0421 // dup v1.4s, v1.s[3] - WORD $0x4e25d405 // fadd v5.4s, v0.4s, v5.4s - WORD $0x4e1c0442 // dup v2.4s, v2.s[3] - WORD $0x4e1c0400 // dup v0.4s, v0.s[3] - WORD $0x4e23d4c3 // fadd v3.4s, v6.4s, v3.4s - WORD $0x4e24d4e4 // fadd v4.4s, v7.4s, v4.4s - WORD $0x4e25d605 // fadd v5.4s, v16.4s, v5.4s - WORD $0x4e23d421 // fadd v1.4s, v1.4s, v3.4s - WORD $0x4e24d442 // fadd v2.4s, v2.4s, v4.4s - WORD $0x4e25d400 // fadd v0.4s, v0.4s, v5.4s - WORD $0xbd000061 // str s1, [x3] - WORD $0xbd000082 // str s2, [x4] - WORD $0xbd0000a0 // str s0, [x5] - BLE LBB2_8 - WORD $0xbd400000 // ldr s0, [x0] - WORD $0xbd400021 // ldr s1, [x1] - WORD $0xf100051f // cmp x8, #1 - WORD $0xbd400062 // ldr s2, [x3] - WORD $0x1f010800 // fmadd s0, s0, s1, s2 - WORD $0xbd000060 // str s0, [x3] - WORD $0xbd400000 // ldr s0, [x0] - WORD $0xbd400081 // ldr s1, [x4] - WORD $0x1f000400 // fmadd s0, s0, s0, s1 - WORD $0xbd000080 // str s0, [x4] - WORD $0xbd400020 // ldr s0, [x1] - WORD $0xbd4000a1 // ldr s1, [x5] - WORD $0x1f000400 // fmadd s0, s0, s0, s1 - WORD $0xbd0000a0 // str s0, [x5] - BEQ LBB2_8 - WORD $0xbd400400 // ldr s0, [x0, #4] - WORD $0xbd400421 // ldr s1, [x1, #4] - WORD $0xf100091f // cmp x8, #2 - WORD $0xbd400062 // ldr s2, [x3] - WORD $0x1f010800 // fmadd s0, s0, s1, s2 - WORD $0xbd000060 // str s0, [x3] - WORD $0xbd400400 // ldr s0, [x0, #4] - WORD $0xbd400081 // ldr s1, [x4] - WORD $0x1f000400 // fmadd s0, s0, s0, s1 - WORD $0xbd000080 // str s0, [x4] - WORD $0xbd400420 // ldr s0, [x1, #4] - WORD $0xbd4000a1 // ldr s1, [x5] - WORD $0x1f000400 // fmadd s0, s0, s0, s1 - WORD $0xbd0000a0 // str s0, [x5] - BNE LBB2_9 - -LBB2_8: - RET - -LBB2_9: - WORD $0xbd400800 // ldr s0, [x0, #8] - WORD $0xbd400821 // ldr s1, [x1, #8] - WORD $0xbd400062 // ldr s2, [x3] - WORD $0x1f010800 // fmadd s0, s0, s1, s2 - WORD $0xbd000060 // str s0, [x3] - WORD $0xbd400800 // ldr s0, [x0, #8] - WORD $0xbd400081 // ldr s1, [x4] - WORD $0x1f000400 // fmadd s0, s0, s0, s1 - WORD $0xbd000080 // str s0, [x4] - WORD $0xbd400820 // ldr s0, [x1, #8] - WORD $0xbd4000a1 // ldr s1, [x5] - WORD $0x1f000400 // fmadd s0, s0, s0, s1 - WORD $0xbd0000a0 // str s0, [x5] - RET diff --git a/internal/floats/src/.gitignore b/internal/floats/src/.gitignore deleted file mode 100644 index d897b56..0000000 --- a/internal/floats/src/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -*.o -*.s diff --git a/internal/floats/src/floats_avx.c b/internal/floats/src/floats_avx.c deleted file mode 100644 index e3f87ce..0000000 --- a/internal/floats/src/floats_avx.c +++ /dev/null @@ -1,90 +0,0 @@ -// Copyright 2026-present the xvec project -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include -#include - -static inline float reduce256(__m256 value) { - float partial[8]; - _mm256_storeu_ps(partial, value); - float sum = 0; - for (int index = 0; index < 8; index++) { - sum += partial[index]; - } - return sum; -} - -void xvec_avx_l2_squared(float *left, float *right, int64_t size, float *output) { - int64_t vectors = size / 8; - int64_t remain = size % 8; - __m256 sum = _mm256_setzero_ps(); - for (int64_t index = 0; index < vectors; index++) { - __m256 left_value = _mm256_loadu_ps(left); - __m256 right_value = _mm256_loadu_ps(right); - __m256 difference = _mm256_sub_ps(left_value, right_value); - sum = _mm256_add_ps(sum, _mm256_mul_ps(difference, difference)); - left += 8; - right += 8; - } - float result = reduce256(sum); - for (int64_t index = 0; index < remain; index++) { - float difference = left[index] - right[index]; - result += difference * difference; - } - *output = result; -} - -void xvec_avx_inner_product(float *left, float *right, int64_t size, float *output) { - int64_t vectors = size / 8; - int64_t remain = size % 8; - __m256 sum = _mm256_setzero_ps(); - for (int64_t index = 0; index < vectors; index++) { - __m256 left_value = _mm256_loadu_ps(left); - __m256 right_value = _mm256_loadu_ps(right); - sum = _mm256_add_ps(sum, _mm256_mul_ps(left_value, right_value)); - left += 8; - right += 8; - } - float result = reduce256(sum); - for (int64_t index = 0; index < remain; index++) { - result += left[index] * right[index]; - } - *output = result; -} - -void xvec_avx_dot_norms(float *left, float *right, int64_t size, - float *dot, float *left_norm, float *right_norm) { - int64_t vectors = size / 8; - int64_t remain = size % 8; - __m256 dot_sum = _mm256_setzero_ps(); - __m256 left_sum = _mm256_setzero_ps(); - __m256 right_sum = _mm256_setzero_ps(); - for (int64_t index = 0; index < vectors; index++) { - __m256 left_value = _mm256_loadu_ps(left); - __m256 right_value = _mm256_loadu_ps(right); - dot_sum = _mm256_add_ps(dot_sum, _mm256_mul_ps(left_value, right_value)); - left_sum = _mm256_add_ps(left_sum, _mm256_mul_ps(left_value, left_value)); - right_sum = _mm256_add_ps(right_sum, _mm256_mul_ps(right_value, right_value)); - left += 8; - right += 8; - } - *dot = reduce256(dot_sum); - *left_norm = reduce256(left_sum); - *right_norm = reduce256(right_sum); - for (int64_t index = 0; index < remain; index++) { - *dot += left[index] * right[index]; - *left_norm += left[index] * left[index]; - *right_norm += right[index] * right[index]; - } -} diff --git a/internal/floats/src/floats_avx512.c b/internal/floats/src/floats_avx512.c deleted file mode 100644 index f3e26d9..0000000 --- a/internal/floats/src/floats_avx512.c +++ /dev/null @@ -1,90 +0,0 @@ -// Copyright 2026-present the xvec project -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include -#include - -static inline float reduce512(__m512 value) { - float partial[16]; - _mm512_storeu_ps(partial, value); - float sum = 0; - for (int index = 0; index < 16; index++) { - sum += partial[index]; - } - return sum; -} - -void xvec_avx512_l2_squared(float *left, float *right, int64_t size, float *output) { - int64_t vectors = size / 16; - int64_t remain = size % 16; - __m512 sum = _mm512_setzero_ps(); - for (int64_t index = 0; index < vectors; index++) { - __m512 left_value = _mm512_loadu_ps(left); - __m512 right_value = _mm512_loadu_ps(right); - __m512 difference = _mm512_sub_ps(left_value, right_value); - sum = _mm512_fmadd_ps(difference, difference, sum); - left += 16; - right += 16; - } - float result = reduce512(sum); - for (int64_t index = 0; index < remain; index++) { - float difference = left[index] - right[index]; - result += difference * difference; - } - *output = result; -} - -void xvec_avx512_inner_product(float *left, float *right, int64_t size, float *output) { - int64_t vectors = size / 16; - int64_t remain = size % 16; - __m512 sum = _mm512_setzero_ps(); - for (int64_t index = 0; index < vectors; index++) { - __m512 left_value = _mm512_loadu_ps(left); - __m512 right_value = _mm512_loadu_ps(right); - sum = _mm512_fmadd_ps(left_value, right_value, sum); - left += 16; - right += 16; - } - float result = reduce512(sum); - for (int64_t index = 0; index < remain; index++) { - result += left[index] * right[index]; - } - *output = result; -} - -void xvec_avx512_dot_norms(float *left, float *right, int64_t size, - float *dot, float *left_norm, float *right_norm) { - int64_t vectors = size / 16; - int64_t remain = size % 16; - __m512 dot_sum = _mm512_setzero_ps(); - __m512 left_sum = _mm512_setzero_ps(); - __m512 right_sum = _mm512_setzero_ps(); - for (int64_t index = 0; index < vectors; index++) { - __m512 left_value = _mm512_loadu_ps(left); - __m512 right_value = _mm512_loadu_ps(right); - dot_sum = _mm512_fmadd_ps(left_value, right_value, dot_sum); - left_sum = _mm512_fmadd_ps(left_value, left_value, left_sum); - right_sum = _mm512_fmadd_ps(right_value, right_value, right_sum); - left += 16; - right += 16; - } - *dot = reduce512(dot_sum); - *left_norm = reduce512(left_sum); - *right_norm = reduce512(right_sum); - for (int64_t index = 0; index < remain; index++) { - *dot += left[index] * right[index]; - *left_norm += left[index] * left[index]; - *right_norm += right[index] * right[index]; - } -} diff --git a/internal/floats/src/floats_neon.c b/internal/floats/src/floats_neon.c deleted file mode 100644 index dd60bf5..0000000 --- a/internal/floats/src/floats_neon.c +++ /dev/null @@ -1,86 +0,0 @@ -// Copyright 2026-present the xvec project -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include -#include - -static inline float reduce_neon(float32x4_t value) { - float partial[4]; - vst1q_f32(partial, value); - return partial[0] + partial[1] + partial[2] + partial[3]; -} - -void xvec_neon_l2_squared(float *left, float *right, int64_t size, float *output) { - int64_t vectors = size / 4; - int64_t remain = size % 4; - float32x4_t sum = vdupq_n_f32(0); - for (int64_t index = 0; index < vectors; index++) { - float32x4_t left_value = vld1q_f32(left); - float32x4_t right_value = vld1q_f32(right); - float32x4_t difference = vsubq_f32(left_value, right_value); - sum = vmlaq_f32(sum, difference, difference); - left += 4; - right += 4; - } - float result = reduce_neon(sum); - for (int64_t index = 0; index < remain; index++) { - float difference = left[index] - right[index]; - result += difference * difference; - } - *output = result; -} - -void xvec_neon_inner_product(float *left, float *right, int64_t size, float *output) { - int64_t vectors = size / 4; - int64_t remain = size % 4; - float32x4_t sum = vdupq_n_f32(0); - for (int64_t index = 0; index < vectors; index++) { - float32x4_t left_value = vld1q_f32(left); - float32x4_t right_value = vld1q_f32(right); - sum = vmlaq_f32(sum, left_value, right_value); - left += 4; - right += 4; - } - float result = reduce_neon(sum); - for (int64_t index = 0; index < remain; index++) { - result += left[index] * right[index]; - } - *output = result; -} - -void xvec_neon_dot_norms(float *left, float *right, int64_t size, - float *dot, float *left_norm, float *right_norm) { - int64_t vectors = size / 4; - int64_t remain = size % 4; - float32x4_t dot_sum = vdupq_n_f32(0); - float32x4_t left_sum = vdupq_n_f32(0); - float32x4_t right_sum = vdupq_n_f32(0); - for (int64_t index = 0; index < vectors; index++) { - float32x4_t left_value = vld1q_f32(left); - float32x4_t right_value = vld1q_f32(right); - dot_sum = vmlaq_f32(dot_sum, left_value, right_value); - left_sum = vmlaq_f32(left_sum, left_value, left_value); - right_sum = vmlaq_f32(right_sum, right_value, right_value); - left += 4; - right += 4; - } - *dot = reduce_neon(dot_sum); - *left_norm = reduce_neon(left_sum); - *right_norm = reduce_neon(right_sum); - for (int64_t index = 0; index < remain; index++) { - *dot += left[index] * right[index]; - *left_norm += left[index] * left[index]; - *right_norm += right[index] * right[index]; - } -}