sevenhill/QSfera
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
4bf54f7fdadcd619063f5597accbf296b9bc898f
QSfera/Server/vendor/github.com/klauspost/crc32/crc32_amd64.s
T
Курнат Андрей 2315f25754 Initial QSfera import
2026-06-07 10:20:04 +03:00

528 lines
11 KiB
ArmAsm
Raw Blame History

// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
#include "textflag.h"
// castagnoliSSE42 updates the (non-inverted) crc with the given buffer.
//
// func castagnoliSSE42(crc uint32, p []byte) uint32
TEXT ·castagnoliSSE42(SB), NOSPLIT, $0
MOVL crc+0(FP), AX // CRC value
MOVQ p+8(FP), SI // data pointer
MOVQ p_len+16(FP), CX // len(p)
// If there are fewer than 8 bytes to process, skip alignment.
CMPQ CX, $8
JL less_than_8
MOVQ SI, BX
ANDQ $7, BX
JZ aligned
// Process the first few bytes to 8-byte align the input.
// BX = 8 - BX. We need to process this many bytes to align.
SUBQ $1, BX
XORQ $7, BX
BTQ $0, BX
JNC align_2
CRC32B (SI), AX
DECQ CX
INCQ SI
align_2:
BTQ $1, BX
JNC align_4
CRC32W (SI), AX
SUBQ $2, CX
ADDQ $2, SI
align_4:
BTQ $2, BX
JNC aligned
CRC32L (SI), AX
SUBQ $4, CX
ADDQ $4, SI
aligned:
// The input is now 8-byte aligned and we can process 8-byte chunks.
CMPQ CX, $8
JL less_than_8
CRC32Q (SI), AX
ADDQ $8, SI
SUBQ $8, CX
JMP aligned
less_than_8:
// We may have some bytes left over; process 4 bytes, then 2, then 1.
BTQ $2, CX
JNC less_than_4
CRC32L (SI), AX
ADDQ $4, SI
less_than_4:
BTQ $1, CX
JNC less_than_2
CRC32W (SI), AX
ADDQ $2, SI
less_than_2:
BTQ $0, CX
JNC done
CRC32B (SI), AX
done:
MOVL AX, ret+32(FP)
RET
// castagnoliSSE42Triple updates three (non-inverted) crcs with (24*rounds)
// bytes from each buffer.
//
// func castagnoliSSE42Triple(
// crc1, crc2, crc3 uint32,
// a, b, c []byte,
// rounds uint32,
// ) (retA uint32, retB uint32, retC uint32)
TEXT ·castagnoliSSE42Triple(SB), NOSPLIT, $0
MOVL crcA+0(FP), AX
MOVL crcB+4(FP), CX
MOVL crcC+8(FP), DX
MOVQ a+16(FP), R8 // data pointer
MOVQ b+40(FP), R9 // data pointer
MOVQ c+64(FP), R10 // data pointer
MOVL rounds+88(FP), R11
loop:
CRC32Q (R8), AX
CRC32Q (R9), CX
CRC32Q (R10), DX
CRC32Q 8(R8), AX
CRC32Q 8(R9), CX
CRC32Q 8(R10), DX
CRC32Q 16(R8), AX
CRC32Q 16(R9), CX
CRC32Q 16(R10), DX
ADDQ $24, R8
ADDQ $24, R9
ADDQ $24, R10
DECQ R11
JNZ loop
MOVL AX, retA+96(FP)
MOVL CX, retB+100(FP)
MOVL DX, retC+104(FP)
RET
// CRC32 polynomial data
//
// These constants are lifted from the
// Linux kernel, since they avoid the costly
// PSHUFB 16 byte reversal proposed in the
// original Intel paper.
DATA r2r1<>+0(SB)/8, $0x154442bd4
DATA r2r1<>+8(SB)/8, $0x1c6e41596
DATA r4r3<>+0(SB)/8, $0x1751997d0
DATA r4r3<>+8(SB)/8, $0x0ccaa009e
DATA rupoly<>+0(SB)/8, $0x1db710641
DATA rupoly<>+8(SB)/8, $0x1f7011641
DATA r5<>+0(SB)/8, $0x163cd6124
GLOBL r2r1<>(SB), RODATA, $16
GLOBL r4r3<>(SB), RODATA, $16
GLOBL rupoly<>(SB), RODATA, $16
GLOBL r5<>(SB), RODATA, $8
// Based on https://www.intel.com/content/dam/www/public/us/en/documents/white-papers/fast-crc-computation-generic-polynomials-pclmulqdq-paper.pdf
// len(p) must be at least 64, and must be a multiple of 16.
// func ieeeCLMUL(crc uint32, p []byte) uint32
TEXT ·ieeeCLMUL(SB), NOSPLIT, $0
MOVL crc+0(FP), X0 // Initial CRC value
MOVQ p+8(FP), SI // data pointer
MOVQ p_len+16(FP), CX // len(p)
MOVOU (SI), X1
MOVOU 16(SI), X2
MOVOU 32(SI), X3
MOVOU 48(SI), X4
PXOR X0, X1
ADDQ $64, SI // buf+=64
SUBQ $64, CX // len-=64
CMPQ CX, $64 // Less than 64 bytes left
JB remain64
MOVOA r2r1<>+0(SB), X0
loopback64:
MOVOA X1, X5
MOVOA X2, X6
MOVOA X3, X7
MOVOA X4, X8
PCLMULQDQ $0, X0, X1
PCLMULQDQ $0, X0, X2
PCLMULQDQ $0, X0, X3
PCLMULQDQ $0, X0, X4
// Load next early
MOVOU (SI), X11
MOVOU 16(SI), X12
MOVOU 32(SI), X13
MOVOU 48(SI), X14
PCLMULQDQ $0x11, X0, X5
PCLMULQDQ $0x11, X0, X6
PCLMULQDQ $0x11, X0, X7
PCLMULQDQ $0x11, X0, X8
PXOR X5, X1
PXOR X6, X2
PXOR X7, X3
PXOR X8, X4
PXOR X11, X1
PXOR X12, X2
PXOR X13, X3
PXOR X14, X4
ADDQ $0x40, DI
ADDQ $64, SI // buf+=64
SUBQ $64, CX // len-=64
CMPQ CX, $64 // Less than 64 bytes left?
JGE loopback64
// Fold result into a single register (X1)
remain64:
MOVOA r4r3<>+0(SB), X0
MOVOA X1, X5
PCLMULQDQ $0, X0, X1
PCLMULQDQ $0x11, X0, X5
PXOR X5, X1
PXOR X2, X1
MOVOA X1, X5
PCLMULQDQ $0, X0, X1
PCLMULQDQ $0x11, X0, X5
PXOR X5, X1
PXOR X3, X1
MOVOA X1, X5
PCLMULQDQ $0, X0, X1
PCLMULQDQ $0x11, X0, X5
PXOR X5, X1
PXOR X4, X1
// If there is less than 16 bytes left we are done
CMPQ CX, $16
JB finish
// Encode 16 bytes
remain16:
MOVOU (SI), X10
MOVOA X1, X5
PCLMULQDQ $0, X0, X1
PCLMULQDQ $0x11, X0, X5
PXOR X5, X1
PXOR X10, X1
SUBQ $16, CX
ADDQ $16, SI
CMPQ CX, $16
JGE remain16
finish:
// Fold final result into 32 bits and return it
PCMPEQB X3, X3
PCLMULQDQ $1, X1, X0
PSRLDQ $8, X1
PXOR X0, X1
MOVOA X1, X2
MOVQ r5<>+0(SB), X0
// Creates 32 bit mask. Note that we don't care about upper half.
PSRLQ $32, X3
PSRLDQ $4, X2
PAND X3, X1
PCLMULQDQ $0, X0, X1
PXOR X2, X1
MOVOA rupoly<>+0(SB), X0
MOVOA X1, X2
PAND X3, X1
PCLMULQDQ $0x10, X0, X1
PAND X3, X1
PCLMULQDQ $0, X0, X1
PXOR X2, X1
PEXTRD $1, X1, AX
MOVL AX, ret+32(FP)
RET
DATA r2r1X<>+0(SB)/8, $0x154442bd4
DATA r2r1X<>+8(SB)/8, $0x1c6e41596
DATA r2r1X<>+16(SB)/8, $0x154442bd4
DATA r2r1X<>+24(SB)/8, $0x1c6e41596
DATA r2r1X<>+32(SB)/8, $0x154442bd4
DATA r2r1X<>+40(SB)/8, $0x1c6e41596
DATA r2r1X<>+48(SB)/8, $0x154442bd4
DATA r2r1X<>+56(SB)/8, $0x1c6e41596
GLOBL r2r1X<>(SB), RODATA, $64
// Based on https://www.intel.com/content/dam/www/public/us/en/documents/white-papers/fast-crc-computation-generic-polynomials-pclmulqdq-paper.pdf
// len(p) must be at least 128, and must be a multiple of 16.
// func ieeeCLMULAvx512(crc uint32, p []byte) uint32
TEXT ·ieeeCLMULAvx512(SB), NOSPLIT, $0
MOVL crc+0(FP), AX // Initial CRC value
MOVQ p+8(FP), SI // data pointer
MOVQ p_len+16(FP), CX // len(p)
VPXORQ Z0, Z0, Z0
VMOVDQU64 (SI), Z1
VMOVQ AX, X0
VPXORQ Z0, Z1, Z1 // Merge initial CRC value into Z1
ADDQ $64, SI // buf+=64
SUBQ $64, CX // len-=64
VMOVDQU64 r2r1X<>+0(SB), Z0
loopback64:
// Load next early
VMOVDQU64 (SI), Z11
VPCLMULQDQ $0x11, Z0, Z1, Z5
VPCLMULQDQ $0, Z0, Z1, Z1
VPTERNLOGD $0x96, Z11, Z5, Z1 // Combine results with xor into Z1
ADDQ $0x40, DI
ADDQ $64, SI // buf+=64
SUBQ $64, CX // len-=64
CMPQ CX, $64 // Less than 64 bytes left?
JGE loopback64
// Fold result into a single register (X1)
remain64:
VEXTRACTF32X4 $1, Z1, X2 // X2: Second 128-bit lane
VEXTRACTF32X4 $2, Z1, X3 // X3: Third 128-bit lane
VEXTRACTF32X4 $3, Z1, X4 // X4: Fourth 128-bit lane
MOVOA r4r3<>+0(SB), X0
MOVOA X1, X5
PCLMULQDQ $0, X0, X1
PCLMULQDQ $0x11, X0, X5
PXOR X5, X1
PXOR X2, X1
MOVOA X1, X5
PCLMULQDQ $0, X0, X1
PCLMULQDQ $0x11, X0, X5
PXOR X5, X1
PXOR X3, X1
MOVOA X1, X5
PCLMULQDQ $0, X0, X1
PCLMULQDQ $0x11, X0, X5
PXOR X5, X1
PXOR X4, X1
// If there is less than 16 bytes left we are done
CMPQ CX, $16
JB finish
// Encode 16 bytes
remain16:
MOVOU (SI), X10
MOVOA X1, X5
PCLMULQDQ $0, X0, X1
PCLMULQDQ $0x11, X0, X5
PXOR X5, X1
PXOR X10, X1
SUBQ $16, CX
ADDQ $16, SI
CMPQ CX, $16
JGE remain16
finish:
// Fold final result into 32 bits and return it
PCMPEQB X3, X3
PCLMULQDQ $1, X1, X0
PSRLDQ $8, X1
PXOR X0, X1
MOVOA X1, X2
MOVQ r5<>+0(SB), X0
// Creates 32 bit mask. Note that we don't care about upper half.
PSRLQ $32, X3
PSRLDQ $4, X2
PAND X3, X1
PCLMULQDQ $0, X0, X1
PXOR X2, X1
MOVOA rupoly<>+0(SB), X0
MOVOA X1, X2
PAND X3, X1
PCLMULQDQ $0x10, X0, X1
PAND X3, X1
PCLMULQDQ $0, X0, X1
PXOR X2, X1
PEXTRD $1, X1, AX
MOVL AX, ret+32(FP)
VZEROUPPER
RET
// Castagonli Polynomial constants
DATA r2r1C<>+0(SB)/8, $0x0740eef02
DATA r2r1C<>+8(SB)/8, $0x09e4addf8
DATA r2r1C<>+16(SB)/8, $0x0740eef02
DATA r2r1C<>+24(SB)/8, $0x09e4addf8
DATA r2r1C<>+32(SB)/8, $0x0740eef02
DATA r2r1C<>+40(SB)/8, $0x09e4addf8
DATA r2r1C<>+48(SB)/8, $0x0740eef02
DATA r2r1C<>+56(SB)/8, $0x09e4addf8
GLOBL r2r1C<>(SB), RODATA, $64
DATA r4r3C<>+0(SB)/8, $0xf20c0dfe
DATA r4r3C<>+8(SB)/8, $0x14cd00bd6
DATA rupolyC<>+0(SB)/8, $0x105ec76f0
DATA rupolyC<>+8(SB)/8, $0xdea713f1
DATA r5C<>+0(SB)/8, $0xdd45aab8
GLOBL r4r3C<>(SB), RODATA, $16
GLOBL rupolyC<>(SB), RODATA, $16
GLOBL r5C<>(SB), RODATA, $8
// Based on https://www.intel.com/content/dam/www/public/us/en/documents/white-papers/fast-crc-computation-generic-polynomials-pclmulqdq-paper.pdf
// len(p) must be at least 128, and must be a multiple of 16.
// func castagnoliCLMULAvx512(crc uint32, p []byte) uint32
TEXT ·castagnoliCLMULAvx512(SB), NOSPLIT, $0
MOVL crc+0(FP), AX // Initial CRC value
MOVQ p+8(FP), SI // data pointer
MOVQ p_len+16(FP), CX // len(p)
VPXORQ Z0, Z0, Z0
VMOVDQU64 (SI), Z1
VMOVQ AX, X0
VPXORQ Z0, Z1, Z1 // Merge initial CRC value into Z1
ADDQ $64, SI // buf+=64
SUBQ $64, CX // len-=64
VMOVDQU64 r2r1C<>+0(SB), Z0
loopback64:
// Load next early
VMOVDQU64 (SI), Z11
VPCLMULQDQ $0x11, Z0, Z1, Z5
VPCLMULQDQ $0, Z0, Z1, Z1
VPTERNLOGD $0x96, Z11, Z5, Z1 // Combine results with xor into Z1
ADDQ $0x40, DI
ADDQ $64, SI // buf+=64
SUBQ $64, CX // len-=64
CMPQ CX, $64 // Less than 64 bytes left?
JGE loopback64
// Fold result into a single register (X1)
remain64:
VEXTRACTF32X4 $1, Z1, X2 // X2: Second 128-bit lane
VEXTRACTF32X4 $2, Z1, X3 // X3: Third 128-bit lane
VEXTRACTF32X4 $3, Z1, X4 // X4: Fourth 128-bit lane
MOVOA r4r3C<>+0(SB), X0
MOVOA X1, X5
PCLMULQDQ $0, X0, X1
PCLMULQDQ $0x11, X0, X5
PXOR X5, X1
PXOR X2, X1
MOVOA X1, X5
PCLMULQDQ $0, X0, X1
PCLMULQDQ $0x11, X0, X5
PXOR X5, X1
PXOR X3, X1
MOVOA X1, X5
PCLMULQDQ $0, X0, X1
PCLMULQDQ $0x11, X0, X5
PXOR X5, X1
PXOR X4, X1
// If there is less than 16 bytes left we are done
CMPQ CX, $16
JB finish
// Encode 16 bytes
remain16:
MOVOU (SI), X10
MOVOA X1, X5
PCLMULQDQ $0, X0, X1
PCLMULQDQ $0x11, X0, X5
PXOR X5, X1
PXOR X10, X1
SUBQ $16, CX
ADDQ $16, SI
CMPQ CX, $16
JGE remain16
finish:
// Fold final result into 32 bits and return it
PCMPEQB X3, X3
PCLMULQDQ $1, X1, X0
PSRLDQ $8, X1
PXOR X0, X1
MOVOA X1, X2
MOVQ r5C<>+0(SB), X0
// Creates 32 bit mask. Note that we don't care about upper half.
PSRLQ $32, X3
PSRLDQ $4, X2
PAND X3, X1
PCLMULQDQ $0, X0, X1
PXOR X2, X1
MOVOA rupolyC<>+0(SB), X0
MOVOA X1, X2
PAND X3, X1
PCLMULQDQ $0x10, X0, X1
PAND X3, X1
PCLMULQDQ $0, X0, X1
PXOR X2, X1
PEXTRD $1, X1, AX
MOVL AX, ret+32(FP)
VZEROUPPER
RET
Reference in New Issue View Git Blame Copy Permalink