sevenhill/QSfera
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
cf366b2699b2ea2cf0103dcbc915991c2eee9acd
QSfera/Server/vendor/github.com/klauspost/crc32/crc32_ppc64le.s
T
Курнат Андрей 2315f25754 Initial QSfera import
2026-06-07 10:20:04 +03:00

737 lines
15 KiB
ArmAsm
Raw Blame History

// Copyright 2017 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// The vectorized implementation found below is a derived work
// from code written by Anton Blanchard <anton@au.ibm.com> found
// at https://github.com/antonblanchard/crc32-vpmsum. The original
// is dual licensed under GPL and Apache 2. As the copyright holder
// for the work, IBM has contributed this new work under
// the golang license.
// Changes include porting to Go assembler with modifications for
// the Go ABI for ppc64le.
#include "textflag.h"
#define POWER8_OFFSET 132
#define off16 R16
#define off32 R17
#define off48 R18
#define off64 R19
#define off80 R20
#define off96 R21
#define off112 R22
#define const1 V24
#define const2 V25
#define byteswap V26
#define mask_32bit V27
#define mask_64bit V28
#define zeroes V29
#define MAX_SIZE 32*1024
#define REFLECT
TEXT ·ppc64SlicingUpdateBy8(SB), NOSPLIT|NOFRAME, $0-44
MOVWZ crc+0(FP), R3 // incoming crc
MOVD table8+8(FP), R4 // *Table
MOVD p+16(FP), R5
MOVD p_len+24(FP), R6 // p len
CMP $0, R6 // len == 0?
BNE start
MOVW R3, ret+40(FP) // return crc
RET
start:
NOR R3, R3, R7 // ^crc
MOVWZ R7, R7 // 32 bits
CMP R6, $16
MOVD R6, CTR
BLT short
SRAD $3, R6, R8 // 8 byte chunks
MOVD R8, CTR
loop:
MOVWZ 0(R5), R8 // 0-3 bytes of p ?Endian?
MOVWZ 4(R5), R9 // 4-7 bytes of p
MOVD R4, R10 // &tab[0]
XOR R7, R8, R7 // crc ^= byte[0:3]
RLDICL $40, R9, $56, R17 // p[7]
SLD $2, R17, R17 // p[7]*4
RLDICL $40, R7, $56, R8 // crc>>24
SLD $2, R8, R8 // crc>>24*4
RLDICL $48, R9, $56, R18 // p[6]
SLD $2, R18, R18 // p[6]*4
MOVWZ (R10)(R17), R21 // tab[0][p[7]]
ADD $1024, R10, R10 // tab[1]
RLDICL $56, R9, $56, R19 // p[5]
SLD $2, R19, R19 // p[5]*4:1
MOVWZ (R10)(R18), R22 // tab[1][p[6]]
ADD $1024, R10, R10 // tab[2]
XOR R21, R22, R21 // xor done R22
CLRLSLDI $56, R9, $2, R20
MOVWZ (R10)(R19), R23 // tab[2][p[5]]
ADD $1024, R10, R10 // &tab[3]
XOR R21, R23, R21 // xor done R23
MOVWZ (R10)(R20), R24 // tab[3][p[4]]
ADD $1024, R10, R10 // &tab[4]
XOR R21, R24, R21 // xor done R24
MOVWZ (R10)(R8), R25 // tab[4][crc>>24]
RLDICL $48, R7, $56, R24 // crc>>16&0xFF
XOR R21, R25, R21 // xor done R25
ADD $1024, R10, R10 // &tab[5]
SLD $2, R24, R24 // crc>>16&0xFF*4
MOVWZ (R10)(R24), R26 // tab[5][crc>>16&0xFF]
XOR R21, R26, R21 // xor done R26
RLDICL $56, R7, $56, R25 // crc>>8
ADD $1024, R10, R10 // &tab[6]
SLD $2, R25, R25 // crc>>8&FF*2
MOVBZ R7, R26 // crc&0xFF
MOVWZ (R10)(R25), R27 // tab[6][crc>>8&0xFF]
ADD $1024, R10, R10 // &tab[7]
SLD $2, R26, R26 // crc&0xFF*2
XOR R21, R27, R21 // xor done R27
ADD $8, R5 // p = p[8:]
MOVWZ (R10)(R26), R28 // tab[7][crc&0xFF]
XOR R21, R28, R21 // xor done R28
MOVWZ R21, R7 // crc for next round
BDNZ loop
ANDCC $7, R6, R8 // any leftover bytes
BEQ done // none --> done
MOVD R8, CTR // byte count
PCALIGN $16 // align short loop
short:
MOVBZ 0(R5), R8 // get v
XOR R8, R7, R8 // byte(crc)^v -> R8
RLDIC $2, R8, $54, R8 // rldicl r8,r8,2,22
SRD $8, R7, R14 // crc>>8
MOVWZ (R4)(R8), R10
ADD $1, R5
XOR R10, R14, R7 // loop crc in R7
BDNZ short
done:
NOR R7, R7, R7 // ^crc
MOVW R7, ret+40(FP) // return crc
RET
#ifdef BYTESWAP_DATA
DATA ·byteswapcons+0(SB)/8, $0x0706050403020100
DATA ·byteswapcons+8(SB)/8, $0x0f0e0d0c0b0a0908
GLOBL ·byteswapcons+0(SB), RODATA, $16
#endif
TEXT ·vectorCrc32(SB), NOSPLIT|NOFRAME, $0-36
MOVWZ crc+0(FP), R3 // incoming crc
MOVWZ ctab+4(FP), R14 // crc poly id
MOVD p+8(FP), R4
MOVD p_len+16(FP), R5 // p len
// R3 = incoming crc
// R14 = constant table identifier
// R5 = address of bytes
// R6 = length of bytes
// defines for index loads
MOVD $16, off16
MOVD $32, off32
MOVD $48, off48
MOVD $64, off64
MOVD $80, off80
MOVD $96, off96
MOVD $112, off112
MOVD $0, R15
MOVD R3, R10 // save initial crc
NOR R3, R3, R3 // ^crc
MOVWZ R3, R3 // 32 bits
VXOR zeroes, zeroes, zeroes // clear the V reg
VSPLTISW $-1, V0
VSLDOI $4, V29, V0, mask_32bit
VSLDOI $8, V29, V0, mask_64bit
VXOR V8, V8, V8
MTVSRD R3, VS40 // crc initial value VS40 = V8
#ifdef REFLECT
VSLDOI $8, zeroes, V8, V8 // or: VSLDOI V29,V8,V27,4 for top 32 bits?
#else
VSLDOI $4, V8, zeroes, V8
#endif
#ifdef BYTESWAP_DATA
MOVD $·byteswapcons(SB), R3
LVX (R3), byteswap
#endif
CMPU R5, $256 // length of bytes
BLT short
RLDICR $0, R5, $56, R6 // chunk to process
// First step for larger sizes
l1:
MOVD $32768, R7
MOVD R7, R9
CMP R6, R7 // compare R6, R7 (MAX SIZE)
BGT top // less than MAX, just do remainder
MOVD R6, R7
top:
SUB R7, R6, R6
// mainloop does 128 bytes at a time
SRD $7, R7
// determine the offset into the constants table to start with.
// Each constant is 128 bytes, used against 16 bytes of data.
SLD $4, R7, R8
SRD $3, R9, R9
SUB R8, R9, R8
// The last iteration is reduced in a separate step
ADD $-1, R7
MOVD R7, CTR
// Determine which constant table (depends on poly)
CMP R14, $1
BNE castTable
MOVD $·IEEEConst(SB), R3
BR startConst
castTable:
MOVD $·CastConst(SB), R3
startConst:
ADD R3, R8, R3 // starting point in constants table
VXOR V0, V0, V0 // clear the V regs
VXOR V1, V1, V1
VXOR V2, V2, V2
VXOR V3, V3, V3
VXOR V4, V4, V4
VXOR V5, V5, V5
VXOR V6, V6, V6
VXOR V7, V7, V7
LVX (R3), const1 // loading constant values
CMP R15, $1 // Identify warm up pass
BEQ next
// First warm up pass: load the bytes to process
LVX (R4), V16
LVX (R4+off16), V17
LVX (R4+off32), V18
LVX (R4+off48), V19
LVX (R4+off64), V20
LVX (R4+off80), V21
LVX (R4+off96), V22
LVX (R4+off112), V23
ADD $128, R4 // bump up to next 128 bytes in buffer
VXOR V16, V8, V16 // xor in initial CRC in V8
next:
BC 18, 0, first_warm_up_done
ADD $16, R3 // bump up to next constants
LVX (R3), const2 // table values
VPMSUMD V16, const1, V8 // second warm up pass
LVX (R4), V16 // load from buffer
OR $0, R2, R2
VPMSUMD V17, const1, V9 // vpmsumd with constants
LVX (R4+off16), V17 // load next from buffer
OR $0, R2, R2
VPMSUMD V18, const1, V10 // vpmsumd with constants
LVX (R4+off32), V18 // load next from buffer
OR $0, R2, R2
VPMSUMD V19, const1, V11 // vpmsumd with constants
LVX (R4+off48), V19 // load next from buffer
OR $0, R2, R2
VPMSUMD V20, const1, V12 // vpmsumd with constants
LVX (R4+off64), V20 // load next from buffer
OR $0, R2, R2
VPMSUMD V21, const1, V13 // vpmsumd with constants
LVX (R4+off80), V21 // load next from buffer
OR $0, R2, R2
VPMSUMD V22, const1, V14 // vpmsumd with constants
LVX (R4+off96), V22 // load next from buffer
OR $0, R2, R2
VPMSUMD V23, const1, V15 // vpmsumd with constants
LVX (R4+off112), V23 // load next from buffer
ADD $128, R4 // bump up to next 128 bytes in buffer
BC 18, 0, first_cool_down
cool_top:
LVX (R3), const1 // constants
ADD $16, R3 // inc to next constants
OR $0, R2, R2
VXOR V0, V8, V0 // xor in previous vpmsumd
VPMSUMD V16, const2, V8 // vpmsumd with constants
LVX (R4), V16 // buffer
OR $0, R2, R2
VXOR V1, V9, V1 // xor in previous
VPMSUMD V17, const2, V9 // vpmsumd with constants
LVX (R4+off16), V17 // next in buffer
OR $0, R2, R2
VXOR V2, V10, V2 // xor in previous
VPMSUMD V18, const2, V10 // vpmsumd with constants
LVX (R4+off32), V18 // next in buffer
OR $0, R2, R2
VXOR V3, V11, V3 // xor in previous
VPMSUMD V19, const2, V11 // vpmsumd with constants
LVX (R4+off48), V19 // next in buffer
LVX (R3), const2 // get next constant
OR $0, R2, R2
VXOR V4, V12, V4 // xor in previous
VPMSUMD V20, const1, V12 // vpmsumd with constants
LVX (R4+off64), V20 // next in buffer
OR $0, R2, R2
VXOR V5, V13, V5 // xor in previous
VPMSUMD V21, const1, V13 // vpmsumd with constants
LVX (R4+off80), V21 // next in buffer
OR $0, R2, R2
VXOR V6, V14, V6 // xor in previous
VPMSUMD V22, const1, V14 // vpmsumd with constants
LVX (R4+off96), V22 // next in buffer
OR $0, R2, R2
VXOR V7, V15, V7 // xor in previous
VPMSUMD V23, const1, V15 // vpmsumd with constants
LVX (R4+off112), V23 // next in buffer
ADD $128, R4 // bump up buffer pointer
BDNZ cool_top // are we done?
first_cool_down:
// load the constants
// xor in the previous value
// vpmsumd the result with constants
LVX (R3), const1
ADD $16, R3
VXOR V0, V8, V0
VPMSUMD V16, const1, V8
OR $0, R2, R2
VXOR V1, V9, V1
VPMSUMD V17, const1, V9
OR $0, R2, R2
VXOR V2, V10, V2
VPMSUMD V18, const1, V10
OR $0, R2, R2
VXOR V3, V11, V3
VPMSUMD V19, const1, V11
OR $0, R2, R2
VXOR V4, V12, V4
VPMSUMD V20, const1, V12
OR $0, R2, R2
VXOR V5, V13, V5
VPMSUMD V21, const1, V13
OR $0, R2, R2
VXOR V6, V14, V6
VPMSUMD V22, const1, V14
OR $0, R2, R2
VXOR V7, V15, V7
VPMSUMD V23, const1, V15
OR $0, R2, R2
second_cool_down:
VXOR V0, V8, V0
VXOR V1, V9, V1
VXOR V2, V10, V2
VXOR V3, V11, V3
VXOR V4, V12, V4
VXOR V5, V13, V5
VXOR V6, V14, V6
VXOR V7, V15, V7
#ifdef REFLECT
VSLDOI $4, V0, zeroes, V0
VSLDOI $4, V1, zeroes, V1
VSLDOI $4, V2, zeroes, V2
VSLDOI $4, V3, zeroes, V3
VSLDOI $4, V4, zeroes, V4
VSLDOI $4, V5, zeroes, V5
VSLDOI $4, V6, zeroes, V6
VSLDOI $4, V7, zeroes, V7
#endif
LVX (R4), V8
LVX (R4+off16), V9
LVX (R4+off32), V10
LVX (R4+off48), V11
LVX (R4+off64), V12
LVX (R4+off80), V13
LVX (R4+off96), V14
LVX (R4+off112), V15
ADD $128, R4
VXOR V0, V8, V16
VXOR V1, V9, V17
VXOR V2, V10, V18
VXOR V3, V11, V19
VXOR V4, V12, V20
VXOR V5, V13, V21
VXOR V6, V14, V22
VXOR V7, V15, V23
MOVD $1, R15
CMP $0, R6
ADD $128, R6
BNE l1
ANDCC $127, R5
SUBC R5, $128, R6
ADD R3, R6, R3
SRD $4, R5, R7
MOVD R7, CTR
LVX (R3), V0
LVX (R3+off16), V1
LVX (R3+off32), V2
LVX (R3+off48), V3
LVX (R3+off64), V4
LVX (R3+off80), V5
LVX (R3+off96), V6
LVX (R3+off112), V7
ADD $128, R3
VPMSUMW V16, V0, V0
VPMSUMW V17, V1, V1
VPMSUMW V18, V2, V2
VPMSUMW V19, V3, V3
VPMSUMW V20, V4, V4
VPMSUMW V21, V5, V5
VPMSUMW V22, V6, V6
VPMSUMW V23, V7, V7
// now reduce the tail
CMP $0, R7
BEQ next1
LVX (R4), V16
LVX (R3), V17
VPMSUMW V16, V17, V16
VXOR V0, V16, V0
BC 18, 0, next1
LVX (R4+off16), V16
LVX (R3+off16), V17
VPMSUMW V16, V17, V16
VXOR V0, V16, V0
BC 18, 0, next1
LVX (R4+off32), V16
LVX (R3+off32), V17
VPMSUMW V16, V17, V16
VXOR V0, V16, V0
BC 18, 0, next1
LVX (R4+off48), V16
LVX (R3+off48), V17
VPMSUMW V16, V17, V16
VXOR V0, V16, V0
BC 18, 0, next1
LVX (R4+off64), V16
LVX (R3+off64), V17
VPMSUMW V16, V17, V16
VXOR V0, V16, V0
BC 18, 0, next1
LVX (R4+off80), V16
LVX (R3+off80), V17
VPMSUMW V16, V17, V16
VXOR V0, V16, V0
BC 18, 0, next1
LVX (R4+off96), V16
LVX (R3+off96), V17
VPMSUMW V16, V17, V16
VXOR V0, V16, V0
next1:
VXOR V0, V1, V0
VXOR V2, V3, V2
VXOR V4, V5, V4
VXOR V6, V7, V6
VXOR V0, V2, V0
VXOR V4, V6, V4
VXOR V0, V4, V0
barrett_reduction:
CMP R14, $1
BNE barcstTable
MOVD $·IEEEBarConst(SB), R3
BR startbarConst
barcstTable:
MOVD $·CastBarConst(SB), R3
startbarConst:
LVX (R3), const1
LVX (R3+off16), const2
VSLDOI $8, V0, V0, V1
VXOR V0, V1, V0
#ifdef REFLECT
VSPLTISB $1, V1
VSL V0, V1, V0
#endif
VAND V0, mask_64bit, V0
#ifndef REFLECT
VPMSUMD V0, const1, V1
VSLDOI $8, zeroes, V1, V1
VPMSUMD V1, const2, V1
VXOR V0, V1, V0
VSLDOI $8, V0, zeroes, V0
#else
VAND V0, mask_32bit, V1
VPMSUMD V1, const1, V1
VAND V1, mask_32bit, V1
VPMSUMD V1, const2, V1
VXOR V0, V1, V0
VSLDOI $4, V0, zeroes, V0
#endif
MFVSRD VS32, R3 // VS32 = V0
NOR R3, R3, R3 // return ^crc
MOVW R3, ret+32(FP)
RET
first_warm_up_done:
LVX (R3), const1
ADD $16, R3
VPMSUMD V16, const1, V8
VPMSUMD V17, const1, V9
VPMSUMD V18, const1, V10
VPMSUMD V19, const1, V11
VPMSUMD V20, const1, V12
VPMSUMD V21, const1, V13
VPMSUMD V22, const1, V14
VPMSUMD V23, const1, V15
BR second_cool_down
short:
CMP $0, R5
BEQ zero
// compute short constants
CMP R14, $1
BNE castshTable
MOVD $·IEEEConst(SB), R3
ADD $4080, R3
BR startshConst
castshTable:
MOVD $·CastConst(SB), R3
ADD $4080, R3
startshConst:
SUBC R5, $256, R6 // sub from 256
ADD R3, R6, R3
// calculate where to start
SRD $4, R5, R7
MOVD R7, CTR
VXOR V19, V19, V19
VXOR V20, V20, V20
LVX (R4), V0
LVX (R3), V16
VXOR V0, V8, V0
VPMSUMW V0, V16, V0
BC 18, 0, v0
LVX (R4+off16), V1
LVX (R3+off16), V17
VPMSUMW V1, V17, V1
BC 18, 0, v1
LVX (R4+off32), V2
LVX (R3+off32), V16
VPMSUMW V2, V16, V2
BC 18, 0, v2
LVX (R4+off48), V3
LVX (R3+off48), V17
VPMSUMW V3, V17, V3
BC 18, 0, v3
LVX (R4+off64), V4
LVX (R3+off64), V16
VPMSUMW V4, V16, V4
BC 18, 0, v4
LVX (R4+off80), V5
LVX (R3+off80), V17
VPMSUMW V5, V17, V5
BC 18, 0, v5
LVX (R4+off96), V6
LVX (R3+off96), V16
VPMSUMW V6, V16, V6
BC 18, 0, v6
LVX (R4+off112), V7
LVX (R3+off112), V17
VPMSUMW V7, V17, V7
BC 18, 0, v7
ADD $128, R3
ADD $128, R4
LVX (R4), V8
LVX (R3), V16
VPMSUMW V8, V16, V8
BC 18, 0, v8
LVX (R4+off16), V9
LVX (R3+off16), V17
VPMSUMW V9, V17, V9
BC 18, 0, v9
LVX (R4+off32), V10
LVX (R3+off32), V16
VPMSUMW V10, V16, V10
BC 18, 0, v10
LVX (R4+off48), V11
LVX (R3+off48), V17
VPMSUMW V11, V17, V11
BC 18, 0, v11
LVX (R4+off64), V12
LVX (R3+off64), V16
VPMSUMW V12, V16, V12
BC 18, 0, v12
LVX (R4+off80), V13
LVX (R3+off80), V17
VPMSUMW V13, V17, V13
BC 18, 0, v13
LVX (R4+off96), V14
LVX (R3+off96), V16
VPMSUMW V14, V16, V14
BC 18, 0, v14
LVX (R4+off112), V15
LVX (R3+off112), V17
VPMSUMW V15, V17, V15
VXOR V19, V15, V19
v14:
VXOR V20, V14, V20
v13:
VXOR V19, V13, V19
v12:
VXOR V20, V12, V20
v11:
VXOR V19, V11, V19
v10:
VXOR V20, V10, V20
v9:
VXOR V19, V9, V19
v8:
VXOR V20, V8, V20
v7:
VXOR V19, V7, V19
v6:
VXOR V20, V6, V20
v5:
VXOR V19, V5, V19
v4:
VXOR V20, V4, V20
v3:
VXOR V19, V3, V19
v2:
VXOR V20, V2, V20
v1:
VXOR V19, V1, V19
v0:
VXOR V20, V0, V20
VXOR V19, V20, V0
BR barrett_reduction
zero:
// This case is the original crc, so just return it
MOVW R10, ret+32(FP)
RET
Reference in New Issue View Git Blame Copy Permalink