xxhash_arm64.s

// +build gc,!purego,!noasm

#include "textflag.h"

// Register allocation.
#define digest	R1
#define h	R2 // Return value.
#define p	R3 // Input pointer.
#define len	R4
#define nblocks	R5 // len / 32.
#define prime1	R7
#define prime2	R8
#define prime3	R9
#define prime4	R10
#define prime5	R11
#define v1	R12
#define v2	R13
#define v3	R14
#define v4	R15
#define x1	R20
#define x2	R21
#define x3	R22
#define x4	R23

#define round(acc, x) \
	MADD prime2, acc, x, acc \
	ROR  $64-31, acc         \
	MUL  prime1, acc         \

// x = round(0, x).
#define round0(x) \
	MUL prime2, x \
	ROR $64-31, x \
	MUL prime1, x \

#define mergeRound(x) \
	round0(x)                 \
	EOR  x, h                 \
	MADD h, prime4, prime1, h \

// Update v[1-4] with 32-byte blocks. Assumes len >= 32.
#define blocksLoop() \
	LSR     $5, len, nblocks \
	PCALIGN $16              \
	loop:                    \
	LDP.P   32(p), (x1, x2)  \
	round(v1, x1)            \
	LDP     -16(p), (x3, x4) \
	round(v2, x2)            \
	round(v3, x3)            \
	round(v4, x4)            \
	SUB     $1, nblocks      \
	CBNZ    nblocks, loop    \

// The primes are repeated here to ensure that they're stored
// in a contiguous array, so we can load them with LDP.
DATA primes<> +0(SB)/8, $11400714785074694791
DATA primes<> +8(SB)/8, $14029467366897019727
DATA primes<>+16(SB)/8, $1609587929392839161
DATA primes<>+24(SB)/8, $9650029242287828579
DATA primes<>+32(SB)/8, $2870177450012600261
GLOBL primes<>(SB), NOPTR+RODATA, $40

// func Sum64(b []byte) uint64
TEXT ·Sum64(SB), NOFRAME+NOSPLIT, $0-32
	LDP b_base+0(FP), (p, len)

	LDP  primes<> +0(SB), (prime1, prime2)
	LDP  primes<>+16(SB), (prime3, prime4)
	MOVD primes<>+32(SB), prime5

	CMP  $32, len
	CSEL LT, prime5, ZR, h // if len < 32 { h = prime5 } else { h = 0 }
	BLT  afterLoop

	ADD  prime1, prime2, v1
	MOVD prime2, v2
	MOVD $0, v3
	NEG  prime1, v4

	blocksLoop()

	ROR $64-1, v1, x1
	ROR $64-7, v2, x2
	ADD x1, x2
	ROR $64-12, v3, x3
	ROR $64-18, v4, x4
	ADD x3, x4
	ADD x2, x4, h

	mergeRound(v1)
	mergeRound(v2)
	mergeRound(v3)
	mergeRound(v4)

afterLoop:
	ADD len, h

	TBZ   $4, len, try8
	LDP.P 16(p), (x1, x2)

	round0(x1)
	ROR  $64-27, h
	EOR  x1 @> 64-27, h, h
	MADD h, prime4, prime1, h

	round0(x2)
	ROR  $64-27, h
	EOR  x2 @> 64-27, h
	MADD h, prime4, prime1, h

try8:
	TBZ    $3, len, try4
	MOVD.P 8(p), x1

	round0(x1)
	ROR  $64-27, h
	EOR  x1 @> 64-27, h
	MADD h, prime4, prime1, h

try4:
	TBZ     $2, len, try2
	MOVWU.P 4(p), x2

	MUL  prime1, x2
	ROR  $64-23, h
	EOR  x2 @> 64-23, h
	MADD h, prime3, prime2, h

try2:
	TBZ     $1, len, try1
	MOVHU.P 2(p), x3
	AND     $255, x3, x1
	LSR     $8, x3, x2

	MUL prime5, x1
	ROR $64-11, h
	EOR x1 @> 64-11, h
	MUL prime1, h

	MUL prime5, x2
	ROR $64-11, h
	EOR x2 @> 64-11, h
	MUL prime1, h

try1:
	TBZ   $0, len, end
	MOVBU (p), x4

	MUL prime5, x4
	ROR $64-11, h
	EOR x4 @> 64-11, h
	MUL prime1, h

end:
	EOR h >> 33, h
	MUL prime2, h
	EOR h >> 29, h
	MUL prime3, h
	EOR h >> 32, h

	MOVD h, ret+24(FP)
	RET

// func writeBlocks(d *Digest, b []byte) int
//
// Assumes len(b) >= 32.
TEXT ·writeBlocks(SB), NOFRAME+NOSPLIT, $0-40
	LDP primes<>(SB), (prime1, prime2)

	// Load state. Assume v[1-4] are stored contiguously.
	MOVD d+0(FP), digest
	LDP  0(digest), (v1, v2)
	LDP  16(digest), (v3, v4)

	LDP b_base+8(FP), (p, len)

	blocksLoop()

	// Store updated state.
	STP (v1, v2), 0(digest)
	STP (v3, v4), 16(digest)

	BIC  $31, len
	MOVD len, ret+32(FP)
	RET