From 637be3d60c561464a39297d1516b06652c1317ea Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Tue, 15 Sep 2026 18:15:41 +0200 Subject: [PATCH 1/2] vendor: github.com/klauspost/compress v1.20.0 - Add lzw reader/writer - ci: test against go1.27.x, drop go1.24.x - flate: Writer L7-9 with dictionary can emit the dictionary to stream - gzhttp: limit jitter hash to jitterBuffer bytes - s2: arm64 assembly encoders, lowered from the amd64 avo program - s2: clamp the Go EncodeBetter skip to match the assembly - xpress: add Microsoft XPRESS (MS-XCA) decompression - xxhash: bound input of non-preemptive assembly - zstd, huff0: improved arm64 lowering, and narrow two table loads to MOVWQZX - zstd/huff0: break false partial-register dependencies in seqdec/4X decoders - zstd/huff0: fold two ISA-gap x86 idioms in the avo arm64 lowering full diff: https://github.com/klauspost/compress/compare/v1.19.2...v1.20.0 Signed-off-by: Sebastiaan van Stijn --- vendor.mod | 2 +- vendor.sum | 4 +- .../github.com/klauspost/compress/README.md | 2 + .../compress/huff0/decompress_amd64.s | 206 ++--- .../compress/huff0/decompress_arm64.s | 249 +++--- .../compress/zstd/fse_decoder_arm64.s | 25 +- .../compress/zstd/internal/xxhash/xxhash.go | 12 + .../zstd/internal/xxhash/xxhash_amd64.s | 6 +- .../zstd/internal/xxhash/xxhash_arm64.s | 6 +- .../klauspost/compress/zstd/seqdec_amd64.s | 732 +++++++++--------- .../klauspost/compress/zstd/seqdec_arm64.s | 606 +++++++-------- vendor/modules.txt | 4 +- 12 files changed, 875 insertions(+), 979 deletions(-) diff --git a/vendor.mod b/vendor.mod index b1bfb7897ab7..e6f777058e62 100644 --- a/vendor.mod +++ b/vendor.mod @@ -87,7 +87,7 @@ require ( github.com/gorilla/mux v1.8.1 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.30.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect - github.com/klauspost/compress v1.19.2 // indirect + github.com/klauspost/compress v1.20.0 // indirect github.com/moby/docker-image-spec v1.3.1 // indirect github.com/moby/sys/user v0.4.1 // indirect github.com/moby/sys/userns v0.2.1 // indirect diff --git a/vendor.sum b/vendor.sum index 36d253337784..a02c98915b57 100644 --- a/vendor.sum +++ b/vendor.sum @@ -78,8 +78,8 @@ github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.19.2 h1:hMRETovs/pu/dVWN7zIT1PGG8t509MwT6bO7XSi26R8= -github.com/klauspost/compress v1.19.2/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= +github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA= +github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/mattn/go-runewidth v0.0.29 h1:3oGF3R/S2N9DQ3ptftzVIvg2eicmojCzlwBEmqEPDfQ= diff --git a/vendor/github.com/klauspost/compress/README.md b/vendor/github.com/klauspost/compress/README.md index 0e9f170d0120..bf31b905f81f 100644 --- a/vendor/github.com/klauspost/compress/README.md +++ b/vendor/github.com/klauspost/compress/README.md @@ -6,7 +6,9 @@ This package provides various compression algorithms. * [S2](https://github.com/klauspost/compress/tree/master/s2#s2-compression) is a high performance replacement for Snappy. * Optimized [deflate](https://godoc.org/github.com/klauspost/compress/flate) packages which can be used as a dropin replacement for [gzip](https://godoc.org/github.com/klauspost/compress/gzip), [zip](https://godoc.org/github.com/klauspost/compress/zip) and [zlib](https://godoc.org/github.com/klauspost/compress/zlib). * [snappy](https://github.com/klauspost/compress/tree/master/snappy) is a drop-in replacement for `github.com/golang/snappy` offering better compression and concurrent streams. +* [lzw](https://github.com/klauspost/compress/tree/master/lzw) is a drop-in replacement for `compress/lzw` with 1.4-4x faster decompression and 1.1-2.7x faster compression, depending on the data. * [huff0](https://github.com/klauspost/compress/tree/master/huff0) and [FSE](https://github.com/klauspost/compress/tree/master/fse) implementations for raw entropy encoding. +* [Xpress](https://github.com/klauspost/compress/tree/master/xpress) decompression of the Microsoft XPRESS (MS-XCA) plain LZ77 and LZ77+Huffman formats (the LZ77+Huffman variant is the one used in WIM images and Windows Compact OS / WOF data). * [gzhttp](https://github.com/klauspost/compress/tree/master/gzhttp) Provides client and server wrappers for handling gzipped/zstd HTTP requests efficiently. * [pgzip](https://github.com/klauspost/pgzip) is a separate package that provides a very fast parallel gzip implementation. diff --git a/vendor/github.com/klauspost/compress/huff0/decompress_amd64.s b/vendor/github.com/klauspost/compress/huff0/decompress_amd64.s index c5d4a710a093..920628c51422 100644 --- a/vendor/github.com/klauspost/compress/huff0/decompress_amd64.s +++ b/vendor/github.com/klauspost/compress/huff0/decompress_amd64.s @@ -47,12 +47,12 @@ skip_fill0: SHRQ CL, R13 // v0 := table[val0&mask] - MOVW (R9)(R13*2), CX + MOVWQZX (R9)(R13*2), CX // br0.advance(uint8(v0.entry) - MOVB CH, AL - SHLQ CL, R11 - ADDB CL, R12 + MOVBLZX CH, AX + SHLQ CL, R11 + ADDB CL, R12 // val1 := br0.peekTopBits(peekBits) MOVQ DI, CX @@ -60,7 +60,7 @@ skip_fill0: SHRQ CL, R13 // v1 := table[val1&mask] - MOVW (R9)(R13*2), CX + MOVWQZX (R9)(R13*2), CX // br0.advance(uint8(v1.entry)) MOVB CH, AH @@ -104,12 +104,12 @@ skip_fill1: SHRQ CL, R13 // v0 := table[val0&mask] - MOVW (R9)(R13*2), CX + MOVWQZX (R9)(R13*2), CX // br1.advance(uint8(v0.entry) - MOVB CH, AL - SHLQ CL, R11 - ADDB CL, R12 + MOVBLZX CH, AX + SHLQ CL, R11 + ADDB CL, R12 // val1 := br1.peekTopBits(peekBits) MOVQ DI, CX @@ -117,7 +117,7 @@ skip_fill1: SHRQ CL, R13 // v1 := table[val1&mask] - MOVW (R9)(R13*2), CX + MOVWQZX (R9)(R13*2), CX // br1.advance(uint8(v1.entry)) MOVB CH, AH @@ -161,12 +161,12 @@ skip_fill2: SHRQ CL, R13 // v0 := table[val0&mask] - MOVW (R9)(R13*2), CX + MOVWQZX (R9)(R13*2), CX // br2.advance(uint8(v0.entry) - MOVB CH, AL - SHLQ CL, R11 - ADDB CL, R12 + MOVBLZX CH, AX + SHLQ CL, R11 + ADDB CL, R12 // val1 := br2.peekTopBits(peekBits) MOVQ DI, CX @@ -174,7 +174,7 @@ skip_fill2: SHRQ CL, R13 // v1 := table[val1&mask] - MOVW (R9)(R13*2), CX + MOVWQZX (R9)(R13*2), CX // br2.advance(uint8(v1.entry)) MOVB CH, AH @@ -218,12 +218,12 @@ skip_fill3: SHRQ CL, R13 // v0 := table[val0&mask] - MOVW (R9)(R13*2), CX + MOVWQZX (R9)(R13*2), CX // br3.advance(uint8(v0.entry) - MOVB CH, AL - SHLQ CL, R11 - ADDB CL, R12 + MOVBLZX CH, AX + SHLQ CL, R11 + ADDB CL, R12 // val1 := br3.peekTopBits(peekBits) MOVQ DI, CX @@ -231,7 +231,7 @@ skip_fill3: SHRQ CL, R13 // v1 := table[val1&mask] - MOVW (R9)(R13*2), CX + MOVWQZX (R9)(R13*2), CX // br3.advance(uint8(v1.entry)) MOVB CH, AH @@ -259,13 +259,13 @@ skip_fill3: // func decompress4x_8b_main_loop_amd64(ctx *decompress4xContext) TEXT ·decompress4x_8b_main_loop_amd64(SB), $0-8 // Preload values - MOVQ ctx+0(FP), CX - MOVBQZX 8(CX), DI - MOVQ 16(CX), BX - MOVQ 48(CX), SI - MOVQ 24(CX), R8 - MOVQ 32(CX), R9 - MOVQ (CX), R10 + MOVQ ctx+0(FP), AX + MOVBQZX 8(AX), DI + MOVQ 16(AX), BX + MOVQ 48(AX), SI + MOVQ 24(AX), R8 + MOVQ 32(AX), R9 + MOVQ (AX), R10 // Main loop main_loop: @@ -278,35 +278,35 @@ main_loop: MOVBQZX 40(R10), R12 CMPQ R12, $0x20 JBE skip_fill0 - MOVQ 24(R10), R13 + MOVQ 24(R10), AX SUBQ $0x20, R12 - SUBQ $0x04, R13 - MOVQ (R10), R14 + SUBQ $0x04, AX + MOVQ (R10), R13 // b.value |= uint64(low) << (b.bitsRead & 63) - MOVL (R13)(R14*1), R14 + MOVL (AX)(R13*1), R13 MOVQ R12, CX - SHLQ CL, R14 - MOVQ R13, 24(R10) - ORQ R14, R11 + SHLQ CL, R13 + MOVQ AX, 24(R10) + ORQ R13, R11 // exhausted += (br0.off < 4) - CMPQ R13, $0x04 + CMPQ AX, $0x04 ADCB $+0, DL skip_fill0: // val0 := br0.peekTopBits(peekBits) - MOVQ R11, R13 + MOVQ R11, AX MOVQ DI, CX - SHRQ CL, R13 + SHRQ CL, AX // v0 := table[val0&mask] - MOVW (R9)(R13*2), CX + MOVWQZX (R9)(AX*2), CX // br0.advance(uint8(v0.entry) - MOVB CH, AL - SHLQ CL, R11 - ADDB CL, R12 + MOVBLZX CH, AX + SHLQ CL, R11 + ADDB CL, R12 // val1 := br0.peekTopBits(peekBits) MOVQ R11, R13 @@ -314,7 +314,7 @@ skip_fill0: SHRQ CL, R13 // v1 := table[val0&mask] - MOVW (R9)(R13*2), CX + MOVWQZX (R9)(R13*2), CX // br0.advance(uint8(v1.entry) MOVB CH, AH @@ -328,7 +328,7 @@ skip_fill0: SHRQ CL, R13 // v2 := table[val0&mask] - MOVW (R9)(R13*2), CX + MOVWQZX (R9)(R13*2), CX // br0.advance(uint8(v2.entry) MOVB CH, AH @@ -341,7 +341,7 @@ skip_fill0: SHRQ CL, R13 // v3 := table[val0&mask] - MOVW (R9)(R13*2), CX + MOVWQZX (R9)(R13*2), CX // br0.advance(uint8(v3.entry) MOVB CH, AL @@ -365,35 +365,35 @@ skip_fill0: MOVBQZX 88(R10), R12 CMPQ R12, $0x20 JBE skip_fill1 - MOVQ 72(R10), R13 + MOVQ 72(R10), AX SUBQ $0x20, R12 - SUBQ $0x04, R13 - MOVQ 48(R10), R14 + SUBQ $0x04, AX + MOVQ 48(R10), R13 // b.value |= uint64(low) << (b.bitsRead & 63) - MOVL (R13)(R14*1), R14 + MOVL (AX)(R13*1), R13 MOVQ R12, CX - SHLQ CL, R14 - MOVQ R13, 72(R10) - ORQ R14, R11 + SHLQ CL, R13 + MOVQ AX, 72(R10) + ORQ R13, R11 // exhausted += (br1.off < 4) - CMPQ R13, $0x04 + CMPQ AX, $0x04 ADCB $+0, DL skip_fill1: // val0 := br1.peekTopBits(peekBits) - MOVQ R11, R13 + MOVQ R11, AX MOVQ DI, CX - SHRQ CL, R13 + SHRQ CL, AX // v0 := table[val0&mask] - MOVW (R9)(R13*2), CX + MOVWQZX (R9)(AX*2), CX // br1.advance(uint8(v0.entry) - MOVB CH, AL - SHLQ CL, R11 - ADDB CL, R12 + MOVBLZX CH, AX + SHLQ CL, R11 + ADDB CL, R12 // val1 := br1.peekTopBits(peekBits) MOVQ R11, R13 @@ -401,7 +401,7 @@ skip_fill1: SHRQ CL, R13 // v1 := table[val0&mask] - MOVW (R9)(R13*2), CX + MOVWQZX (R9)(R13*2), CX // br1.advance(uint8(v1.entry) MOVB CH, AH @@ -415,7 +415,7 @@ skip_fill1: SHRQ CL, R13 // v2 := table[val0&mask] - MOVW (R9)(R13*2), CX + MOVWQZX (R9)(R13*2), CX // br1.advance(uint8(v2.entry) MOVB CH, AH @@ -428,7 +428,7 @@ skip_fill1: SHRQ CL, R13 // v3 := table[val0&mask] - MOVW (R9)(R13*2), CX + MOVWQZX (R9)(R13*2), CX // br1.advance(uint8(v3.entry) MOVB CH, AL @@ -452,35 +452,35 @@ skip_fill1: MOVBQZX 136(R10), R12 CMPQ R12, $0x20 JBE skip_fill2 - MOVQ 120(R10), R13 + MOVQ 120(R10), AX SUBQ $0x20, R12 - SUBQ $0x04, R13 - MOVQ 96(R10), R14 + SUBQ $0x04, AX + MOVQ 96(R10), R13 // b.value |= uint64(low) << (b.bitsRead & 63) - MOVL (R13)(R14*1), R14 + MOVL (AX)(R13*1), R13 MOVQ R12, CX - SHLQ CL, R14 - MOVQ R13, 120(R10) - ORQ R14, R11 + SHLQ CL, R13 + MOVQ AX, 120(R10) + ORQ R13, R11 // exhausted += (br2.off < 4) - CMPQ R13, $0x04 + CMPQ AX, $0x04 ADCB $+0, DL skip_fill2: // val0 := br2.peekTopBits(peekBits) - MOVQ R11, R13 + MOVQ R11, AX MOVQ DI, CX - SHRQ CL, R13 + SHRQ CL, AX // v0 := table[val0&mask] - MOVW (R9)(R13*2), CX + MOVWQZX (R9)(AX*2), CX // br2.advance(uint8(v0.entry) - MOVB CH, AL - SHLQ CL, R11 - ADDB CL, R12 + MOVBLZX CH, AX + SHLQ CL, R11 + ADDB CL, R12 // val1 := br2.peekTopBits(peekBits) MOVQ R11, R13 @@ -488,7 +488,7 @@ skip_fill2: SHRQ CL, R13 // v1 := table[val0&mask] - MOVW (R9)(R13*2), CX + MOVWQZX (R9)(R13*2), CX // br2.advance(uint8(v1.entry) MOVB CH, AH @@ -502,7 +502,7 @@ skip_fill2: SHRQ CL, R13 // v2 := table[val0&mask] - MOVW (R9)(R13*2), CX + MOVWQZX (R9)(R13*2), CX // br2.advance(uint8(v2.entry) MOVB CH, AH @@ -515,7 +515,7 @@ skip_fill2: SHRQ CL, R13 // v3 := table[val0&mask] - MOVW (R9)(R13*2), CX + MOVWQZX (R9)(R13*2), CX // br2.advance(uint8(v3.entry) MOVB CH, AL @@ -539,35 +539,35 @@ skip_fill2: MOVBQZX 184(R10), R12 CMPQ R12, $0x20 JBE skip_fill3 - MOVQ 168(R10), R13 + MOVQ 168(R10), AX SUBQ $0x20, R12 - SUBQ $0x04, R13 - MOVQ 144(R10), R14 + SUBQ $0x04, AX + MOVQ 144(R10), R13 // b.value |= uint64(low) << (b.bitsRead & 63) - MOVL (R13)(R14*1), R14 + MOVL (AX)(R13*1), R13 MOVQ R12, CX - SHLQ CL, R14 - MOVQ R13, 168(R10) - ORQ R14, R11 + SHLQ CL, R13 + MOVQ AX, 168(R10) + ORQ R13, R11 // exhausted += (br3.off < 4) - CMPQ R13, $0x04 + CMPQ AX, $0x04 ADCB $+0, DL skip_fill3: // val0 := br3.peekTopBits(peekBits) - MOVQ R11, R13 + MOVQ R11, AX MOVQ DI, CX - SHRQ CL, R13 + SHRQ CL, AX // v0 := table[val0&mask] - MOVW (R9)(R13*2), CX + MOVWQZX (R9)(AX*2), CX // br3.advance(uint8(v0.entry) - MOVB CH, AL - SHLQ CL, R11 - ADDB CL, R12 + MOVBLZX CH, AX + SHLQ CL, R11 + ADDB CL, R12 // val1 := br3.peekTopBits(peekBits) MOVQ R11, R13 @@ -575,7 +575,7 @@ skip_fill3: SHRQ CL, R13 // v1 := table[val0&mask] - MOVW (R9)(R13*2), CX + MOVWQZX (R9)(R13*2), CX // br3.advance(uint8(v1.entry) MOVB CH, AH @@ -589,7 +589,7 @@ skip_fill3: SHRQ CL, R13 // v2 := table[val0&mask] - MOVW (R9)(R13*2), CX + MOVWQZX (R9)(R13*2), CX // br3.advance(uint8(v2.entry) MOVB CH, AH @@ -602,7 +602,7 @@ skip_fill3: SHRQ CL, R13 // v3 := table[val0&mask] - MOVW (R9)(R13*2), CX + MOVWQZX (R9)(R13*2), CX // br3.advance(uint8(v3.entry) MOVB CH, AL @@ -667,7 +667,7 @@ bitReader_fillFast_1_end: MOVQ DI, CX MOVQ R10, R12 SHRQ CL, R12 - MOVW (SI)(R12*2), CX + MOVWQZX (SI)(R12*2), CX MOVB CH, AL MOVBQZX CL, CX ADDQ CX, R11 @@ -675,7 +675,7 @@ bitReader_fillFast_1_end: MOVQ DI, CX MOVQ R10, R12 SHRQ CL, R12 - MOVW (SI)(R12*2), CX + MOVWQZX (SI)(R12*2), CX MOVB CH, AH MOVBQZX CL, CX ADDQ CX, R11 @@ -694,7 +694,7 @@ bitReader_fillFast_2_end: MOVQ DI, CX MOVQ R10, R12 SHRQ CL, R12 - MOVW (SI)(R12*2), CX + MOVWQZX (SI)(R12*2), CX MOVB CH, AH MOVBQZX CL, CX ADDQ CX, R11 @@ -702,7 +702,7 @@ bitReader_fillFast_2_end: MOVQ DI, CX MOVQ R10, R12 SHRQ CL, R12 - MOVW (SI)(R12*2), CX + MOVWQZX (SI)(R12*2), CX MOVB CH, AL MOVBQZX CL, CX ADDQ CX, R11 @@ -769,13 +769,13 @@ main_loop: bitReader_fillFast_1_end: SHRXQ DI, R10, CX - MOVW (SI)(CX*2), CX + MOVWQZX (SI)(CX*2), CX MOVB CH, AL MOVBQZX CL, CX ADDQ CX, R11 SHLXQ CX, R10, R10 SHRXQ DI, R10, CX - MOVW (SI)(CX*2), CX + MOVWQZX (SI)(CX*2), CX MOVB CH, AH MOVBQZX CL, CX ADDQ CX, R11 @@ -791,13 +791,13 @@ bitReader_fillFast_1_end: bitReader_fillFast_2_end: SHRXQ DI, R10, CX - MOVW (SI)(CX*2), CX + MOVWQZX (SI)(CX*2), CX MOVB CH, AH MOVBQZX CL, CX ADDQ CX, R11 SHLXQ CX, R10, R10 SHRXQ DI, R10, CX - MOVW (SI)(CX*2), CX + MOVWQZX (SI)(CX*2), CX MOVB CH, AL MOVBQZX CL, CX ADDQ CX, R11 diff --git a/vendor/github.com/klauspost/compress/huff0/decompress_arm64.s b/vendor/github.com/klauspost/compress/huff0/decompress_arm64.s index e16012721a1c..357df28e9d6c 100644 --- a/vendor/github.com/klauspost/compress/huff0/decompress_arm64.s +++ b/vendor/github.com/klauspost/compress/huff0/decompress_arm64.s @@ -1,7 +1,7 @@ // Code generated by command: go run gen.go -out ../decompress.s -arch amd64,arm64 -pkg=huff0. DO NOT EDIT. // EXPERIMENTAL arm64 output lowered from an amd64 avo program. -//go:build arm64 && !appengine && !noasm && gc +//go:build arm64 && (!appengine && !noasm && gc) // func decompress4x_main_loop_amd64(ctx *decompress4xContext) TEXT ·decompress4x_main_loop_arm64(SB), $0-8 @@ -32,8 +32,7 @@ main_loop: MOVD (R9), R12 // b.value |= uint64(low) << (b.bitsRead & 63) - ADD R12, R0, R15 - MOVWU (R15), R12 + MOVWU (R0)(R12), R12 MOVD R11, R1 LSL R1, R12, R12 MOVD R0, 24(R9) @@ -51,12 +50,10 @@ skip_fill0: LSR R1, R12, R12 // v0 := table[val0&mask] - ADD R12<<1, R8, R15 - MOVHU (R15), R1 + MOVHU (R8)(R12<<1), R1 // br0.advance(uint8(v0.entry) - UBFX $8, R1, $8, R16 - BFI $0, R16, $8, R0 + UBFX $8, R1, $8, R0 LSL R1, R10, R10 ADD R1, R11, R15 BFI $0, R15, $8, R11 @@ -67,8 +64,7 @@ skip_fill0: LSR R1, R12, R12 // v1 := table[val1&mask] - ADD R12<<1, R8, R15 - MOVHU (R15), R1 + MOVHU (R8)(R12<<1), R1 // br0.advance(uint8(v1.entry)) UBFX $8, R1, $8, R16 @@ -97,8 +93,7 @@ skip_fill0: MOVD 48(R9), R12 // b.value |= uint64(low) << (b.bitsRead & 63) - ADD R12, R0, R15 - MOVWU (R15), R12 + MOVWU (R0)(R12), R12 MOVD R11, R1 LSL R1, R12, R12 MOVD R0, 72(R9) @@ -116,12 +111,10 @@ skip_fill1: LSR R1, R12, R12 // v0 := table[val0&mask] - ADD R12<<1, R8, R15 - MOVHU (R15), R1 + MOVHU (R8)(R12<<1), R1 // br1.advance(uint8(v0.entry) - UBFX $8, R1, $8, R16 - BFI $0, R16, $8, R0 + UBFX $8, R1, $8, R0 LSL R1, R10, R10 ADD R1, R11, R15 BFI $0, R15, $8, R11 @@ -132,8 +125,7 @@ skip_fill1: LSR R1, R12, R12 // v1 := table[val1&mask] - ADD R12<<1, R8, R15 - MOVHU (R15), R1 + MOVHU (R8)(R12<<1), R1 // br1.advance(uint8(v1.entry)) UBFX $8, R1, $8, R16 @@ -145,8 +137,7 @@ skip_fill1: // these two writes get coalesced // out[id * dstEvery + 0] = uint8(v0.entry >> 8) // out[id * dstEvery + 1] = uint8(v1.entry >> 8) - ADD R7, R3, R15 - MOVH R0, (R15) + MOVH R0, (R3)(R7) // update the bitreader structure MOVD R10, 80(R9) @@ -163,8 +154,7 @@ skip_fill1: MOVD 96(R9), R12 // b.value |= uint64(low) << (b.bitsRead & 63) - ADD R12, R0, R15 - MOVWU (R15), R12 + MOVWU (R0)(R12), R12 MOVD R11, R1 LSL R1, R12, R12 MOVD R0, 120(R9) @@ -182,12 +172,10 @@ skip_fill2: LSR R1, R12, R12 // v0 := table[val0&mask] - ADD R12<<1, R8, R15 - MOVHU (R15), R1 + MOVHU (R8)(R12<<1), R1 // br2.advance(uint8(v0.entry) - UBFX $8, R1, $8, R16 - BFI $0, R16, $8, R0 + UBFX $8, R1, $8, R0 LSL R1, R10, R10 ADD R1, R11, R15 BFI $0, R15, $8, R11 @@ -198,8 +186,7 @@ skip_fill2: LSR R1, R12, R12 // v1 := table[val1&mask] - ADD R12<<1, R8, R15 - MOVHU (R15), R1 + MOVHU (R8)(R12<<1), R1 // br2.advance(uint8(v1.entry)) UBFX $8, R1, $8, R16 @@ -211,8 +198,7 @@ skip_fill2: // these two writes get coalesced // out[id * dstEvery + 0] = uint8(v0.entry >> 8) // out[id * dstEvery + 1] = uint8(v1.entry >> 8) - ADD R7<<1, R3, R15 - MOVH R0, (R15) + MOVH R0, (R3)(R7<<1) // update the bitreader structure MOVD R10, 128(R9) @@ -229,8 +215,7 @@ skip_fill2: MOVD 144(R9), R12 // b.value |= uint64(low) << (b.bitsRead & 63) - ADD R12, R0, R15 - MOVWU (R15), R12 + MOVWU (R0)(R12), R12 MOVD R11, R1 LSL R1, R12, R12 MOVD R0, 168(R9) @@ -248,12 +233,10 @@ skip_fill3: LSR R1, R12, R12 // v0 := table[val0&mask] - ADD R12<<1, R8, R15 - MOVHU (R15), R1 + MOVHU (R8)(R12<<1), R1 // br3.advance(uint8(v0.entry) - UBFX $8, R1, $8, R16 - BFI $0, R16, $8, R0 + UBFX $8, R1, $8, R0 LSL R1, R10, R10 ADD R1, R11, R15 BFI $0, R15, $8, R11 @@ -264,8 +247,7 @@ skip_fill3: LSR R1, R12, R12 // v1 := table[val1&mask] - ADD R12<<1, R8, R15 - MOVHU (R15), R1 + MOVHU (R8)(R12<<1), R1 // br3.advance(uint8(v1.entry)) UBFX $8, R1, $8, R16 @@ -278,16 +260,13 @@ skip_fill3: // out[id * dstEvery + 0] = uint8(v0.entry >> 8) // out[id * dstEvery + 1] = uint8(v1.entry >> 8) ADD R7<<1, R7, R1 - ADD R1, R3, R15 - MOVH R0, (R15) + MOVH R0, (R3)(R1) // update the bitreader structure MOVD R10, 176(R9) MOVB R11, 184(R9) ADD $0x02, R3, R3 - AND $0xff, R2, R15 - AND $0xff, R2, R16 - TST R16, R15 + TST $0xff, R2 BEQ main_loop MOVD ctx+0(FP), R0 MOVD 16(R0), R16 @@ -299,13 +278,13 @@ skip_fill3: // func decompress4x_8b_main_loop_amd64(ctx *decompress4xContext) TEXT ·decompress4x_8b_main_loop_arm64(SB), $0-8 // Preload values - MOVD ctx+0(FP), R1 - MOVBU 8(R1), R6 - MOVD 16(R1), R3 - MOVD 48(R1), R5 - MOVD 24(R1), R7 - MOVD 32(R1), R8 - MOVD (R1), R9 + MOVD ctx+0(FP), R0 + MOVBU 8(R0), R6 + MOVD 16(R0), R3 + MOVD 48(R0), R5 + MOVD 24(R0), R7 + MOVD 32(R0), R8 + MOVD (R0), R9 // Main loop main_loop: @@ -319,37 +298,34 @@ main_loop: MOVBU 40(R9), R11 CMP $0x20, R11 BLS skip_fill0 - MOVD 24(R9), R12 + MOVD 24(R9), R0 SUB $0x20, R11, R11 - SUB $0x04, R12, R12 - MOVD (R9), R13 + SUB $0x04, R0, R0 + MOVD (R9), R12 // b.value |= uint64(low) << (b.bitsRead & 63) - ADD R13, R12, R15 - MOVWU (R15), R13 + MOVWU (R0)(R12), R12 MOVD R11, R1 - LSL R1, R13, R13 - MOVD R12, 24(R9) - ORR R13, R10, R10 + LSL R1, R12, R12 + MOVD R0, 24(R9) + ORR R12, R10, R10 // exhausted += (br0.off < 4) - CMP $0x04, R12 + CMP $0x04, R0 CSINC HS, R2, R2, R16 BFI $0, R16, $8, R2 skip_fill0: // val0 := br0.peekTopBits(peekBits) - MOVD R10, R12 + MOVD R10, R0 MOVD R6, R1 - LSR R1, R12, R12 + LSR R1, R0, R0 // v0 := table[val0&mask] - ADD R12<<1, R8, R15 - MOVHU (R15), R1 + MOVHU (R8)(R0<<1), R1 // br0.advance(uint8(v0.entry) - UBFX $8, R1, $8, R16 - BFI $0, R16, $8, R0 + UBFX $8, R1, $8, R0 LSL R1, R10, R10 ADD R1, R11, R15 BFI $0, R15, $8, R11 @@ -360,8 +336,7 @@ skip_fill0: LSR R1, R12, R12 // v1 := table[val0&mask] - ADD R12<<1, R8, R15 - MOVHU (R15), R1 + MOVHU (R8)(R12<<1), R1 // br0.advance(uint8(v1.entry) UBFX $8, R1, $8, R16 @@ -377,8 +352,7 @@ skip_fill0: LSR R1, R12, R12 // v2 := table[val0&mask] - ADD R12<<1, R8, R15 - MOVHU (R15), R1 + MOVHU (R8)(R12<<1), R1 // br0.advance(uint8(v2.entry) UBFX $8, R1, $8, R16 @@ -393,8 +367,7 @@ skip_fill0: LSR R1, R12, R12 // v3 := table[val0&mask] - ADD R12<<1, R8, R15 - MOVHU (R15), R1 + MOVHU (R8)(R12<<1), R1 // br0.advance(uint8(v3.entry) UBFX $8, R1, $8, R16 @@ -420,37 +393,34 @@ skip_fill0: MOVBU 88(R9), R11 CMP $0x20, R11 BLS skip_fill1 - MOVD 72(R9), R12 + MOVD 72(R9), R0 SUB $0x20, R11, R11 - SUB $0x04, R12, R12 - MOVD 48(R9), R13 + SUB $0x04, R0, R0 + MOVD 48(R9), R12 // b.value |= uint64(low) << (b.bitsRead & 63) - ADD R13, R12, R15 - MOVWU (R15), R13 + MOVWU (R0)(R12), R12 MOVD R11, R1 - LSL R1, R13, R13 - MOVD R12, 72(R9) - ORR R13, R10, R10 + LSL R1, R12, R12 + MOVD R0, 72(R9) + ORR R12, R10, R10 // exhausted += (br1.off < 4) - CMP $0x04, R12 + CMP $0x04, R0 CSINC HS, R2, R2, R16 BFI $0, R16, $8, R2 skip_fill1: // val0 := br1.peekTopBits(peekBits) - MOVD R10, R12 + MOVD R10, R0 MOVD R6, R1 - LSR R1, R12, R12 + LSR R1, R0, R0 // v0 := table[val0&mask] - ADD R12<<1, R8, R15 - MOVHU (R15), R1 + MOVHU (R8)(R0<<1), R1 // br1.advance(uint8(v0.entry) - UBFX $8, R1, $8, R16 - BFI $0, R16, $8, R0 + UBFX $8, R1, $8, R0 LSL R1, R10, R10 ADD R1, R11, R15 BFI $0, R15, $8, R11 @@ -461,8 +431,7 @@ skip_fill1: LSR R1, R12, R12 // v1 := table[val0&mask] - ADD R12<<1, R8, R15 - MOVHU (R15), R1 + MOVHU (R8)(R12<<1), R1 // br1.advance(uint8(v1.entry) UBFX $8, R1, $8, R16 @@ -478,8 +447,7 @@ skip_fill1: LSR R1, R12, R12 // v2 := table[val0&mask] - ADD R12<<1, R8, R15 - MOVHU (R15), R1 + MOVHU (R8)(R12<<1), R1 // br1.advance(uint8(v2.entry) UBFX $8, R1, $8, R16 @@ -494,8 +462,7 @@ skip_fill1: LSR R1, R12, R12 // v3 := table[val0&mask] - ADD R12<<1, R8, R15 - MOVHU (R15), R1 + MOVHU (R8)(R12<<1), R1 // br1.advance(uint8(v3.entry) UBFX $8, R1, $8, R16 @@ -510,8 +477,7 @@ skip_fill1: // out[id * dstEvery + 1] = uint8(v1.entry >> 8) // out[id * dstEvery + 3] = uint8(v2.entry >> 8) // out[id * dstEvery + 4] = uint8(v3.entry >> 8) - ADD R7, R3, R15 - MOVW R0, (R15) + MOVW R0, (R3)(R7) // update the bitreader structure MOVD R10, 80(R9) @@ -522,37 +488,34 @@ skip_fill1: MOVBU 136(R9), R11 CMP $0x20, R11 BLS skip_fill2 - MOVD 120(R9), R12 + MOVD 120(R9), R0 SUB $0x20, R11, R11 - SUB $0x04, R12, R12 - MOVD 96(R9), R13 + SUB $0x04, R0, R0 + MOVD 96(R9), R12 // b.value |= uint64(low) << (b.bitsRead & 63) - ADD R13, R12, R15 - MOVWU (R15), R13 + MOVWU (R0)(R12), R12 MOVD R11, R1 - LSL R1, R13, R13 - MOVD R12, 120(R9) - ORR R13, R10, R10 + LSL R1, R12, R12 + MOVD R0, 120(R9) + ORR R12, R10, R10 // exhausted += (br2.off < 4) - CMP $0x04, R12 + CMP $0x04, R0 CSINC HS, R2, R2, R16 BFI $0, R16, $8, R2 skip_fill2: // val0 := br2.peekTopBits(peekBits) - MOVD R10, R12 + MOVD R10, R0 MOVD R6, R1 - LSR R1, R12, R12 + LSR R1, R0, R0 // v0 := table[val0&mask] - ADD R12<<1, R8, R15 - MOVHU (R15), R1 + MOVHU (R8)(R0<<1), R1 // br2.advance(uint8(v0.entry) - UBFX $8, R1, $8, R16 - BFI $0, R16, $8, R0 + UBFX $8, R1, $8, R0 LSL R1, R10, R10 ADD R1, R11, R15 BFI $0, R15, $8, R11 @@ -563,8 +526,7 @@ skip_fill2: LSR R1, R12, R12 // v1 := table[val0&mask] - ADD R12<<1, R8, R15 - MOVHU (R15), R1 + MOVHU (R8)(R12<<1), R1 // br2.advance(uint8(v1.entry) UBFX $8, R1, $8, R16 @@ -580,8 +542,7 @@ skip_fill2: LSR R1, R12, R12 // v2 := table[val0&mask] - ADD R12<<1, R8, R15 - MOVHU (R15), R1 + MOVHU (R8)(R12<<1), R1 // br2.advance(uint8(v2.entry) UBFX $8, R1, $8, R16 @@ -596,8 +557,7 @@ skip_fill2: LSR R1, R12, R12 // v3 := table[val0&mask] - ADD R12<<1, R8, R15 - MOVHU (R15), R1 + MOVHU (R8)(R12<<1), R1 // br2.advance(uint8(v3.entry) UBFX $8, R1, $8, R16 @@ -624,37 +584,34 @@ skip_fill2: MOVBU 184(R9), R11 CMP $0x20, R11 BLS skip_fill3 - MOVD 168(R9), R12 + MOVD 168(R9), R0 SUB $0x20, R11, R11 - SUB $0x04, R12, R12 - MOVD 144(R9), R13 + SUB $0x04, R0, R0 + MOVD 144(R9), R12 // b.value |= uint64(low) << (b.bitsRead & 63) - ADD R13, R12, R15 - MOVWU (R15), R13 + MOVWU (R0)(R12), R12 MOVD R11, R1 - LSL R1, R13, R13 - MOVD R12, 168(R9) - ORR R13, R10, R10 + LSL R1, R12, R12 + MOVD R0, 168(R9) + ORR R12, R10, R10 // exhausted += (br3.off < 4) - CMP $0x04, R12 + CMP $0x04, R0 CSINC HS, R2, R2, R16 BFI $0, R16, $8, R2 skip_fill3: // val0 := br3.peekTopBits(peekBits) - MOVD R10, R12 + MOVD R10, R0 MOVD R6, R1 - LSR R1, R12, R12 + LSR R1, R0, R0 // v0 := table[val0&mask] - ADD R12<<1, R8, R15 - MOVHU (R15), R1 + MOVHU (R8)(R0<<1), R1 // br3.advance(uint8(v0.entry) - UBFX $8, R1, $8, R16 - BFI $0, R16, $8, R0 + UBFX $8, R1, $8, R0 LSL R1, R10, R10 ADD R1, R11, R15 BFI $0, R15, $8, R11 @@ -665,8 +622,7 @@ skip_fill3: LSR R1, R12, R12 // v1 := table[val0&mask] - ADD R12<<1, R8, R15 - MOVHU (R15), R1 + MOVHU (R8)(R12<<1), R1 // br3.advance(uint8(v1.entry) UBFX $8, R1, $8, R16 @@ -682,8 +638,7 @@ skip_fill3: LSR R1, R12, R12 // v2 := table[val0&mask] - ADD R12<<1, R8, R15 - MOVHU (R15), R1 + MOVHU (R8)(R12<<1), R1 // br3.advance(uint8(v2.entry) UBFX $8, R1, $8, R16 @@ -698,8 +653,7 @@ skip_fill3: LSR R1, R12, R12 // v3 := table[val0&mask] - ADD R12<<1, R8, R15 - MOVHU (R15), R1 + MOVHU (R8)(R12<<1), R1 // br3.advance(uint8(v3.entry) UBFX $8, R1, $8, R16 @@ -715,16 +669,13 @@ skip_fill3: // out[id * dstEvery + 3] = uint8(v2.entry >> 8) // out[id * dstEvery + 4] = uint8(v3.entry >> 8) ADD R7<<1, R7, R1 - ADD R1, R3, R15 - MOVW R0, (R15) + MOVW R0, (R3)(R1) // update the bitreader structure MOVD R10, 176(R9) MOVB R11, 184(R9) ADD $0x04, R3, R3 - AND $0xff, R2, R15 - AND $0xff, R2, R16 - TST R16, R15 + TST $0xff, R2 BEQ main_loop MOVD ctx+0(FP), R0 MOVD 16(R0), R16 @@ -761,8 +712,7 @@ main_loop: BLT bitReader_fillFast_1_end SUB $0x20, R10, R10 SUB $0x04, R8, R8 - ADD R8, R7, R15 - MOVWU (R15), R11 + MOVWU (R7)(R8), R11 MOVD R10, R1 LSL R1, R11, R11 ORR R11, R9, R9 @@ -771,8 +721,7 @@ bitReader_fillFast_1_end: MOVD R6, R1 MOVD R9, R11 LSR R1, R11, R11 - ADD R11<<1, R5, R15 - MOVHU (R15), R1 + MOVHU (R5)(R11<<1), R1 UBFX $8, R1, $8, R16 BFI $0, R16, $8, R0 MOVBU R1, R1 @@ -781,8 +730,7 @@ bitReader_fillFast_1_end: MOVD R6, R1 MOVD R9, R11 LSR R1, R11, R11 - ADD R11<<1, R5, R15 - MOVHU (R15), R1 + MOVHU (R5)(R11<<1), R1 UBFX $8, R1, $8, R16 BFI $8, R16, $8, R0 MOVBU R1, R1 @@ -793,8 +741,7 @@ bitReader_fillFast_1_end: BLT bitReader_fillFast_2_end SUB $0x20, R10, R10 SUB $0x04, R8, R8 - ADD R8, R7, R15 - MOVWU (R15), R11 + MOVWU (R7)(R8), R11 MOVD R10, R1 LSL R1, R11, R11 ORR R11, R9, R9 @@ -803,8 +750,7 @@ bitReader_fillFast_2_end: MOVD R6, R1 MOVD R9, R11 LSR R1, R11, R11 - ADD R11<<1, R5, R15 - MOVHU (R15), R1 + MOVHU (R5)(R11<<1), R1 UBFX $8, R1, $8, R16 BFI $8, R16, $8, R0 MOVBU R1, R1 @@ -813,8 +759,7 @@ bitReader_fillFast_2_end: MOVD R6, R1 MOVD R9, R11 LSR R1, R11, R11 - ADD R11<<1, R5, R15 - MOVHU (R15), R1 + MOVHU (R5)(R11<<1), R1 UBFX $8, R1, $8, R16 BFI $0, R16, $8, R0 MOVBU R1, R1 diff --git a/vendor/github.com/klauspost/compress/zstd/fse_decoder_arm64.s b/vendor/github.com/klauspost/compress/zstd/fse_decoder_arm64.s index 77ee3913f056..b6b460787803 100644 --- a/vendor/github.com/klauspost/compress/zstd/fse_decoder_arm64.s +++ b/vendor/github.com/klauspost/compress/zstd/fse_decoder_arm64.s @@ -1,7 +1,7 @@ // Code generated by command: go run gen_fse.go -out ../fse_decoder.s -arch amd64,arm64 -pkg=zstd. DO NOT EDIT. // EXPERIMENTAL arm64 output lowered from an amd64 avo program. -//go:build arm64 && !appengine && !noasm && gc && !noasm +//go:build arm64 && (!appengine && !noasm && gc && !noasm) // func buildDtable_asm(s *fseDecoder, ctx *buildDtableAsmContext) int TEXT ·buildDtable_asm(SB), $0-24 @@ -26,11 +26,10 @@ TEXT ·buildDtable_asm(SB), $0-24 JMP init_main_loop_condition init_main_loop: - ADD R8<<1, R1, R15 - MOVH (R15), R9 - AND $0xffff, R9, R15 + MOVH (R1)(R8<<1), R9 MOVD $-1, R16 AND $0xffff, R16, R16 + AND $0xffff, R9, R15 CMP R16, R15 BNE do_not_update_high_threshold ADD R7<<3, R5, R15 @@ -39,8 +38,7 @@ init_main_loop: MOVD $0x0000000000000001, R9 do_not_update_high_threshold: - ADD R8<<1, R3, R15 - MOVH R9, (R15) + MOVH R9, (R3)(R8<<1) ADD $1, R8, R8 init_main_loop_condition: @@ -64,8 +62,7 @@ init_main_loop_condition: spread_main_loop: MOVD $0, R12 - ADD R11<<1, R1, R15 - MOVH (R15), R13 + MOVH (R1)(R11<<1), R13 JMP spread_inner_loop_condition spread_inner_loop: @@ -102,11 +99,9 @@ spread_check_ok: build_table_main_table: ADD R6<<3, R5, R15 MOVBU 1(R15), R1 - ADD R1<<1, R3, R15 - MOVHU (R15), R7 + MOVHU (R3)(R1<<1), R7 ADD $1, R7, R8 - ADD R1<<1, R3, R15 - MOVH R8, (R15) + MOVH R8, (R3)(R1<<1) MOVD R7, R8 CLZ R8, R16 MOVD $63, R8 @@ -129,12 +124,10 @@ build_table_main_table: RET build_table_check1_ok: - AND $0xff, R1, R15 - AND $0xff, R1, R16 - TST R16, R15 + TST $0xff, R1 BNE build_table_check2_ok - AND $0xffff, R7, R15 AND $0xffff, R6, R16 + AND $0xffff, R7, R15 CMP R16, R15 BNE build_table_check2_ok MOVD ctx+8(FP), R0 diff --git a/vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash.go b/vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash.go index fc40c820016c..9c0cbc1681db 100644 --- a/vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash.go +++ b/vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash.go @@ -52,6 +52,11 @@ func (d *Digest) Reset() { d.n = 0 } +// maxAsmSize bounds the input handed to a single writeBlocks call. Assembly is +// never preemptible, so an unbounded call holds every P in stop-the-world for +// its duration. Whole 32-byte blocks, so Write can feed it in pieces. +const maxAsmSize = 128 << 10 // 4096 whole 32-byte blocks + // Size always returns 8 bytes. func (d *Digest) Size() int { return 8 } @@ -83,6 +88,13 @@ func (d *Digest) Write(b []byte) (n int, err error) { d.n = 0 } + for len(b) >= maxAsmSize { + // Assembly is not preemptible, so hashing a large buffer in one call + // blocks every stop-the-world for the whole call. Feed it in chunks. + writeBlocks(d, b[:maxAsmSize]) + b = b[maxAsmSize:] + } + if len(b) >= 32 { // One or more full blocks left. nw := writeBlocks(d, b) diff --git a/vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash_amd64.s b/vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash_amd64.s index ddb63aa91b14..e4c688efd32a 100644 --- a/vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash_amd64.s +++ b/vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash_amd64.s @@ -175,7 +175,11 @@ finalize: RET // func writeBlocks(d *Digest, b []byte) int -TEXT ·writeBlocks(SB), NOSPLIT|NOFRAME, $0-40 +// +// Deliberately not NOSPLIT: the stack check is the preemption point that lets a +// stop-the-world proceed between chunks. The frame must be >= abi.StackSmall +// (128), or the assembler marks this leaf NOSPLIT anyway and drops that check. +TEXT ·writeBlocks(SB), $128-40 // Load fixed primes needed for round. MOVQ ·primes+0(SB), prime1 MOVQ ·primes+8(SB), prime2 diff --git a/vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash_arm64.s b/vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash_arm64.s index ae7d4d3295a4..e88cc2fc9172 100644 --- a/vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash_arm64.s +++ b/vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash_arm64.s @@ -163,7 +163,11 @@ finalize: RET // func writeBlocks(s *Digest, b []byte) int -TEXT ·writeBlocks(SB), NOSPLIT|NOFRAME, $0-40 +// +// Deliberately not NOSPLIT: the stack check is the preemption point that lets a +// stop-the-world proceed between chunks. The frame must be >= abi.StackSmall +// (128), or the assembler marks this leaf NOSPLIT anyway and drops that check. +TEXT ·writeBlocks(SB), $128-40 LDP ·primes+0(SB), (prime1, prime2) // Load state. Assume v[1-4] are stored contiguously. diff --git a/vendor/github.com/klauspost/compress/zstd/seqdec_amd64.s b/vendor/github.com/klauspost/compress/zstd/seqdec_amd64.s index 4d3188ff49b9..9c4f8d3528dd 100644 --- a/vendor/github.com/klauspost/compress/zstd/seqdec_amd64.s +++ b/vendor/github.com/klauspost/compress/zstd/seqdec_amd64.s @@ -55,43 +55,43 @@ sequenceDecs_decode_amd64_fill_check_overread: sequenceDecs_decode_amd64_fill_end: // Update offset - MOVQ R9, AX - MOVQ BX, CX - MOVQ DX, R15 - SHLQ CL, R15 - MOVB AH, CL - SHRQ $0x20, AX - TESTQ CX, CX - JZ sequenceDecs_decode_amd64_of_update_zero - ADDQ CX, BX - CMPQ BX, $0x40 - JA sequenceDecs_decode_amd64_of_update_zero - CMPQ CX, $0x40 - JAE sequenceDecs_decode_amd64_of_update_zero - NEGQ CX - SHRQ CL, R15 - ADDQ R15, AX + MOVQ R9, AX + MOVQ BX, CX + MOVQ DX, R15 + SHLQ CL, R15 + MOVBLZX AH, CX + SHRQ $0x20, AX + TESTQ CX, CX + JZ sequenceDecs_decode_amd64_of_update_zero + ADDQ CX, BX + CMPQ BX, $0x40 + JA sequenceDecs_decode_amd64_of_update_zero + CMPQ CX, $0x40 + JAE sequenceDecs_decode_amd64_of_update_zero + NEGQ CX + SHRQ CL, R15 + ADDQ R15, AX sequenceDecs_decode_amd64_of_update_zero: MOVQ AX, 16(R10) // Update match length - MOVQ R8, AX - MOVQ BX, CX - MOVQ DX, R15 - SHLQ CL, R15 - MOVB AH, CL - SHRQ $0x20, AX - TESTQ CX, CX - JZ sequenceDecs_decode_amd64_ml_update_zero - ADDQ CX, BX - CMPQ BX, $0x40 - JA sequenceDecs_decode_amd64_ml_update_zero - CMPQ CX, $0x40 - JAE sequenceDecs_decode_amd64_ml_update_zero - NEGQ CX - SHRQ CL, R15 - ADDQ R15, AX + MOVQ R8, AX + MOVQ BX, CX + MOVQ DX, R15 + SHLQ CL, R15 + MOVBLZX AH, CX + SHRQ $0x20, AX + TESTQ CX, CX + JZ sequenceDecs_decode_amd64_ml_update_zero + ADDQ CX, BX + CMPQ BX, $0x40 + JA sequenceDecs_decode_amd64_ml_update_zero + CMPQ CX, $0x40 + JAE sequenceDecs_decode_amd64_ml_update_zero + NEGQ CX + SHRQ CL, R15 + ADDQ R15, AX sequenceDecs_decode_amd64_ml_update_zero: MOVQ AX, 8(R10) @@ -126,22 +126,22 @@ sequenceDecs_decode_amd64_fill_2_check_overread: sequenceDecs_decode_amd64_fill_2_end: // Update literal length - MOVQ DI, AX - MOVQ BX, CX - MOVQ DX, R15 - SHLQ CL, R15 - MOVB AH, CL - SHRQ $0x20, AX - TESTQ CX, CX - JZ sequenceDecs_decode_amd64_ll_update_zero - ADDQ CX, BX - CMPQ BX, $0x40 - JA sequenceDecs_decode_amd64_ll_update_zero - CMPQ CX, $0x40 - JAE sequenceDecs_decode_amd64_ll_update_zero - NEGQ CX - SHRQ CL, R15 - ADDQ R15, AX + MOVQ DI, AX + MOVQ BX, CX + MOVQ DX, R15 + SHLQ CL, R15 + MOVBLZX AH, CX + SHRQ $0x20, AX + TESTQ CX, CX + JZ sequenceDecs_decode_amd64_ll_update_zero + ADDQ CX, BX + CMPQ BX, $0x40 + JA sequenceDecs_decode_amd64_ll_update_zero + CMPQ CX, $0x40 + JAE sequenceDecs_decode_amd64_ll_update_zero + NEGQ CX + SHRQ CL, R15 + ADDQ R15, AX sequenceDecs_decode_amd64_ll_update_zero: MOVQ AX, (R10) @@ -163,7 +163,7 @@ sequenceDecs_decode_amd64_ll_update_zero: MOVQ CX, BX ROLQ CL, R15 MOVL $0x00000001, BP - MOVB R14, CL + MOVBLZX R14, CX SHLL CL, BP DECL BP ANDQ BP, R15 @@ -182,7 +182,7 @@ sequenceDecs_decode_amd64_ll_update_zero: MOVQ CX, BX ROLQ CL, R15 MOVL $0x00000001, BP - MOVB R14, CL + MOVBLZX R14, CX SHLL CL, BP DECL BP ANDQ BP, R15 @@ -201,7 +201,7 @@ sequenceDecs_decode_amd64_ll_update_zero: MOVQ CX, BX ROLQ CL, R15 MOVL $0x00000001, BP - MOVB R14, CL + MOVBLZX R14, CX SHLL CL, BP DECL BP ANDQ BP, R15 @@ -383,64 +383,64 @@ sequenceDecs_decode_56_amd64_fill_check_overread: sequenceDecs_decode_56_amd64_fill_end: // Update offset - MOVQ R9, AX - MOVQ BX, CX - MOVQ DX, R15 - SHLQ CL, R15 - MOVB AH, CL - SHRQ $0x20, AX - TESTQ CX, CX - JZ sequenceDecs_decode_56_amd64_of_update_zero - ADDQ CX, BX - CMPQ BX, $0x40 - JA sequenceDecs_decode_56_amd64_of_update_zero - CMPQ CX, $0x40 - JAE sequenceDecs_decode_56_amd64_of_update_zero - NEGQ CX - SHRQ CL, R15 - ADDQ R15, AX + MOVQ R9, AX + MOVQ BX, CX + MOVQ DX, R15 + SHLQ CL, R15 + MOVBLZX AH, CX + SHRQ $0x20, AX + TESTQ CX, CX + JZ sequenceDecs_decode_56_amd64_of_update_zero + ADDQ CX, BX + CMPQ BX, $0x40 + JA sequenceDecs_decode_56_amd64_of_update_zero + CMPQ CX, $0x40 + JAE sequenceDecs_decode_56_amd64_of_update_zero + NEGQ CX + SHRQ CL, R15 + ADDQ R15, AX sequenceDecs_decode_56_amd64_of_update_zero: MOVQ AX, 16(R10) // Update match length - MOVQ R8, AX - MOVQ BX, CX - MOVQ DX, R15 - SHLQ CL, R15 - MOVB AH, CL - SHRQ $0x20, AX - TESTQ CX, CX - JZ sequenceDecs_decode_56_amd64_ml_update_zero - ADDQ CX, BX - CMPQ BX, $0x40 - JA sequenceDecs_decode_56_amd64_ml_update_zero - CMPQ CX, $0x40 - JAE sequenceDecs_decode_56_amd64_ml_update_zero - NEGQ CX - SHRQ CL, R15 - ADDQ R15, AX + MOVQ R8, AX + MOVQ BX, CX + MOVQ DX, R15 + SHLQ CL, R15 + MOVBLZX AH, CX + SHRQ $0x20, AX + TESTQ CX, CX + JZ sequenceDecs_decode_56_amd64_ml_update_zero + ADDQ CX, BX + CMPQ BX, $0x40 + JA sequenceDecs_decode_56_amd64_ml_update_zero + CMPQ CX, $0x40 + JAE sequenceDecs_decode_56_amd64_ml_update_zero + NEGQ CX + SHRQ CL, R15 + ADDQ R15, AX sequenceDecs_decode_56_amd64_ml_update_zero: MOVQ AX, 8(R10) // Update literal length - MOVQ DI, AX - MOVQ BX, CX - MOVQ DX, R15 - SHLQ CL, R15 - MOVB AH, CL - SHRQ $0x20, AX - TESTQ CX, CX - JZ sequenceDecs_decode_56_amd64_ll_update_zero - ADDQ CX, BX - CMPQ BX, $0x40 - JA sequenceDecs_decode_56_amd64_ll_update_zero - CMPQ CX, $0x40 - JAE sequenceDecs_decode_56_amd64_ll_update_zero - NEGQ CX - SHRQ CL, R15 - ADDQ R15, AX + MOVQ DI, AX + MOVQ BX, CX + MOVQ DX, R15 + SHLQ CL, R15 + MOVBLZX AH, CX + SHRQ $0x20, AX + TESTQ CX, CX + JZ sequenceDecs_decode_56_amd64_ll_update_zero + ADDQ CX, BX + CMPQ BX, $0x40 + JA sequenceDecs_decode_56_amd64_ll_update_zero + CMPQ CX, $0x40 + JAE sequenceDecs_decode_56_amd64_ll_update_zero + NEGQ CX + SHRQ CL, R15 + ADDQ R15, AX sequenceDecs_decode_56_amd64_ll_update_zero: MOVQ AX, (R10) @@ -462,7 +462,7 @@ sequenceDecs_decode_56_amd64_ll_update_zero: MOVQ CX, BX ROLQ CL, R15 MOVL $0x00000001, BP - MOVB R14, CL + MOVBLZX R14, CX SHLL CL, BP DECL BP ANDQ BP, R15 @@ -481,7 +481,7 @@ sequenceDecs_decode_56_amd64_ll_update_zero: MOVQ CX, BX ROLQ CL, R15 MOVL $0x00000001, BP - MOVB R14, CL + MOVBLZX R14, CX SHLL CL, BP DECL BP ANDQ BP, R15 @@ -500,7 +500,7 @@ sequenceDecs_decode_56_amd64_ll_update_zero: MOVQ CX, BX ROLQ CL, R15 MOVL $0x00000001, BP - MOVB R14, CL + MOVBLZX R14, CX SHLL CL, BP DECL BP ANDQ BP, R15 @@ -1258,13 +1258,13 @@ copy_4_small: JMP copy_4_move_8through16 copy_4_move_3: - MOVW (R14), R11 - MOVB 2(R14), R12 - MOVW R11, (BX) - MOVB R12, 2(BX) - ADDQ R13, R14 - ADDQ R13, BX - JMP copy_4_end + MOVWQZX (R14), R11 + MOVB 2(R14), R12 + MOVW R11, (BX) + MOVB R12, 2(BX) + ADDQ R13, R14 + ADDQ R13, BX + JMP copy_4_end copy_4_move_4through7: MOVL (R14), R11 @@ -1327,13 +1327,13 @@ copy_5_move_1or2: JMP copy_5_end copy_5_move_3: - MOVW (R14), R15 - MOVB 2(R14), BP - MOVW R15, (BX) - MOVB BP, 2(BX) - ADDQ R11, R14 - ADDQ R11, BX - JMP copy_5_end + MOVWQZX (R14), R15 + MOVB 2(R14), BP + MOVW R15, (BX) + MOVB BP, 2(BX) + ADDQ R11, R14 + ADDQ R11, BX + JMP copy_5_end copy_5_move_4through7: MOVL (R14), R15 @@ -1384,12 +1384,12 @@ copy_overlapping_match: ADDQ R13, DI copy_slow_3: - MOVB (R11), R12 - MOVB R12, (BX) - INCQ R11 - INCQ BX - DECQ R13 - JNZ copy_slow_3 + MOVBQZX (R11), R12 + MOVB R12, (BX) + INCQ R11 + INCQ BX + DECQ R13 + JNZ copy_slow_3 handle_loop: ADDQ $0x18, AX @@ -1494,13 +1494,13 @@ copy_1_move_1or2: JMP copy_1_end copy_1_move_3: - MOVW (SI), R14 - MOVB 2(SI), R15 - MOVW R14, (BX) - MOVB R15, 2(BX) - ADDQ R11, SI - ADDQ R11, BX - JMP copy_1_end + MOVWQZX (SI), R14 + MOVB 2(SI), R15 + MOVW R14, (BX) + MOVB R15, 2(BX) + ADDQ R11, SI + ADDQ R11, BX + JMP copy_1_end copy_1_move_4through7: MOVL (SI), R14 @@ -1563,13 +1563,13 @@ copy_4_small: JMP copy_4_move_8through16 copy_4_move_3: - MOVW (R14), R11 - MOVB 2(R14), R12 - MOVW R11, (BX) - MOVB R12, 2(BX) - ADDQ R13, R14 - ADDQ R13, BX - JMP copy_4_end + MOVWQZX (R14), R11 + MOVB 2(R14), R12 + MOVW R11, (BX) + MOVB R12, 2(BX) + ADDQ R13, R14 + ADDQ R13, BX + JMP copy_4_end copy_4_move_4through7: MOVL (R14), R11 @@ -1632,13 +1632,13 @@ copy_5_move_1or2: JMP copy_5_end copy_5_move_3: - MOVW (R14), R15 - MOVB 2(R14), BP - MOVW R15, (BX) - MOVB BP, 2(BX) - ADDQ R11, R14 - ADDQ R11, BX - JMP copy_5_end + MOVWQZX (R14), R15 + MOVB 2(R14), BP + MOVW R15, (BX) + MOVB BP, 2(BX) + ADDQ R11, R14 + ADDQ R11, BX + JMP copy_5_end copy_5_move_4through7: MOVL (R14), R15 @@ -1707,13 +1707,13 @@ copy_2_move_1or2: JMP copy_2_end copy_2_move_3: - MOVW (R11), R12 - MOVB 2(R11), R14 - MOVW R12, (BX) - MOVB R14, 2(BX) - ADDQ R13, R11 - ADDQ R13, BX - JMP copy_2_end + MOVWQZX (R11), R12 + MOVB 2(R11), R14 + MOVW R12, (BX) + MOVB R14, 2(BX) + ADDQ R13, R11 + ADDQ R13, BX + JMP copy_2_end copy_2_move_4through7: MOVL (R11), R12 @@ -1740,12 +1740,12 @@ copy_overlapping_match: ADDQ R13, DI copy_slow_3: - MOVB (R11), R12 - MOVB R12, (BX) - INCQ R11 - INCQ BX - DECQ R13 - JNZ copy_slow_3 + MOVBQZX (R11), R12 + MOVB R12, (BX) + INCQ R11 + INCQ BX + DECQ R13 + JNZ copy_slow_3 handle_loop: ADDQ $0x18, AX @@ -1853,43 +1853,43 @@ sequenceDecs_decodeSync_amd64_fill_check_overread: sequenceDecs_decodeSync_amd64_fill_end: // Update offset - MOVQ R9, AX - MOVQ BX, CX - MOVQ DX, R14 - SHLQ CL, R14 - MOVB AH, CL - SHRQ $0x20, AX - TESTQ CX, CX - JZ sequenceDecs_decodeSync_amd64_of_update_zero - ADDQ CX, BX - CMPQ BX, $0x40 - JA sequenceDecs_decodeSync_amd64_of_update_zero - CMPQ CX, $0x40 - JAE sequenceDecs_decodeSync_amd64_of_update_zero - NEGQ CX - SHRQ CL, R14 - ADDQ R14, AX + MOVQ R9, AX + MOVQ BX, CX + MOVQ DX, R14 + SHLQ CL, R14 + MOVBLZX AH, CX + SHRQ $0x20, AX + TESTQ CX, CX + JZ sequenceDecs_decodeSync_amd64_of_update_zero + ADDQ CX, BX + CMPQ BX, $0x40 + JA sequenceDecs_decodeSync_amd64_of_update_zero + CMPQ CX, $0x40 + JAE sequenceDecs_decodeSync_amd64_of_update_zero + NEGQ CX + SHRQ CL, R14 + ADDQ R14, AX sequenceDecs_decodeSync_amd64_of_update_zero: MOVQ AX, 8(SP) // Update match length - MOVQ R8, AX - MOVQ BX, CX - MOVQ DX, R14 - SHLQ CL, R14 - MOVB AH, CL - SHRQ $0x20, AX - TESTQ CX, CX - JZ sequenceDecs_decodeSync_amd64_ml_update_zero - ADDQ CX, BX - CMPQ BX, $0x40 - JA sequenceDecs_decodeSync_amd64_ml_update_zero - CMPQ CX, $0x40 - JAE sequenceDecs_decodeSync_amd64_ml_update_zero - NEGQ CX - SHRQ CL, R14 - ADDQ R14, AX + MOVQ R8, AX + MOVQ BX, CX + MOVQ DX, R14 + SHLQ CL, R14 + MOVBLZX AH, CX + SHRQ $0x20, AX + TESTQ CX, CX + JZ sequenceDecs_decodeSync_amd64_ml_update_zero + ADDQ CX, BX + CMPQ BX, $0x40 + JA sequenceDecs_decodeSync_amd64_ml_update_zero + CMPQ CX, $0x40 + JAE sequenceDecs_decodeSync_amd64_ml_update_zero + NEGQ CX + SHRQ CL, R14 + ADDQ R14, AX sequenceDecs_decodeSync_amd64_ml_update_zero: MOVQ AX, 16(SP) @@ -1924,22 +1924,22 @@ sequenceDecs_decodeSync_amd64_fill_2_check_overread: sequenceDecs_decodeSync_amd64_fill_2_end: // Update literal length - MOVQ DI, AX - MOVQ BX, CX - MOVQ DX, R14 - SHLQ CL, R14 - MOVB AH, CL - SHRQ $0x20, AX - TESTQ CX, CX - JZ sequenceDecs_decodeSync_amd64_ll_update_zero - ADDQ CX, BX - CMPQ BX, $0x40 - JA sequenceDecs_decodeSync_amd64_ll_update_zero - CMPQ CX, $0x40 - JAE sequenceDecs_decodeSync_amd64_ll_update_zero - NEGQ CX - SHRQ CL, R14 - ADDQ R14, AX + MOVQ DI, AX + MOVQ BX, CX + MOVQ DX, R14 + SHLQ CL, R14 + MOVBLZX AH, CX + SHRQ $0x20, AX + TESTQ CX, CX + JZ sequenceDecs_decodeSync_amd64_ll_update_zero + ADDQ CX, BX + CMPQ BX, $0x40 + JA sequenceDecs_decodeSync_amd64_ll_update_zero + CMPQ CX, $0x40 + JAE sequenceDecs_decodeSync_amd64_ll_update_zero + NEGQ CX + SHRQ CL, R14 + ADDQ R14, AX sequenceDecs_decodeSync_amd64_ll_update_zero: MOVQ AX, 24(SP) @@ -1961,7 +1961,7 @@ sequenceDecs_decodeSync_amd64_ll_update_zero: MOVQ CX, BX ROLQ CL, R14 MOVL $0x00000001, R15 - MOVB R13, CL + MOVBLZX R13, CX SHLL CL, R15 DECL R15 ANDQ R15, R14 @@ -1980,7 +1980,7 @@ sequenceDecs_decodeSync_amd64_ll_update_zero: MOVQ CX, BX ROLQ CL, R14 MOVL $0x00000001, R15 - MOVB R13, CL + MOVBLZX R13, CX SHLL CL, R15 DECL R15 ANDQ R15, R14 @@ -1999,7 +1999,7 @@ sequenceDecs_decodeSync_amd64_ll_update_zero: MOVQ CX, BX ROLQ CL, R14 MOVL $0x00000001, R15 - MOVB R13, CL + MOVBLZX R13, CX SHLL CL, R15 DECL R15 ANDQ R15, R14 @@ -2143,13 +2143,13 @@ copy_4_small: JMP copy_4_move_8through16 copy_4_move_3: - MOVW (R14), AX - MOVB 2(R14), CL - MOVW AX, (R10) - MOVB CL, 2(R10) - ADDQ R13, R14 - ADDQ R13, R10 - JMP copy_4_end + MOVWQZX (R14), AX + MOVB 2(R14), CL + MOVW AX, (R10) + MOVB CL, 2(R10) + ADDQ R13, R14 + ADDQ R13, R10 + JMP copy_4_end copy_4_move_4through7: MOVL (R14), AX @@ -2209,13 +2209,13 @@ copy_5_move_1or2: JMP copy_5_end copy_5_move_3: - MOVW (R14), R15 - MOVB 2(R14), BP - MOVW R15, (R10) - MOVB BP, 2(R10) - ADDQ AX, R14 - ADDQ AX, R10 - JMP copy_5_end + MOVWQZX (R14), R15 + MOVB 2(R14), BP + MOVW R15, (R10) + MOVB BP, 2(R10) + ADDQ AX, R14 + ADDQ AX, R10 + JMP copy_5_end copy_5_move_4through7: MOVL (R14), R15 @@ -2266,12 +2266,12 @@ copy_overlapping_match: ADDQ R13, R12 copy_slow_3: - MOVB (AX), CL - MOVB CL, (R10) - INCQ AX - INCQ R10 - DECQ R13 - JNZ copy_slow_3 + MOVBQZX (AX), CX + MOVB CL, (R10) + INCQ AX + INCQ R10 + DECQ R13 + JNZ copy_slow_3 handle_loop: MOVQ ctx+16(FP), AX @@ -2663,13 +2663,13 @@ copy_4_small: JMP copy_4_move_8through16 copy_4_move_3: - MOVW (R14), CX - MOVB 2(R14), R12 - MOVW CX, (R9) - MOVB R12, 2(R9) - ADDQ R13, R14 - ADDQ R13, R9 - JMP copy_4_end + MOVWQZX (R14), CX + MOVB 2(R14), R12 + MOVW CX, (R9) + MOVB R12, 2(R9) + ADDQ R13, R14 + ADDQ R13, R9 + JMP copy_4_end copy_4_move_4through7: MOVL (R14), CX @@ -2729,13 +2729,13 @@ copy_5_move_1or2: JMP copy_5_end copy_5_move_3: - MOVW (R14), R15 - MOVB 2(R14), BP - MOVW R15, (R9) - MOVB BP, 2(R9) - ADDQ CX, R14 - ADDQ CX, R9 - JMP copy_5_end + MOVWQZX (R14), R15 + MOVB 2(R14), BP + MOVW R15, (R9) + MOVB BP, 2(R9) + ADDQ CX, R14 + ADDQ CX, R9 + JMP copy_5_end copy_5_move_4through7: MOVL (R14), R15 @@ -2786,12 +2786,12 @@ copy_overlapping_match: ADDQ R13, R11 copy_slow_3: - MOVB (CX), R12 - MOVB R12, (R9) - INCQ CX - INCQ R9 - DECQ R13 - JNZ copy_slow_3 + MOVBQZX (CX), R12 + MOVB R12, (R9) + INCQ CX + INCQ R9 + DECQ R13 + JNZ copy_slow_3 handle_loop: MOVQ ctx+16(FP), CX @@ -2935,43 +2935,43 @@ sequenceDecs_decodeSync_safe_amd64_fill_check_overread: sequenceDecs_decodeSync_safe_amd64_fill_end: // Update offset - MOVQ R9, AX - MOVQ BX, CX - MOVQ DX, R14 - SHLQ CL, R14 - MOVB AH, CL - SHRQ $0x20, AX - TESTQ CX, CX - JZ sequenceDecs_decodeSync_safe_amd64_of_update_zero - ADDQ CX, BX - CMPQ BX, $0x40 - JA sequenceDecs_decodeSync_safe_amd64_of_update_zero - CMPQ CX, $0x40 - JAE sequenceDecs_decodeSync_safe_amd64_of_update_zero - NEGQ CX - SHRQ CL, R14 - ADDQ R14, AX + MOVQ R9, AX + MOVQ BX, CX + MOVQ DX, R14 + SHLQ CL, R14 + MOVBLZX AH, CX + SHRQ $0x20, AX + TESTQ CX, CX + JZ sequenceDecs_decodeSync_safe_amd64_of_update_zero + ADDQ CX, BX + CMPQ BX, $0x40 + JA sequenceDecs_decodeSync_safe_amd64_of_update_zero + CMPQ CX, $0x40 + JAE sequenceDecs_decodeSync_safe_amd64_of_update_zero + NEGQ CX + SHRQ CL, R14 + ADDQ R14, AX sequenceDecs_decodeSync_safe_amd64_of_update_zero: MOVQ AX, 8(SP) // Update match length - MOVQ R8, AX - MOVQ BX, CX - MOVQ DX, R14 - SHLQ CL, R14 - MOVB AH, CL - SHRQ $0x20, AX - TESTQ CX, CX - JZ sequenceDecs_decodeSync_safe_amd64_ml_update_zero - ADDQ CX, BX - CMPQ BX, $0x40 - JA sequenceDecs_decodeSync_safe_amd64_ml_update_zero - CMPQ CX, $0x40 - JAE sequenceDecs_decodeSync_safe_amd64_ml_update_zero - NEGQ CX - SHRQ CL, R14 - ADDQ R14, AX + MOVQ R8, AX + MOVQ BX, CX + MOVQ DX, R14 + SHLQ CL, R14 + MOVBLZX AH, CX + SHRQ $0x20, AX + TESTQ CX, CX + JZ sequenceDecs_decodeSync_safe_amd64_ml_update_zero + ADDQ CX, BX + CMPQ BX, $0x40 + JA sequenceDecs_decodeSync_safe_amd64_ml_update_zero + CMPQ CX, $0x40 + JAE sequenceDecs_decodeSync_safe_amd64_ml_update_zero + NEGQ CX + SHRQ CL, R14 + ADDQ R14, AX sequenceDecs_decodeSync_safe_amd64_ml_update_zero: MOVQ AX, 16(SP) @@ -3006,22 +3006,22 @@ sequenceDecs_decodeSync_safe_amd64_fill_2_check_overread: sequenceDecs_decodeSync_safe_amd64_fill_2_end: // Update literal length - MOVQ DI, AX - MOVQ BX, CX - MOVQ DX, R14 - SHLQ CL, R14 - MOVB AH, CL - SHRQ $0x20, AX - TESTQ CX, CX - JZ sequenceDecs_decodeSync_safe_amd64_ll_update_zero - ADDQ CX, BX - CMPQ BX, $0x40 - JA sequenceDecs_decodeSync_safe_amd64_ll_update_zero - CMPQ CX, $0x40 - JAE sequenceDecs_decodeSync_safe_amd64_ll_update_zero - NEGQ CX - SHRQ CL, R14 - ADDQ R14, AX + MOVQ DI, AX + MOVQ BX, CX + MOVQ DX, R14 + SHLQ CL, R14 + MOVBLZX AH, CX + SHRQ $0x20, AX + TESTQ CX, CX + JZ sequenceDecs_decodeSync_safe_amd64_ll_update_zero + ADDQ CX, BX + CMPQ BX, $0x40 + JA sequenceDecs_decodeSync_safe_amd64_ll_update_zero + CMPQ CX, $0x40 + JAE sequenceDecs_decodeSync_safe_amd64_ll_update_zero + NEGQ CX + SHRQ CL, R14 + ADDQ R14, AX sequenceDecs_decodeSync_safe_amd64_ll_update_zero: MOVQ AX, 24(SP) @@ -3043,7 +3043,7 @@ sequenceDecs_decodeSync_safe_amd64_ll_update_zero: MOVQ CX, BX ROLQ CL, R14 MOVL $0x00000001, R15 - MOVB R13, CL + MOVBLZX R13, CX SHLL CL, R15 DECL R15 ANDQ R15, R14 @@ -3062,7 +3062,7 @@ sequenceDecs_decodeSync_safe_amd64_ll_update_zero: MOVQ CX, BX ROLQ CL, R14 MOVL $0x00000001, R15 - MOVB R13, CL + MOVBLZX R13, CX SHLL CL, R15 DECL R15 ANDQ R15, R14 @@ -3081,7 +3081,7 @@ sequenceDecs_decodeSync_safe_amd64_ll_update_zero: MOVQ CX, BX ROLQ CL, R14 MOVL $0x00000001, R15 - MOVB R13, CL + MOVBLZX R13, CX SHLL CL, R15 DECL R15 ANDQ R15, R14 @@ -3206,13 +3206,13 @@ copy_1_move_1or2: JMP copy_1_end copy_1_move_3: - MOVW (R11), R14 - MOVB 2(R11), R15 - MOVW R14, (R10) - MOVB R15, 2(R10) - ADDQ AX, R11 - ADDQ AX, R10 - JMP copy_1_end + MOVWQZX (R11), R14 + MOVB 2(R11), R15 + MOVW R14, (R10) + MOVB R15, 2(R10) + ADDQ AX, R11 + ADDQ AX, R10 + JMP copy_1_end copy_1_move_4through7: MOVL (R11), R14 @@ -3276,13 +3276,13 @@ copy_4_small: JMP copy_4_move_8through16 copy_4_move_3: - MOVW (R14), AX - MOVB 2(R14), CL - MOVW AX, (R10) - MOVB CL, 2(R10) - ADDQ R13, R14 - ADDQ R13, R10 - JMP copy_4_end + MOVWQZX (R14), AX + MOVB 2(R14), CL + MOVW AX, (R10) + MOVB CL, 2(R10) + ADDQ R13, R14 + ADDQ R13, R10 + JMP copy_4_end copy_4_move_4through7: MOVL (R14), AX @@ -3342,13 +3342,13 @@ copy_5_move_1or2: JMP copy_5_end copy_5_move_3: - MOVW (R14), R15 - MOVB 2(R14), BP - MOVW R15, (R10) - MOVB BP, 2(R10) - ADDQ AX, R14 - ADDQ AX, R10 - JMP copy_5_end + MOVWQZX (R14), R15 + MOVB 2(R14), BP + MOVW R15, (R10) + MOVB BP, 2(R10) + ADDQ AX, R14 + ADDQ AX, R10 + JMP copy_5_end copy_5_move_4through7: MOVL (R14), R15 @@ -3417,13 +3417,13 @@ copy_2_move_1or2: JMP copy_2_end copy_2_move_3: - MOVW (AX), CX - MOVB 2(AX), R14 - MOVW CX, (R10) - MOVB R14, 2(R10) - ADDQ R13, AX - ADDQ R13, R10 - JMP copy_2_end + MOVWQZX (AX), CX + MOVB 2(AX), R14 + MOVW CX, (R10) + MOVB R14, 2(R10) + ADDQ R13, AX + ADDQ R13, R10 + JMP copy_2_end copy_2_move_4through7: MOVL (AX), CX @@ -3450,12 +3450,12 @@ copy_overlapping_match: ADDQ R13, R12 copy_slow_3: - MOVB (AX), CL - MOVB CL, (R10) - INCQ AX - INCQ R10 - DECQ R13 - JNZ copy_slow_3 + MOVBQZX (AX), CX + MOVB CL, (R10) + INCQ AX + INCQ R10 + DECQ R13 + JNZ copy_slow_3 handle_loop: MOVQ ctx+16(FP), AX @@ -3828,13 +3828,13 @@ copy_1_move_1or2: JMP copy_1_end copy_1_move_3: - MOVW (R10), R14 - MOVB 2(R10), R15 - MOVW R14, (R9) - MOVB R15, 2(R9) - ADDQ CX, R10 - ADDQ CX, R9 - JMP copy_1_end + MOVWQZX (R10), R14 + MOVB 2(R10), R15 + MOVW R14, (R9) + MOVB R15, 2(R9) + ADDQ CX, R10 + ADDQ CX, R9 + JMP copy_1_end copy_1_move_4through7: MOVL (R10), R14 @@ -3898,13 +3898,13 @@ copy_4_small: JMP copy_4_move_8through16 copy_4_move_3: - MOVW (R14), CX - MOVB 2(R14), R12 - MOVW CX, (R9) - MOVB R12, 2(R9) - ADDQ R13, R14 - ADDQ R13, R9 - JMP copy_4_end + MOVWQZX (R14), CX + MOVB 2(R14), R12 + MOVW CX, (R9) + MOVB R12, 2(R9) + ADDQ R13, R14 + ADDQ R13, R9 + JMP copy_4_end copy_4_move_4through7: MOVL (R14), CX @@ -3964,13 +3964,13 @@ copy_5_move_1or2: JMP copy_5_end copy_5_move_3: - MOVW (R14), R15 - MOVB 2(R14), BP - MOVW R15, (R9) - MOVB BP, 2(R9) - ADDQ CX, R14 - ADDQ CX, R9 - JMP copy_5_end + MOVWQZX (R14), R15 + MOVB 2(R14), BP + MOVW R15, (R9) + MOVB BP, 2(R9) + ADDQ CX, R14 + ADDQ CX, R9 + JMP copy_5_end copy_5_move_4through7: MOVL (R14), R15 @@ -4039,13 +4039,13 @@ copy_2_move_1or2: JMP copy_2_end copy_2_move_3: - MOVW (CX), R12 - MOVB 2(CX), R14 - MOVW R12, (R9) - MOVB R14, 2(R9) - ADDQ R13, CX - ADDQ R13, R9 - JMP copy_2_end + MOVWQZX (CX), R12 + MOVB 2(CX), R14 + MOVW R12, (R9) + MOVB R14, 2(R9) + ADDQ R13, CX + ADDQ R13, R9 + JMP copy_2_end copy_2_move_4through7: MOVL (CX), R12 @@ -4072,12 +4072,12 @@ copy_overlapping_match: ADDQ R13, R11 copy_slow_3: - MOVB (CX), R12 - MOVB R12, (R9) - INCQ CX - INCQ R9 - DECQ R13 - JNZ copy_slow_3 + MOVBQZX (CX), R12 + MOVB R12, (R9) + INCQ CX + INCQ R9 + DECQ R13 + JNZ copy_slow_3 handle_loop: MOVQ ctx+16(FP), CX diff --git a/vendor/github.com/klauspost/compress/zstd/seqdec_arm64.s b/vendor/github.com/klauspost/compress/zstd/seqdec_arm64.s index 6f54ef21ba12..fa48be19e4e8 100644 --- a/vendor/github.com/klauspost/compress/zstd/seqdec_arm64.s +++ b/vendor/github.com/klauspost/compress/zstd/seqdec_arm64.s @@ -1,7 +1,7 @@ // Code generated by command: go run gen.go -out ../seqdec.s -arch amd64,arm64 -pkg=zstd. DO NOT EDIT. // EXPERIMENTAL arm64 output lowered from an amd64 avo program. -//go:build arm64 && !appengine && !noasm && gc && !noasm +//go:build arm64 && (!appengine && !noasm && gc && !noasm) // func sequenceDecs_decode_amd64(s *sequenceDecs, br *bitReader, ctx *decodeAsmContext) int // Requires: CMOV @@ -60,8 +60,7 @@ sequenceDecs_decode_amd64_fill_end: MOVD R3, R1 MOVD R2, R14 LSL R1, R14, R14 - UBFX $8, R0, $8, R16 - BFI $0, R16, $8, R1 + UBFX $8, R0, $8, R1 LSR $0x20, R0, R0 TST R1, R1 BEQ sequenceDecs_decode_amd64_of_update_zero @@ -82,8 +81,7 @@ sequenceDecs_decode_amd64_of_update_zero: MOVD R3, R1 MOVD R2, R14 LSL R1, R14, R14 - UBFX $8, R0, $8, R16 - BFI $0, R16, $8, R1 + UBFX $8, R0, $8, R1 LSR $0x20, R0, R0 TST R1, R1 BEQ sequenceDecs_decode_amd64_ml_update_zero @@ -133,8 +131,7 @@ sequenceDecs_decode_amd64_fill_2_end: MOVD R3, R1 MOVD R2, R14 LSL R1, R14, R14 - UBFX $8, R0, $8, R16 - BFI $0, R16, $8, R1 + UBFX $8, R0, $8, R1 LSR $0x20, R0, R0 TST R1, R1 BEQ sequenceDecs_decode_amd64_ll_update_zero @@ -151,14 +148,12 @@ sequenceDecs_decode_amd64_ll_update_zero: MOVD R0, (R9) // Fill bitreader for state updates - MOVD R13, 8(RSP) - MOVD R8, R0 - LSR $0x08, R0, R0 - MOVBU R0, R0 - MOVD ctx+16(FP), R1 - MOVD 96(R1), R16 - CMP $0x00, R16 - BEQ sequenceDecs_decode_amd64_skip_update + MOVD R13, 8(RSP) + UBFX $8, R8, $8, R0 + MOVD ctx+16(FP), R1 + MOVD 96(R1), R16 + CMP $0x00, R16 + BEQ sequenceDecs_decode_amd64_skip_update // Update Literal Length State MOVBU R6, R13 @@ -169,7 +164,7 @@ sequenceDecs_decode_amd64_ll_update_zero: NEG R1, R16 ROR R16, R14, R14 MOVD $0x00000001, R4 - BFI $0, R13, $8, R1 + MOVBU R13, R1 LSLW R1, R4, R4 SUBW $1, R4, R4 AND R4, R14, R14 @@ -178,8 +173,7 @@ sequenceDecs_decode_amd64_ll_update_zero: // Load ctx.llTable MOVD ctx+16(FP), R1 MOVD (R1), R1 - ADD R6<<3, R1, R15 - MOVD (R15), R6 + MOVD (R1)(R6<<3), R6 // Update Match Length State MOVBU R7, R13 @@ -190,7 +184,7 @@ sequenceDecs_decode_amd64_ll_update_zero: NEG R1, R16 ROR R16, R14, R14 MOVD $0x00000001, R4 - BFI $0, R13, $8, R1 + MOVBU R13, R1 LSLW R1, R4, R4 SUBW $1, R4, R4 AND R4, R14, R14 @@ -199,8 +193,7 @@ sequenceDecs_decode_amd64_ll_update_zero: // Load ctx.mlTable MOVD ctx+16(FP), R1 MOVD 24(R1), R1 - ADD R7<<3, R1, R15 - MOVD (R15), R7 + MOVD (R1)(R7<<3), R7 // Update Offset State MOVBU R8, R13 @@ -211,7 +204,7 @@ sequenceDecs_decode_amd64_ll_update_zero: NEG R1, R16 ROR R16, R14, R14 MOVD $0x00000001, R4 - BFI $0, R13, $8, R1 + MOVBU R13, R1 LSLW R1, R4, R4 SUBW $1, R4, R4 AND R4, R14, R14 @@ -220,8 +213,7 @@ sequenceDecs_decode_amd64_ll_update_zero: // Load ctx.ofTable MOVD ctx+16(FP), R1 MOVD 48(R1), R1 - ADD R8<<3, R1, R15 - MOVD (R15), R8 + MOVD (R1)(R8<<3), R8 sequenceDecs_decode_amd64_skip_update: // Adjust offset @@ -411,8 +403,7 @@ sequenceDecs_decode_56_amd64_fill_end: MOVD R3, R1 MOVD R2, R14 LSL R1, R14, R14 - UBFX $8, R0, $8, R16 - BFI $0, R16, $8, R1 + UBFX $8, R0, $8, R1 LSR $0x20, R0, R0 TST R1, R1 BEQ sequenceDecs_decode_56_amd64_of_update_zero @@ -433,8 +424,7 @@ sequenceDecs_decode_56_amd64_of_update_zero: MOVD R3, R1 MOVD R2, R14 LSL R1, R14, R14 - UBFX $8, R0, $8, R16 - BFI $0, R16, $8, R1 + UBFX $8, R0, $8, R1 LSR $0x20, R0, R0 TST R1, R1 BEQ sequenceDecs_decode_56_amd64_ml_update_zero @@ -455,8 +445,7 @@ sequenceDecs_decode_56_amd64_ml_update_zero: MOVD R3, R1 MOVD R2, R14 LSL R1, R14, R14 - UBFX $8, R0, $8, R16 - BFI $0, R16, $8, R1 + UBFX $8, R0, $8, R1 LSR $0x20, R0, R0 TST R1, R1 BEQ sequenceDecs_decode_56_amd64_ll_update_zero @@ -473,14 +462,12 @@ sequenceDecs_decode_56_amd64_ll_update_zero: MOVD R0, (R9) // Fill bitreader for state updates - MOVD R13, 8(RSP) - MOVD R8, R0 - LSR $0x08, R0, R0 - MOVBU R0, R0 - MOVD ctx+16(FP), R1 - MOVD 96(R1), R16 - CMP $0x00, R16 - BEQ sequenceDecs_decode_56_amd64_skip_update + MOVD R13, 8(RSP) + UBFX $8, R8, $8, R0 + MOVD ctx+16(FP), R1 + MOVD 96(R1), R16 + CMP $0x00, R16 + BEQ sequenceDecs_decode_56_amd64_skip_update // Update Literal Length State MOVBU R6, R13 @@ -491,7 +478,7 @@ sequenceDecs_decode_56_amd64_ll_update_zero: NEG R1, R16 ROR R16, R14, R14 MOVD $0x00000001, R4 - BFI $0, R13, $8, R1 + MOVBU R13, R1 LSLW R1, R4, R4 SUBW $1, R4, R4 AND R4, R14, R14 @@ -500,8 +487,7 @@ sequenceDecs_decode_56_amd64_ll_update_zero: // Load ctx.llTable MOVD ctx+16(FP), R1 MOVD (R1), R1 - ADD R6<<3, R1, R15 - MOVD (R15), R6 + MOVD (R1)(R6<<3), R6 // Update Match Length State MOVBU R7, R13 @@ -512,7 +498,7 @@ sequenceDecs_decode_56_amd64_ll_update_zero: NEG R1, R16 ROR R16, R14, R14 MOVD $0x00000001, R4 - BFI $0, R13, $8, R1 + MOVBU R13, R1 LSLW R1, R4, R4 SUBW $1, R4, R4 AND R4, R14, R14 @@ -521,8 +507,7 @@ sequenceDecs_decode_56_amd64_ll_update_zero: // Load ctx.mlTable MOVD ctx+16(FP), R1 MOVD 24(R1), R1 - ADD R7<<3, R1, R15 - MOVD (R15), R7 + MOVD (R1)(R7<<3), R7 // Update Offset State MOVBU R8, R13 @@ -533,7 +518,7 @@ sequenceDecs_decode_56_amd64_ll_update_zero: NEG R1, R16 ROR R16, R14, R14 MOVD $0x00000001, R4 - BFI $0, R13, $8, R1 + MOVBU R13, R1 LSLW R1, R4, R4 SUBW $1, R4, R4 AND R4, R14, R14 @@ -542,8 +527,7 @@ sequenceDecs_decode_56_amd64_ll_update_zero: // Load ctx.ofTable MOVD ctx+16(FP), R1 MOVD 48(R1), R1 - ADD R8<<3, R1, R15 - MOVD (R15), R8 + MOVD (R1)(R8<<3), R8 sequenceDecs_decode_56_amd64_skip_update: // Adjust offset @@ -716,16 +700,16 @@ main_loop: MOVD $0, R13 copy_1: - ADD R13, R5, R15 - VLD1 (R15), [V0.B16] - ADD R13, R3, R15 - VST1 [V0.B16], (R15) - ADD $0x10, R13, R13 - CMP R10, R13 - BLO copy_1 - ADD R10, R5, R5 - ADD R10, R3, R3 - ADD R10, R6, R6 + ADD R13, R5, R15 + FMOVQ (R15), F0 + ADD R13, R3, R15 + FMOVQ F0, (R15) + ADD $0x10, R13, R13 + CMP R10, R13 + BLO copy_1 + ADD R10, R5, R5 + ADD R10, R3, R3 + ADD R10, R6, R6 // Malformed input if seq.mo > t+len(hist) || seq.mo > s.windowSize) check_offset: @@ -748,21 +732,19 @@ check_offset: BLO copy_4_small copy_4_loop: - VLD1 (R13), [V0.B16] - VST1 [V0.B16], (R3) - ADD $0x10, R13, R13 - ADD $0x10, R3, R3 - SUBS $0x10, R10, R10 - BHS copy_4_loop - ADD R10, R13, R13 - ADD $16, R13, R13 - ADD R10, R3, R3 - ADD $16, R3, R3 - ADD $-16, R13, R15 - VLD1 (R15), [V0.B16] - ADD $-16, R3, R15 - VST1 [V0.B16], (R15) - JMP copy_4_end + FMOVQ (R13), F0 + FMOVQ F0, (R3) + ADD $0x10, R13, R13 + ADD $0x10, R3, R3 + SUBS $0x10, R10, R10 + BHS copy_4_loop + ADD R10, R13, R13 + ADD $16, R13, R13 + ADD R10, R3, R3 + ADD $16, R3, R3 + FMOVQ -16(R13), F0 + FMOVQ F0, -16(R3) + JMP copy_4_end copy_4_small: CMP $0x03, R12 @@ -816,21 +798,19 @@ copy_all_from_history: BLO copy_5_small copy_5_loop: - VLD1 (R13), [V0.B16] - VST1 [V0.B16], (R3) - ADD $0x10, R13, R13 - ADD $0x10, R3, R3 - SUBS $0x10, R14, R14 - BHS copy_5_loop - ADD R14, R13, R13 - ADD $16, R13, R13 - ADD R14, R3, R3 - ADD $16, R3, R3 - ADD $-16, R13, R15 - VLD1 (R15), [V0.B16] - ADD $-16, R3, R15 - VST1 [V0.B16], (R15) - JMP copy_5_end + FMOVQ (R13), F0 + FMOVQ F0, (R3) + ADD $0x10, R13, R13 + ADD $0x10, R3, R3 + SUBS $0x10, R14, R14 + BHS copy_5_loop + ADD R14, R13, R13 + ADD $16, R13, R13 + ADD R14, R3, R3 + ADD $16, R3, R3 + FMOVQ -16(R13), F0 + FMOVQ F0, -16(R3) + JMP copy_5_end copy_5_small: CMP $0x03, R10 @@ -903,21 +883,20 @@ copy_match: ADD R12, R3, R3 copy_2: - VLD1 (R10), [V0.B16] - VST1 [V0.B16], (R11) - ADD $0x10, R10, R10 - ADD $0x10, R11, R11 - SUBS $0x10, R12, R12 - BHI copy_2 - JMP handle_loop + FMOVQ (R10), F0 + FMOVQ F0, (R11) + ADD $0x10, R10, R10 + ADD $0x10, R11, R11 + SUBS $0x10, R12, R12 + BHI copy_2 + JMP handle_loop // Copy overlapping match copy_overlapping_match: ADD R12, R6, R6 copy_slow_3: - MOVBU (R10), R16 - BFI $0, R16, $8, R11 + MOVBU (R10), R11 MOVB R11, (R3) ADD $1, R10, R10 ADD $1, R3, R3 @@ -1002,21 +981,19 @@ main_loop: BLO copy_1_small copy_1_loop: - VLD1 (R5), [V0.B16] - VST1 [V0.B16], (R3) - ADD $0x10, R5, R5 - ADD $0x10, R3, R3 - SUBS $0x10, R13, R13 - BHS copy_1_loop - ADD R13, R5, R5 - ADD $16, R5, R5 - ADD R13, R3, R3 - ADD $16, R3, R3 - ADD $-16, R5, R15 - VLD1 (R15), [V0.B16] - ADD $-16, R3, R15 - VST1 [V0.B16], (R15) - JMP copy_1_end + FMOVQ (R5), F0 + FMOVQ F0, (R3) + ADD $0x10, R5, R5 + ADD $0x10, R3, R3 + SUBS $0x10, R13, R13 + BHS copy_1_loop + ADD R13, R5, R5 + ADD $16, R5, R5 + ADD R13, R3, R3 + ADD $16, R3, R3 + FMOVQ -16(R5), F0 + FMOVQ F0, -16(R3) + JMP copy_1_end copy_1_small: CMP $0x03, R10 @@ -1094,21 +1071,19 @@ check_offset: BLO copy_4_small copy_4_loop: - VLD1 (R13), [V0.B16] - VST1 [V0.B16], (R3) - ADD $0x10, R13, R13 - ADD $0x10, R3, R3 - SUBS $0x10, R10, R10 - BHS copy_4_loop - ADD R10, R13, R13 - ADD $16, R13, R13 - ADD R10, R3, R3 - ADD $16, R3, R3 - ADD $-16, R13, R15 - VLD1 (R15), [V0.B16] - ADD $-16, R3, R15 - VST1 [V0.B16], (R15) - JMP copy_4_end + FMOVQ (R13), F0 + FMOVQ F0, (R3) + ADD $0x10, R13, R13 + ADD $0x10, R3, R3 + SUBS $0x10, R10, R10 + BHS copy_4_loop + ADD R10, R13, R13 + ADD $16, R13, R13 + ADD R10, R3, R3 + ADD $16, R3, R3 + FMOVQ -16(R13), F0 + FMOVQ F0, -16(R3) + JMP copy_4_end copy_4_small: CMP $0x03, R12 @@ -1162,21 +1137,19 @@ copy_all_from_history: BLO copy_5_small copy_5_loop: - VLD1 (R13), [V0.B16] - VST1 [V0.B16], (R3) - ADD $0x10, R13, R13 - ADD $0x10, R3, R3 - SUBS $0x10, R14, R14 - BHS copy_5_loop - ADD R14, R13, R13 - ADD $16, R13, R13 - ADD R14, R3, R3 - ADD $16, R3, R3 - ADD $-16, R13, R15 - VLD1 (R15), [V0.B16] - ADD $-16, R3, R15 - VST1 [V0.B16], (R15) - JMP copy_5_end + FMOVQ (R13), F0 + FMOVQ F0, (R3) + ADD $0x10, R13, R13 + ADD $0x10, R3, R3 + SUBS $0x10, R14, R14 + BHS copy_5_loop + ADD R14, R13, R13 + ADD $16, R13, R13 + ADD R14, R3, R3 + ADD $16, R3, R3 + FMOVQ -16(R13), F0 + FMOVQ F0, -16(R3) + JMP copy_5_end copy_5_small: CMP $0x03, R10 @@ -1250,21 +1223,19 @@ copy_match: BLO copy_2_small copy_2_loop: - VLD1 (R10), [V0.B16] - VST1 [V0.B16], (R3) - ADD $0x10, R10, R10 - ADD $0x10, R3, R3 - SUBS $0x10, R11, R11 - BHS copy_2_loop - ADD R11, R10, R10 - ADD $16, R10, R10 - ADD R11, R3, R3 - ADD $16, R3, R3 - ADD $-16, R10, R15 - VLD1 (R15), [V0.B16] - ADD $-16, R3, R15 - VST1 [V0.B16], (R15) - JMP copy_2_end + FMOVQ (R10), F0 + FMOVQ F0, (R3) + ADD $0x10, R10, R10 + ADD $0x10, R3, R3 + SUBS $0x10, R11, R11 + BHS copy_2_loop + ADD R11, R10, R10 + ADD $16, R10, R10 + ADD R11, R3, R3 + ADD $16, R3, R3 + FMOVQ -16(R10), F0 + FMOVQ F0, -16(R3) + JMP copy_2_end copy_2_small: CMP $0x03, R12 @@ -1326,8 +1297,7 @@ copy_overlapping_match: ADD R12, R6, R6 copy_slow_3: - MOVBU (R10), R16 - BFI $0, R16, $8, R11 + MOVBU (R10), R11 MOVB R11, (R3) ADD $1, R10, R10 ADD $1, R3, R3 @@ -1453,8 +1423,7 @@ sequenceDecs_decodeSync_amd64_fill_end: MOVD R3, R1 MOVD R2, R13 LSL R1, R13, R13 - UBFX $8, R0, $8, R16 - BFI $0, R16, $8, R1 + UBFX $8, R0, $8, R1 LSR $0x20, R0, R0 TST R1, R1 BEQ sequenceDecs_decodeSync_amd64_of_update_zero @@ -1475,8 +1444,7 @@ sequenceDecs_decodeSync_amd64_of_update_zero: MOVD R3, R1 MOVD R2, R13 LSL R1, R13, R13 - UBFX $8, R0, $8, R16 - BFI $0, R16, $8, R1 + UBFX $8, R0, $8, R1 LSR $0x20, R0, R0 TST R1, R1 BEQ sequenceDecs_decodeSync_amd64_ml_update_zero @@ -1526,8 +1494,7 @@ sequenceDecs_decodeSync_amd64_fill_2_end: MOVD R3, R1 MOVD R2, R13 LSL R1, R13, R13 - UBFX $8, R0, $8, R16 - BFI $0, R16, $8, R1 + UBFX $8, R0, $8, R1 LSR $0x20, R0, R0 TST R1, R1 BEQ sequenceDecs_decodeSync_amd64_ll_update_zero @@ -1544,14 +1511,12 @@ sequenceDecs_decodeSync_amd64_ll_update_zero: MOVD R0, 32(RSP) // Fill bitreader for state updates - MOVD R12, 8(RSP) - MOVD R8, R0 - LSR $0x08, R0, R0 - MOVBU R0, R0 - MOVD ctx+16(FP), R1 - MOVD 96(R1), R16 - CMP $0x00, R16 - BEQ sequenceDecs_decodeSync_amd64_skip_update + MOVD R12, 8(RSP) + UBFX $8, R8, $8, R0 + MOVD ctx+16(FP), R1 + MOVD 96(R1), R16 + CMP $0x00, R16 + BEQ sequenceDecs_decodeSync_amd64_skip_update // Update Literal Length State MOVBU R6, R12 @@ -1562,7 +1527,7 @@ sequenceDecs_decodeSync_amd64_ll_update_zero: NEG R1, R16 ROR R16, R13, R13 MOVD $0x00000001, R14 - BFI $0, R12, $8, R1 + MOVBU R12, R1 LSLW R1, R14, R14 SUBW $1, R14, R14 AND R14, R13, R13 @@ -1571,8 +1536,7 @@ sequenceDecs_decodeSync_amd64_ll_update_zero: // Load ctx.llTable MOVD ctx+16(FP), R1 MOVD (R1), R1 - ADD R6<<3, R1, R15 - MOVD (R15), R6 + MOVD (R1)(R6<<3), R6 // Update Match Length State MOVBU R7, R12 @@ -1583,7 +1547,7 @@ sequenceDecs_decodeSync_amd64_ll_update_zero: NEG R1, R16 ROR R16, R13, R13 MOVD $0x00000001, R14 - BFI $0, R12, $8, R1 + MOVBU R12, R1 LSLW R1, R14, R14 SUBW $1, R14, R14 AND R14, R13, R13 @@ -1592,8 +1556,7 @@ sequenceDecs_decodeSync_amd64_ll_update_zero: // Load ctx.mlTable MOVD ctx+16(FP), R1 MOVD 24(R1), R1 - ADD R7<<3, R1, R15 - MOVD (R15), R7 + MOVD (R1)(R7<<3), R7 // Update Offset State MOVBU R8, R12 @@ -1604,7 +1567,7 @@ sequenceDecs_decodeSync_amd64_ll_update_zero: NEG R1, R16 ROR R16, R13, R13 MOVD $0x00000001, R14 - BFI $0, R12, $8, R1 + MOVBU R12, R1 LSLW R1, R14, R14 SUBW $1, R14, R14 AND R14, R13, R13 @@ -1613,21 +1576,18 @@ sequenceDecs_decodeSync_amd64_ll_update_zero: // Load ctx.ofTable MOVD ctx+16(FP), R1 MOVD 48(R1), R1 - ADD R8<<3, R1, R15 - MOVD (R15), R8 + MOVD (R1)(R8<<3), R8 sequenceDecs_decodeSync_amd64_skip_update: // Adjust offset - MOVD s+0(FP), R1 - MOVD 16(RSP), R12 - CMP $0x01, R0 - BLS sequenceDecs_decodeSync_amd64_adjust_offsetB_1_or_0 - ADD $144, R1, R15 - VLD1 (R15), [V0.B16] - MOVD R12, 144(R1) - ADD $152, R1, R15 - VST1 [V0.B16], (R15) - JMP sequenceDecs_decodeSync_amd64_after_adjust + MOVD s+0(FP), R1 + MOVD 16(RSP), R12 + CMP $0x01, R0 + BLS sequenceDecs_decodeSync_amd64_adjust_offsetB_1_or_0 + FMOVQ 144(R1), F0 + MOVD R12, 144(R1) + FMOVQ F0, 152(R1) + JMP sequenceDecs_decodeSync_amd64_after_adjust sequenceDecs_decodeSync_amd64_adjust_offsetB_1_or_0: MOVD 32(RSP), R16 @@ -1709,16 +1669,16 @@ sequenceDecs_decodeSync_amd64_match_len_ofs_ok: MOVD $0, R13 copy_1: - ADD R13, R10, R15 - VLD1 (R15), [V0.B16] - ADD R13, R9, R15 - VST1 [V0.B16], (R15) - ADD $0x10, R13, R13 - CMP R0, R13 - BLO copy_1 - ADD R0, R10, R10 - ADD R0, R9, R9 - ADD R0, R11, R11 + ADD R13, R10, R15 + FMOVQ (R15), F0 + ADD R13, R9, R15 + FMOVQ F0, (R15) + ADD $0x10, R13, R13 + CMP R0, R13 + BLO copy_1 + ADD R0, R10, R10 + ADD R0, R9, R9 + ADD R0, R11, R11 // Malformed input if seq.mo > t+len(hist) || seq.mo > s.windowSize) check_offset: @@ -1744,21 +1704,19 @@ check_offset: BLO copy_4_small copy_4_loop: - VLD1 (R13), [V0.B16] - VST1 [V0.B16], (R9) - ADD $0x10, R13, R13 - ADD $0x10, R9, R9 - SUBS $0x10, R0, R0 - BHS copy_4_loop - ADD R0, R13, R13 - ADD $16, R13, R13 - ADD R0, R9, R9 - ADD $16, R9, R9 - ADD $-16, R13, R15 - VLD1 (R15), [V0.B16] - ADD $-16, R9, R15 - VST1 [V0.B16], (R15) - JMP copy_4_end + FMOVQ (R13), F0 + FMOVQ F0, (R9) + ADD $0x10, R13, R13 + ADD $0x10, R9, R9 + SUBS $0x10, R0, R0 + BHS copy_4_loop + ADD R0, R13, R13 + ADD $16, R13, R13 + ADD R0, R9, R9 + ADD $16, R9, R9 + FMOVQ -16(R13), F0 + FMOVQ F0, -16(R9) + JMP copy_4_end copy_4_small: CMP $0x03, R12 @@ -1809,21 +1767,19 @@ copy_all_from_history: BLO copy_5_small copy_5_loop: - VLD1 (R13), [V0.B16] - VST1 [V0.B16], (R9) - ADD $0x10, R13, R13 - ADD $0x10, R9, R9 - SUBS $0x10, R14, R14 - BHS copy_5_loop - ADD R14, R13, R13 - ADD $16, R13, R13 - ADD R14, R9, R9 - ADD $16, R9, R9 - ADD $-16, R13, R15 - VLD1 (R15), [V0.B16] - ADD $-16, R9, R15 - VST1 [V0.B16], (R15) - JMP copy_5_end + FMOVQ (R13), F0 + FMOVQ F0, (R9) + ADD $0x10, R13, R13 + ADD $0x10, R9, R9 + SUBS $0x10, R14, R14 + BHS copy_5_loop + ADD R14, R13, R13 + ADD $16, R13, R13 + ADD R14, R9, R9 + ADD $16, R9, R9 + FMOVQ -16(R13), F0 + FMOVQ F0, -16(R9) + JMP copy_5_end copy_5_small: CMP $0x03, R0 @@ -1896,21 +1852,20 @@ copy_match: ADD R12, R9, R9 copy_2: - VLD1 (R0), [V0.B16] - VST1 [V0.B16], (R1) - ADD $0x10, R0, R0 - ADD $0x10, R1, R1 - SUBS $0x10, R12, R12 - BHI copy_2 - JMP handle_loop + FMOVQ (R0), F0 + FMOVQ F0, (R1) + ADD $0x10, R0, R0 + ADD $0x10, R1, R1 + SUBS $0x10, R12, R12 + BHI copy_2 + JMP handle_loop // Copy overlapping match copy_overlapping_match: ADD R12, R11, R11 copy_slow_3: - MOVBU (R0), R16 - BFI $0, R16, $8, R1 + MOVBU (R0), R1 MOVB R1, (R9) ADD $1, R0, R0 ADD $1, R9, R9 @@ -2078,8 +2033,7 @@ sequenceDecs_decodeSync_safe_amd64_fill_end: MOVD R3, R1 MOVD R2, R13 LSL R1, R13, R13 - UBFX $8, R0, $8, R16 - BFI $0, R16, $8, R1 + UBFX $8, R0, $8, R1 LSR $0x20, R0, R0 TST R1, R1 BEQ sequenceDecs_decodeSync_safe_amd64_of_update_zero @@ -2100,8 +2054,7 @@ sequenceDecs_decodeSync_safe_amd64_of_update_zero: MOVD R3, R1 MOVD R2, R13 LSL R1, R13, R13 - UBFX $8, R0, $8, R16 - BFI $0, R16, $8, R1 + UBFX $8, R0, $8, R1 LSR $0x20, R0, R0 TST R1, R1 BEQ sequenceDecs_decodeSync_safe_amd64_ml_update_zero @@ -2151,8 +2104,7 @@ sequenceDecs_decodeSync_safe_amd64_fill_2_end: MOVD R3, R1 MOVD R2, R13 LSL R1, R13, R13 - UBFX $8, R0, $8, R16 - BFI $0, R16, $8, R1 + UBFX $8, R0, $8, R1 LSR $0x20, R0, R0 TST R1, R1 BEQ sequenceDecs_decodeSync_safe_amd64_ll_update_zero @@ -2169,14 +2121,12 @@ sequenceDecs_decodeSync_safe_amd64_ll_update_zero: MOVD R0, 32(RSP) // Fill bitreader for state updates - MOVD R12, 8(RSP) - MOVD R8, R0 - LSR $0x08, R0, R0 - MOVBU R0, R0 - MOVD ctx+16(FP), R1 - MOVD 96(R1), R16 - CMP $0x00, R16 - BEQ sequenceDecs_decodeSync_safe_amd64_skip_update + MOVD R12, 8(RSP) + UBFX $8, R8, $8, R0 + MOVD ctx+16(FP), R1 + MOVD 96(R1), R16 + CMP $0x00, R16 + BEQ sequenceDecs_decodeSync_safe_amd64_skip_update // Update Literal Length State MOVBU R6, R12 @@ -2187,7 +2137,7 @@ sequenceDecs_decodeSync_safe_amd64_ll_update_zero: NEG R1, R16 ROR R16, R13, R13 MOVD $0x00000001, R14 - BFI $0, R12, $8, R1 + MOVBU R12, R1 LSLW R1, R14, R14 SUBW $1, R14, R14 AND R14, R13, R13 @@ -2196,8 +2146,7 @@ sequenceDecs_decodeSync_safe_amd64_ll_update_zero: // Load ctx.llTable MOVD ctx+16(FP), R1 MOVD (R1), R1 - ADD R6<<3, R1, R15 - MOVD (R15), R6 + MOVD (R1)(R6<<3), R6 // Update Match Length State MOVBU R7, R12 @@ -2208,7 +2157,7 @@ sequenceDecs_decodeSync_safe_amd64_ll_update_zero: NEG R1, R16 ROR R16, R13, R13 MOVD $0x00000001, R14 - BFI $0, R12, $8, R1 + MOVBU R12, R1 LSLW R1, R14, R14 SUBW $1, R14, R14 AND R14, R13, R13 @@ -2217,8 +2166,7 @@ sequenceDecs_decodeSync_safe_amd64_ll_update_zero: // Load ctx.mlTable MOVD ctx+16(FP), R1 MOVD 24(R1), R1 - ADD R7<<3, R1, R15 - MOVD (R15), R7 + MOVD (R1)(R7<<3), R7 // Update Offset State MOVBU R8, R12 @@ -2229,7 +2177,7 @@ sequenceDecs_decodeSync_safe_amd64_ll_update_zero: NEG R1, R16 ROR R16, R13, R13 MOVD $0x00000001, R14 - BFI $0, R12, $8, R1 + MOVBU R12, R1 LSLW R1, R14, R14 SUBW $1, R14, R14 AND R14, R13, R13 @@ -2238,21 +2186,18 @@ sequenceDecs_decodeSync_safe_amd64_ll_update_zero: // Load ctx.ofTable MOVD ctx+16(FP), R1 MOVD 48(R1), R1 - ADD R8<<3, R1, R15 - MOVD (R15), R8 + MOVD (R1)(R8<<3), R8 sequenceDecs_decodeSync_safe_amd64_skip_update: // Adjust offset - MOVD s+0(FP), R1 - MOVD 16(RSP), R12 - CMP $0x01, R0 - BLS sequenceDecs_decodeSync_safe_amd64_adjust_offsetB_1_or_0 - ADD $144, R1, R15 - VLD1 (R15), [V0.B16] - MOVD R12, 144(R1) - ADD $152, R1, R15 - VST1 [V0.B16], (R15) - JMP sequenceDecs_decodeSync_safe_amd64_after_adjust + MOVD s+0(FP), R1 + MOVD 16(RSP), R12 + CMP $0x01, R0 + BLS sequenceDecs_decodeSync_safe_amd64_adjust_offsetB_1_or_0 + FMOVQ 144(R1), F0 + MOVD R12, 144(R1) + FMOVQ F0, 152(R1) + JMP sequenceDecs_decodeSync_safe_amd64_after_adjust sequenceDecs_decodeSync_safe_amd64_adjust_offsetB_1_or_0: MOVD 32(RSP), R16 @@ -2335,21 +2280,19 @@ sequenceDecs_decodeSync_safe_amd64_match_len_ofs_ok: BLO copy_1_small copy_1_loop: - VLD1 (R10), [V0.B16] - VST1 [V0.B16], (R9) - ADD $0x10, R10, R10 - ADD $0x10, R9, R9 - SUBS $0x10, R13, R13 - BHS copy_1_loop - ADD R13, R10, R10 - ADD $16, R10, R10 - ADD R13, R9, R9 - ADD $16, R9, R9 - ADD $-16, R10, R15 - VLD1 (R15), [V0.B16] - ADD $-16, R9, R15 - VST1 [V0.B16], (R15) - JMP copy_1_end + FMOVQ (R10), F0 + FMOVQ F0, (R9) + ADD $0x10, R10, R10 + ADD $0x10, R9, R9 + SUBS $0x10, R13, R13 + BHS copy_1_loop + ADD R13, R10, R10 + ADD $16, R10, R10 + ADD R13, R9, R9 + ADD $16, R9, R9 + FMOVQ -16(R10), F0 + FMOVQ F0, -16(R9) + JMP copy_1_end copy_1_small: CMP $0x03, R0 @@ -2430,21 +2373,19 @@ check_offset: BLO copy_4_small copy_4_loop: - VLD1 (R13), [V0.B16] - VST1 [V0.B16], (R9) - ADD $0x10, R13, R13 - ADD $0x10, R9, R9 - SUBS $0x10, R0, R0 - BHS copy_4_loop - ADD R0, R13, R13 - ADD $16, R13, R13 - ADD R0, R9, R9 - ADD $16, R9, R9 - ADD $-16, R13, R15 - VLD1 (R15), [V0.B16] - ADD $-16, R9, R15 - VST1 [V0.B16], (R15) - JMP copy_4_end + FMOVQ (R13), F0 + FMOVQ F0, (R9) + ADD $0x10, R13, R13 + ADD $0x10, R9, R9 + SUBS $0x10, R0, R0 + BHS copy_4_loop + ADD R0, R13, R13 + ADD $16, R13, R13 + ADD R0, R9, R9 + ADD $16, R9, R9 + FMOVQ -16(R13), F0 + FMOVQ F0, -16(R9) + JMP copy_4_end copy_4_small: CMP $0x03, R12 @@ -2495,21 +2436,19 @@ copy_all_from_history: BLO copy_5_small copy_5_loop: - VLD1 (R13), [V0.B16] - VST1 [V0.B16], (R9) - ADD $0x10, R13, R13 - ADD $0x10, R9, R9 - SUBS $0x10, R14, R14 - BHS copy_5_loop - ADD R14, R13, R13 - ADD $16, R13, R13 - ADD R14, R9, R9 - ADD $16, R9, R9 - ADD $-16, R13, R15 - VLD1 (R15), [V0.B16] - ADD $-16, R9, R15 - VST1 [V0.B16], (R15) - JMP copy_5_end + FMOVQ (R13), F0 + FMOVQ F0, (R9) + ADD $0x10, R13, R13 + ADD $0x10, R9, R9 + SUBS $0x10, R14, R14 + BHS copy_5_loop + ADD R14, R13, R13 + ADD $16, R13, R13 + ADD R14, R9, R9 + ADD $16, R9, R9 + FMOVQ -16(R13), F0 + FMOVQ F0, -16(R9) + JMP copy_5_end copy_5_small: CMP $0x03, R0 @@ -2583,21 +2522,19 @@ copy_match: BLO copy_2_small copy_2_loop: - VLD1 (R0), [V0.B16] - VST1 [V0.B16], (R9) - ADD $0x10, R0, R0 - ADD $0x10, R9, R9 - SUBS $0x10, R1, R1 - BHS copy_2_loop - ADD R1, R0, R0 - ADD $16, R0, R0 - ADD R1, R9, R9 - ADD $16, R9, R9 - ADD $-16, R0, R15 - VLD1 (R15), [V0.B16] - ADD $-16, R9, R15 - VST1 [V0.B16], (R15) - JMP copy_2_end + FMOVQ (R0), F0 + FMOVQ F0, (R9) + ADD $0x10, R0, R0 + ADD $0x10, R9, R9 + SUBS $0x10, R1, R1 + BHS copy_2_loop + ADD R1, R0, R0 + ADD $16, R0, R0 + ADD R1, R9, R9 + ADD $16, R9, R9 + FMOVQ -16(R0), F0 + FMOVQ F0, -16(R9) + JMP copy_2_end copy_2_small: CMP $0x03, R12 @@ -2659,8 +2596,7 @@ copy_overlapping_match: ADD R12, R11, R11 copy_slow_3: - MOVBU (R0), R16 - BFI $0, R16, $8, R1 + MOVBU (R0), R1 MOVB R1, (R9) ADD $1, R0, R0 ADD $1, R9, R9 diff --git a/vendor/modules.txt b/vendor/modules.txt index 501237c5a1e8..d1e1f791be17 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -145,8 +145,8 @@ github.com/grpc-ecosystem/grpc-gateway/v2/utilities # github.com/inconshreveable/mousetrap v1.1.0 ## explicit; go 1.18 github.com/inconshreveable/mousetrap -# github.com/klauspost/compress v1.19.2 -## explicit; go 1.24 +# github.com/klauspost/compress v1.20.0 +## explicit; go 1.25 github.com/klauspost/compress github.com/klauspost/compress/fse github.com/klauspost/compress/huff0 From 07339fa78f92d29941a2a02380a1948e2e7b632d Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Thu, 1 Oct 2026 19:34:03 +0200 Subject: [PATCH 2/2] vendor: github.com/klauspost/compress v1.20.1 full diff: https://github.com/klauspost/compress/compare/v1.20.0...v1.20.1 Signed-off-by: Sebastiaan van Stijn --- vendor.mod | 2 +- vendor.sum | 4 +- .../klauspost/compress/huff0/bitreader.go | 63 + .../compress/huff0/decompress_amd64.go | 95 +- .../compress/huff0/decompress_amd64.s | 4092 +++++++++++++---- .../compress/huff0/decompress_arm64.go | 43 +- .../compress/huff0/decompress_arm64.s | 2309 +++++++--- .../compress/huff0/decompress_asm.go | 98 +- .../klauspost/compress/zstd/blockdec.go | 3 + .../klauspost/compress/zstd/decoder.go | 125 +- .../klauspost/compress/zstd/framedec.go | 11 +- .../compress/zstd/fse_decoder_arm64.s | 10 +- .../klauspost/compress/zstd/seqdec.go | 23 + .../klauspost/compress/zstd/seqdec_amd64.s | 1669 +++---- .../klauspost/compress/zstd/seqdec_arm64.s | 1125 +++-- .../klauspost/compress/zstd/seqdec_asm.go | 7 +- vendor/modules.txt | 2 +- 17 files changed, 6670 insertions(+), 3011 deletions(-) diff --git a/vendor.mod b/vendor.mod index e6f777058e62..f334409c135e 100644 --- a/vendor.mod +++ b/vendor.mod @@ -87,7 +87,7 @@ require ( github.com/gorilla/mux v1.8.1 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.30.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect - github.com/klauspost/compress v1.20.0 // indirect + github.com/klauspost/compress v1.20.1 // indirect github.com/moby/docker-image-spec v1.3.1 // indirect github.com/moby/sys/user v0.4.1 // indirect github.com/moby/sys/userns v0.2.1 // indirect diff --git a/vendor.sum b/vendor.sum index a02c98915b57..d1581457c7cb 100644 --- a/vendor.sum +++ b/vendor.sum @@ -78,8 +78,8 @@ github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA= -github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI= +github.com/klauspost/compress v1.20.1 h1:T7kKElXUMXrUJ2E9QhQhxFtcK5rPyLdsGZvdbLMPdiQ= +github.com/klauspost/compress v1.20.1/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/mattn/go-runewidth v0.0.29 h1:3oGF3R/S2N9DQ3ptftzVIvg2eicmojCzlwBEmqEPDfQ= diff --git a/vendor/github.com/klauspost/compress/huff0/bitreader.go b/vendor/github.com/klauspost/compress/huff0/bitreader.go index bfc7a523dee7..faaa418a7599 100644 --- a/vendor/github.com/klauspost/compress/huff0/bitreader.go +++ b/vendor/github.com/klauspost/compress/huff0/bitreader.go @@ -9,6 +9,7 @@ import ( "errors" "fmt" "io" + "math/bits" "github.com/klauspost/compress/internal/le" ) @@ -210,6 +211,68 @@ func (b *bitReaderShifted) remaining() uint { return b.off*8 + uint(64-b.bitsRead) } +// canUseAsm reports whether the reader has a full 8-byte window ahead of +// its read pointer, which the Decompress4X asm loops need to take over. +func (b *bitReaderShifted) canUseAsm() bool { + return b.off >= 8 +} + +// prepareForAsm establishes the invariant the Decompress4X asm loops rely +// on: value == load64(in[off:off+8]) << bitsRead with bitsRead <= 7, so +// that a full group of symbols can never shift their sentinel bit out of +// the container. init leaves bitsRead == 8 when the final byte of the +// stream is exactly 0x01; that whole byte is consumed, so the same position +// is the window one byte lower with nothing consumed. Requires canUseAsm. +func (b *bitReaderShifted) prepareForAsm() { + if b.bitsRead >= 8 { + b.off-- + b.value = le.Load64(b.in, b.off) + b.bitsRead -= 8 + } +} + +// restoreFromAsm converts the state left behind by the Decompress4X asm +// loops back to the invariant the Go code relies on: value holds the 8 +// bytes at in[off:off+8] shifted left by bitsRead, with the low bitsRead +// bits zero. +// +// The asm keeps a sentinel bit in value just below the unread bits, so its +// trailing zero count is the number of consumed bits. The sentinel is ORed +// over the lowest bit of the window, which the loop never consumes before +// re-reading memory, so the window is re-read here too rather than taken +// from value. The asm reports off as the signed distance from the start of +// the stream, and it can be negative: a reload always reads a whole 8-byte +// window, so a stream that is nearly drained ends with up to 7 bytes of the +// previous stream (or the jump table) below its start inside the window. +// Those bytes sit below the stream's own bits and count as consumed. +// Anything further below, or more bits consumed than the stream holds, is +// corruption. +func (b *bitReaderShifted) restoreFromAsm() error { + off := int(b.off) + consumed := uint(bits.TrailingZeros64(b.value)) + if off < 0 { + if off < -7 { + return errors.New("corruption detected: stream underrun") + } + consumed += uint(-off) * 8 + if consumed > 64 { + return errors.New("corruption detected: stream underrun") + } + off = 0 + } + if off+8 > len(b.in) { + return errors.New("corruption detected: stream overrun") + } + b.off = uint(off) + b.bitsRead = uint8(consumed) + if consumed >= 64 { + b.value = 0 + } else { + b.value = le.Load64(b.in, b.off) << consumed + } + return nil +} + // close the bitstream and returns an error if out-of-buffer reads occurred. func (b *bitReaderShifted) close() error { // Release reference. diff --git a/vendor/github.com/klauspost/compress/huff0/decompress_amd64.go b/vendor/github.com/klauspost/compress/huff0/decompress_amd64.go index 7035d656d991..e651f30ecbd8 100644 --- a/vendor/github.com/klauspost/compress/huff0/decompress_amd64.go +++ b/vendor/github.com/klauspost/compress/huff0/decompress_amd64.go @@ -8,36 +8,99 @@ import ( ) // decompress4x_main_loop_amd64 is an x86 assembler implementation -// of Decompress4X when tablelog > 8. +// of Decompress4X when tablelog > 8, decoding fastSymbols symbols per +// stream between bit container reloads. // //go:noescape func decompress4x_main_loop_amd64(ctx *decompress4xContext) // decompress4x_8b_main_loop_amd64 is an x86 assembler implementation -// of Decompress4X when tablelog <= 8 which decodes 4 entries -// per loop. +// of Decompress4X when tablelog <= 8, decoding fast8bSymbols symbols +// per stream between bit container reloads. // //go:noescape func decompress4x_8b_main_loop_amd64(ctx *decompress4xContext) +// decompress4x_4b_main_loop_amd64 is an x86 assembler implementation +// of Decompress4X when tablelog <= 4, decoding fast4bSymbols symbols +// per stream between bit container reloads. +// +//go:noescape +func decompress4x_4b_main_loop_amd64(ctx *decompress4xContext) + +// decompress4x_4b_main_loop_bmi2 is the BMI2 twin of decompress4x_4b_main_loop_amd64. +// +//go:noescape +func decompress4x_4b_main_loop_bmi2(ctx *decompress4xContext) + +// decompress4x_main_loop_bmi2 is the BMI2 twin of decompress4x_main_loop_amd64. +// +//go:noescape +func decompress4x_main_loop_bmi2(ctx *decompress4xContext) + +// decompress4x_8b_main_loop_bmi2 is the BMI2 twin of decompress4x_8b_main_loop_amd64. +// +//go:noescape +func decompress4x_8b_main_loop_bmi2(ctx *decompress4xContext) + // decompress1x_main_loop_amd64 is an x86 assembler implementation -// of Decompress1X when tablelog > 8. +// of Decompress1X when tablelog > 8, decoding fastSymbols symbols +// between bit container reloads. // //go:noescape func decompress1x_main_loop_amd64(ctx *decompress1xContext) -// decompress1x_main_loop_bmi2 is an x86 with BMI2 assembler implementation -// of Decompress1X when tablelog > 8. +// decompress1x_8b_main_loop_amd64 is an x86 assembler implementation +// of Decompress1X when tablelog <= 8, decoding fast8bSymbols symbols +// between bit container reloads. +// +//go:noescape +func decompress1x_8b_main_loop_amd64(ctx *decompress1xContext) + +// decompress1x_4b_main_loop_amd64 is an x86 assembler implementation +// of Decompress1X when tablelog <= 4, decoding fast4bSymbols symbols +// between bit container reloads. +// +//go:noescape +func decompress1x_4b_main_loop_amd64(ctx *decompress1xContext) + +// decompress1x_main_loop_bmi2 is the BMI2 twin of decompress1x_main_loop_amd64. // //go:noescape func decompress1x_main_loop_bmi2(ctx *decompress1xContext) +// decompress1x_8b_main_loop_bmi2 is the BMI2 twin of decompress1x_8b_main_loop_amd64. +// +//go:noescape +func decompress1x_8b_main_loop_bmi2(ctx *decompress1xContext) + +// decompress1x_4b_main_loop_bmi2 is the BMI2 twin of decompress1x_4b_main_loop_amd64. +// +//go:noescape +func decompress1x_4b_main_loop_bmi2(ctx *decompress1xContext) + func decompress4x_main_loop_asm(ctx *decompress4xContext) { - decompress4x_main_loop_amd64(ctx) + if cpuinfo.HasBMI2() { + decompress4x_main_loop_bmi2(ctx) + } else { + decompress4x_main_loop_amd64(ctx) + } } func decompress4x_8b_main_loop_asm(ctx *decompress4xContext) { - decompress4x_8b_main_loop_amd64(ctx) + if cpuinfo.HasBMI2() { + decompress4x_8b_main_loop_bmi2(ctx) + } else { + decompress4x_8b_main_loop_amd64(ctx) + } +} + +func decompress4x_4b_main_loop_asm(ctx *decompress4xContext) { + if cpuinfo.HasBMI2() { + decompress4x_4b_main_loop_bmi2(ctx) + } else { + decompress4x_4b_main_loop_amd64(ctx) + } } func decompress1x_main_loop_asm(ctx *decompress1xContext) { @@ -47,3 +110,19 @@ func decompress1x_main_loop_asm(ctx *decompress1xContext) { decompress1x_main_loop_amd64(ctx) } } + +func decompress1x_8b_main_loop_asm(ctx *decompress1xContext) { + if cpuinfo.HasBMI2() { + decompress1x_8b_main_loop_bmi2(ctx) + } else { + decompress1x_8b_main_loop_amd64(ctx) + } +} + +func decompress1x_4b_main_loop_asm(ctx *decompress1xContext) { + if cpuinfo.HasBMI2() { + decompress1x_4b_main_loop_bmi2(ctx) + } else { + decompress1x_4b_main_loop_amd64(ctx) + } +} diff --git a/vendor/github.com/klauspost/compress/huff0/decompress_amd64.s b/vendor/github.com/klauspost/compress/huff0/decompress_amd64.s index 920628c51422..f072c18f04ed 100644 --- a/vendor/github.com/klauspost/compress/huff0/decompress_amd64.s +++ b/vendor/github.com/klauspost/compress/huff0/decompress_amd64.s @@ -3,828 +3,3384 @@ //go:build !appengine && !noasm && gc // func decompress4x_main_loop_amd64(ctx *decompress4xContext) +// Requires: BMI, CMOV TEXT ·decompress4x_main_loop_amd64(SB), $0-8 - // Preload values - MOVQ ctx+0(FP), AX - MOVBQZX 8(AX), DI - MOVQ 16(AX), BX - MOVQ 48(AX), SI - MOVQ 24(AX), R8 - MOVQ 32(AX), R9 - MOVQ (AX), R10 - - // Main loop -main_loop: - XORL DX, DX - CMPQ BX, SI - SETGE DL - - // br0.fillFast32() - MOVQ 32(R10), R11 - MOVBQZX 40(R10), R12 - CMPQ R12, $0x20 - JBE skip_fill0 - MOVQ 24(R10), AX - SUBQ $0x20, R12 - SUBQ $0x04, AX - MOVQ (R10), R13 - - // b.value |= uint64(low) << (b.bitsRead & 63) - MOVL (AX)(R13*1), R13 - MOVQ R12, CX - SHLQ CL, R13 - MOVQ AX, 24(R10) - ORQ R13, R11 - - // exhausted += (br0.off < 4) - CMPQ AX, $0x04 - ADCB $+0, DL - -skip_fill0: - // val0 := br0.peekTopBits(peekBits) - MOVQ R11, R13 - MOVQ DI, CX - SHRQ CL, R13 - - // v0 := table[val0&mask] - MOVWQZX (R9)(R13*2), CX + MOVQ ctx+0(FP), AX - // br0.advance(uint8(v0.entry) - MOVBLZX CH, AX + // Preload values + MOVBQZX 8(AX), DX + MOVQ 32(AX), BX + MOVQ 16(AX), SI + MOVQ 24(AX), CX + LEAQ (SI)(CX*1), R8 + LEAQ (SI)(CX*2), R10 + LEAQ (CX)(CX*2), R12 + ADDQ SI, R12 + + // Convert each bit reader to sentinel form: bits = value | 1<> 8) - // out[id * dstEvery + 1] = uint8(v1.entry >> 8) - MOVW AX, (BX) - - // update the bitreader structure - MOVQ R11, 32(R10) - MOVB R12, 40(R10) - - // br1.fillFast32() - MOVQ 80(R10), R11 - MOVBQZX 88(R10), R12 - CMPQ R12, $0x20 - JBE skip_fill1 - MOVQ 72(R10), AX - SUBQ $0x20, R12 - SUBQ $0x04, AX - MOVQ 48(R10), R13 - - // b.value |= uint64(low) << (b.bitsRead & 63) - MOVL (AX)(R13*1), R13 - MOVQ R12, CX - SHLQ CL, R13 - MOVQ AX, 72(R10) - ORQ R13, R11 - - // exhausted += (br1.off < 4) - CMPQ AX, $0x04 - ADCB $+0, DL - -skip_fill1: - // val0 := br1.peekTopBits(peekBits) - MOVQ R11, R13 - MOVQ DI, CX - SHRQ CL, R13 - - // v0 := table[val0&mask] - MOVWQZX (R9)(R13*2), CX - - // br1.advance(uint8(v0.entry) - MOVBLZX CH, AX + SHRQ $0x08, CX + MOVB CL, (R10) + + // stream 3, symbol 0 + MOVQ DX, CX + MOVQ R13, R14 + SHRQ CL, R14 + MOVWQZX (BX)(R14*2), CX + SHLQ CL, R13 + SHRQ $0x08, CX + MOVB CL, (R12) + + // stream 0, symbol 1 + MOVQ DX, CX + MOVQ DI, R14 + SHRQ CL, R14 + MOVWQZX (BX)(R14*2), CX + SHLQ CL, DI + SHRQ $0x08, CX + MOVB CL, 1(SI) + + // stream 1, symbol 1 + MOVQ DX, CX + MOVQ R9, R14 + SHRQ CL, R14 + MOVWQZX (BX)(R14*2), CX + SHLQ CL, R9 + SHRQ $0x08, CX + MOVB CL, 1(R8) + + // stream 2, symbol 1 + MOVQ DX, CX + MOVQ R11, R14 + SHRQ CL, R14 + MOVWQZX (BX)(R14*2), CX SHLQ CL, R11 - ADDB CL, R12 - - // val1 := br1.peekTopBits(peekBits) - MOVQ DI, CX - MOVQ R11, R13 - SHRQ CL, R13 - - // v1 := table[val1&mask] - MOVWQZX (R9)(R13*2), CX - - // br1.advance(uint8(v1.entry)) - MOVB CH, AH - SHLQ CL, R11 - ADDB CL, R12 - - // these two writes get coalesced - // out[id * dstEvery + 0] = uint8(v0.entry >> 8) - // out[id * dstEvery + 1] = uint8(v1.entry >> 8) - MOVW AX, (BX)(R8*1) - - // update the bitreader structure - MOVQ R11, 80(R10) - MOVB R12, 88(R10) - - // br2.fillFast32() - MOVQ 128(R10), R11 - MOVBQZX 136(R10), R12 - CMPQ R12, $0x20 - JBE skip_fill2 - MOVQ 120(R10), AX - SUBQ $0x20, R12 - SUBQ $0x04, AX - MOVQ 96(R10), R13 - - // b.value |= uint64(low) << (b.bitsRead & 63) - MOVL (AX)(R13*1), R13 - MOVQ R12, CX - SHLQ CL, R13 - MOVQ AX, 120(R10) - ORQ R13, R11 - - // exhausted += (br2.off < 4) - CMPQ AX, $0x04 - ADCB $+0, DL - -skip_fill2: - // val0 := br2.peekTopBits(peekBits) - MOVQ R11, R13 - MOVQ DI, CX - SHRQ CL, R13 - - // v0 := table[val0&mask] - MOVWQZX (R9)(R13*2), CX - - // br2.advance(uint8(v0.entry) - MOVBLZX CH, AX + SHRQ $0x08, CX + MOVB CL, 1(R10) + + // stream 3, symbol 1 + MOVQ DX, CX + MOVQ R13, R14 + SHRQ CL, R14 + MOVWQZX (BX)(R14*2), CX + SHLQ CL, R13 + SHRQ $0x08, CX + MOVB CL, 1(R12) + + // stream 0, symbol 2 + MOVQ DX, CX + MOVQ DI, R14 + SHRQ CL, R14 + MOVWQZX (BX)(R14*2), CX + SHLQ CL, DI + SHRQ $0x08, CX + MOVB CL, 2(SI) + + // stream 1, symbol 2 + MOVQ DX, CX + MOVQ R9, R14 + SHRQ CL, R14 + MOVWQZX (BX)(R14*2), CX + SHLQ CL, R9 + SHRQ $0x08, CX + MOVB CL, 2(R8) + + // stream 2, symbol 2 + MOVQ DX, CX + MOVQ R11, R14 + SHRQ CL, R14 + MOVWQZX (BX)(R14*2), CX SHLQ CL, R11 - ADDB CL, R12 - - // val1 := br2.peekTopBits(peekBits) - MOVQ DI, CX - MOVQ R11, R13 - SHRQ CL, R13 - - // v1 := table[val1&mask] - MOVWQZX (R9)(R13*2), CX - - // br2.advance(uint8(v1.entry)) - MOVB CH, AH - SHLQ CL, R11 - ADDB CL, R12 - - // these two writes get coalesced - // out[id * dstEvery + 0] = uint8(v0.entry >> 8) - // out[id * dstEvery + 1] = uint8(v1.entry >> 8) - MOVW AX, (BX)(R8*2) - - // update the bitreader structure - MOVQ R11, 128(R10) - MOVB R12, 136(R10) - - // br3.fillFast32() - MOVQ 176(R10), R11 - MOVBQZX 184(R10), R12 - CMPQ R12, $0x20 - JBE skip_fill3 - MOVQ 168(R10), AX - SUBQ $0x20, R12 - SUBQ $0x04, AX - MOVQ 144(R10), R13 - - // b.value |= uint64(low) << (b.bitsRead & 63) - MOVL (AX)(R13*1), R13 - MOVQ R12, CX - SHLQ CL, R13 - MOVQ AX, 168(R10) - ORQ R13, R11 - - // exhausted += (br3.off < 4) - CMPQ AX, $0x04 - ADCB $+0, DL - -skip_fill3: - // val0 := br3.peekTopBits(peekBits) - MOVQ R11, R13 - MOVQ DI, CX - SHRQ CL, R13 - - // v0 := table[val0&mask] - MOVWQZX (R9)(R13*2), CX - - // br3.advance(uint8(v0.entry) - MOVBLZX CH, AX + SHRQ $0x08, CX + MOVB CL, 2(R10) + + // stream 3, symbol 2 + MOVQ DX, CX + MOVQ R13, R14 + SHRQ CL, R14 + MOVWQZX (BX)(R14*2), CX + SHLQ CL, R13 + SHRQ $0x08, CX + MOVB CL, 2(R12) + + // stream 0, symbol 3 + MOVQ DX, CX + MOVQ DI, R14 + SHRQ CL, R14 + MOVWQZX (BX)(R14*2), CX + SHLQ CL, DI + SHRQ $0x08, CX + MOVB CL, 3(SI) + + // stream 1, symbol 3 + MOVQ DX, CX + MOVQ R9, R14 + SHRQ CL, R14 + MOVWQZX (BX)(R14*2), CX + SHLQ CL, R9 + SHRQ $0x08, CX + MOVB CL, 3(R8) + + // stream 2, symbol 3 + MOVQ DX, CX + MOVQ R11, R14 + SHRQ CL, R14 + MOVWQZX (BX)(R14*2), CX SHLQ CL, R11 - ADDB CL, R12 - - // val1 := br3.peekTopBits(peekBits) - MOVQ DI, CX - MOVQ R11, R13 - SHRQ CL, R13 - - // v1 := table[val1&mask] - MOVWQZX (R9)(R13*2), CX - - // br3.advance(uint8(v1.entry)) - MOVB CH, AH - SHLQ CL, R11 - ADDB CL, R12 - - // these two writes get coalesced - // out[id * dstEvery + 0] = uint8(v0.entry >> 8) - // out[id * dstEvery + 1] = uint8(v1.entry >> 8) - LEAQ (R8)(R8*2), CX - MOVW AX, (BX)(CX*1) - - // update the bitreader structure - MOVQ R11, 176(R10) - MOVB R12, 184(R10) - ADDQ $0x02, BX - TESTB DL, DL - JZ main_loop - MOVQ ctx+0(FP), AX - SUBQ 16(AX), BX - SHLQ $0x02, BX - MOVQ BX, 40(AX) + SHRQ $0x08, CX + MOVB CL, 3(R10) + + // stream 3, symbol 3 + MOVQ DX, CX + MOVQ R13, R14 + SHRQ CL, R14 + MOVWQZX (BX)(R14*2), CX + SHLQ CL, R13 + SHRQ $0x08, CX + MOVB CL, 3(R12) + + // stream 0, symbol 4 + MOVQ DX, CX + MOVQ DI, R14 + SHRQ CL, R14 + MOVWQZX (BX)(R14*2), CX + SHLQ CL, DI + SHRQ $0x08, CX + MOVB CL, 4(SI) + + // stream 1, symbol 4 + MOVQ DX, CX + MOVQ R9, R14 + SHRQ CL, R14 + MOVWQZX (BX)(R14*2), CX + SHLQ CL, R9 + SHRQ $0x08, CX + MOVB CL, 4(R8) + + // stream 2, symbol 4 + MOVQ DX, CX + MOVQ R11, R14 + SHRQ CL, R14 + MOVWQZX (BX)(R14*2), CX + SHLQ CL, R11 + SHRQ $0x08, CX + MOVB CL, 4(R10) + + // stream 3, symbol 4 + MOVQ DX, CX + MOVQ R13, R14 + SHRQ CL, R14 + MOVWQZX (BX)(R14*2), CX + SHLQ CL, R13 + SHRQ $0x08, CX + MOVB CL, 4(R12) + + // Reload the four bit containers + TZCNTQ DI, R14 + MOVQ R14, CX + ANDQ $0x07, CX + SHRQ $0x03, R14 + MOVQ 64(AX), DI + SUBQ R14, DI + MOVQ DI, 64(AX) + MOVQ (DI), DI + ORQ $0x01, DI + SHLQ CL, DI + TZCNTQ R9, R14 + MOVQ R14, CX + ANDQ $0x07, CX + SHRQ $0x03, R14 + MOVQ 72(AX), R9 + SUBQ R14, R9 + MOVQ R9, 72(AX) + MOVQ (R9), R9 + ORQ $0x01, R9 + SHLQ CL, R9 + TZCNTQ R11, R14 + MOVQ R14, CX + ANDQ $0x07, CX + SHRQ $0x03, R14 + MOVQ 80(AX), R11 + SUBQ R14, R11 + MOVQ R11, 80(AX) + MOVQ (R11), R11 + ORQ $0x01, R11 + SHLQ CL, R11 + TZCNTQ R13, R14 + MOVQ R14, CX + ANDQ $0x07, CX + SHRQ $0x03, R14 + MOVQ 88(AX), R13 + SUBQ R14, R13 + MOVQ R13, 88(AX) + MOVQ (R13), R13 + ORQ $0x01, R13 + SHLQ CL, R13 + ADDQ $0x05, SI + ADDQ $0x05, R8 + ADDQ $0x05, R10 + ADDQ $0x05, R12 + CMPQ SI, 96(AX) + JB inner_loop + JMP outer_loop + +done: + // Hand the state back: off = ip - in (negative when the window reached below the stream start), + // value = the sentinel-form container. bitReaderShifted.restoreFromAsm normalizes both. + MOVQ (AX), CX + MOVQ 64(AX), DX + SUBQ (CX), DX + MOVQ DX, 24(CX) + MOVQ DI, 32(CX) + MOVQ 72(AX), DX + SUBQ 48(CX), DX + MOVQ DX, 72(CX) + MOVQ R9, 80(CX) + MOVQ 80(AX), DX + SUBQ 96(CX), DX + MOVQ DX, 120(CX) + MOVQ R11, 128(CX) + MOVQ 88(AX), DX + SUBQ 144(CX), DX + MOVQ DX, 168(CX) + MOVQ R13, 176(CX) + SUBQ 16(AX), SI + SHLQ $0x02, SI + MOVQ SI, 40(AX) RET // func decompress4x_8b_main_loop_amd64(ctx *decompress4xContext) +// Requires: BMI, CMOV TEXT ·decompress4x_8b_main_loop_amd64(SB), $0-8 - // Preload values - MOVQ ctx+0(FP), AX - MOVBQZX 8(AX), DI - MOVQ 16(AX), BX - MOVQ 48(AX), SI - MOVQ 24(AX), R8 - MOVQ 32(AX), R9 - MOVQ (AX), R10 - - // Main loop -main_loop: - XORL DX, DX - CMPQ BX, SI - SETGE DL - - // br0.fillFast32() - MOVQ 32(R10), R11 - MOVBQZX 40(R10), R12 - CMPQ R12, $0x20 - JBE skip_fill0 - MOVQ 24(R10), AX - SUBQ $0x20, R12 - SUBQ $0x04, AX - MOVQ (R10), R13 - - // b.value |= uint64(low) << (b.bitsRead & 63) - MOVL (AX)(R13*1), R13 - MOVQ R12, CX - SHLQ CL, R13 - MOVQ AX, 24(R10) - ORQ R13, R11 - - // exhausted += (br0.off < 4) - CMPQ AX, $0x04 - ADCB $+0, DL - -skip_fill0: - // val0 := br0.peekTopBits(peekBits) - MOVQ R11, AX - MOVQ DI, CX - SHRQ CL, AX - - // v0 := table[val0&mask] - MOVWQZX (R9)(AX*2), CX + MOVQ ctx+0(FP), AX - // br0.advance(uint8(v0.entry) - MOVBLZX CH, AX + // Preload values + MOVBQZX 8(AX), DX + MOVQ 32(AX), BX + MOVQ 16(AX), SI + MOVQ 24(AX), CX + LEAQ (SI)(CX*1), R8 + LEAQ (SI)(CX*2), R10 + LEAQ (CX)(CX*2), R12 + ADDQ SI, R12 + + // Convert each bit reader to sentinel form: bits = value | 1<> 8) - // out[id * dstEvery + 1] = uint8(v1.entry >> 8) - // out[id * dstEvery + 3] = uint8(v2.entry >> 8) - // out[id * dstEvery + 4] = uint8(v3.entry >> 8) - MOVL AX, (BX) - - // update the bitreader structure - MOVQ R11, 32(R10) - MOVB R12, 40(R10) - - // br1.fillFast32() - MOVQ 80(R10), R11 - MOVBQZX 88(R10), R12 - CMPQ R12, $0x20 - JBE skip_fill1 - MOVQ 72(R10), AX - SUBQ $0x20, R12 - SUBQ $0x04, AX - MOVQ 48(R10), R13 - - // b.value |= uint64(low) << (b.bitsRead & 63) - MOVL (AX)(R13*1), R13 - MOVQ R12, CX - SHLQ CL, R13 - MOVQ AX, 72(R10) - ORQ R13, R11 - - // exhausted += (br1.off < 4) - CMPQ AX, $0x04 - ADCB $+0, DL - -skip_fill1: - // val0 := br1.peekTopBits(peekBits) - MOVQ R11, AX - MOVQ DI, CX - SHRQ CL, AX - - // v0 := table[val0&mask] - MOVWQZX (R9)(AX*2), CX - - // br1.advance(uint8(v0.entry) - MOVBLZX CH, AX + SHRQ $0x08, CX + MOVB CL, (R10) + + // stream 3, symbol 0 + MOVQ DX, CX + MOVQ R13, R14 + SHRQ CL, R14 + MOVWQZX (BX)(R14*2), CX + SHLQ CL, R13 + SHRQ $0x08, CX + MOVB CL, (R12) + + // stream 0, symbol 1 + MOVQ DX, CX + MOVQ DI, R14 + SHRQ CL, R14 + MOVWQZX (BX)(R14*2), CX + SHLQ CL, DI + SHRQ $0x08, CX + MOVB CL, 1(SI) + + // stream 1, symbol 1 + MOVQ DX, CX + MOVQ R9, R14 + SHRQ CL, R14 + MOVWQZX (BX)(R14*2), CX + SHLQ CL, R9 + SHRQ $0x08, CX + MOVB CL, 1(R8) + + // stream 2, symbol 1 + MOVQ DX, CX + MOVQ R11, R14 + SHRQ CL, R14 + MOVWQZX (BX)(R14*2), CX SHLQ CL, R11 - ADDB CL, R12 - - // val1 := br1.peekTopBits(peekBits) - MOVQ R11, R13 - MOVQ DI, CX - SHRQ CL, R13 - - // v1 := table[val0&mask] - MOVWQZX (R9)(R13*2), CX - - // br1.advance(uint8(v1.entry) - MOVB CH, AH - SHLQ CL, R11 - ADDB CL, R12 - BSWAPL AX - - // val2 := br1.peekTopBits(peekBits) - MOVQ R11, R13 - MOVQ DI, CX - SHRQ CL, R13 - - // v2 := table[val0&mask] - MOVWQZX (R9)(R13*2), CX - - // br1.advance(uint8(v2.entry) - MOVB CH, AH - SHLQ CL, R11 - ADDB CL, R12 - - // val3 := br1.peekTopBits(peekBits) - MOVQ R11, R13 - MOVQ DI, CX - SHRQ CL, R13 - - // v3 := table[val0&mask] - MOVWQZX (R9)(R13*2), CX - - // br1.advance(uint8(v3.entry) - MOVB CH, AL + SHRQ $0x08, CX + MOVB CL, 1(R10) + + // stream 3, symbol 1 + MOVQ DX, CX + MOVQ R13, R14 + SHRQ CL, R14 + MOVWQZX (BX)(R14*2), CX + SHLQ CL, R13 + SHRQ $0x08, CX + MOVB CL, 1(R12) + + // stream 0, symbol 2 + MOVQ DX, CX + MOVQ DI, R14 + SHRQ CL, R14 + MOVWQZX (BX)(R14*2), CX + SHLQ CL, DI + SHRQ $0x08, CX + MOVB CL, 2(SI) + + // stream 1, symbol 2 + MOVQ DX, CX + MOVQ R9, R14 + SHRQ CL, R14 + MOVWQZX (BX)(R14*2), CX + SHLQ CL, R9 + SHRQ $0x08, CX + MOVB CL, 2(R8) + + // stream 2, symbol 2 + MOVQ DX, CX + MOVQ R11, R14 + SHRQ CL, R14 + MOVWQZX (BX)(R14*2), CX + SHLQ CL, R11 + SHRQ $0x08, CX + MOVB CL, 2(R10) + + // stream 3, symbol 2 + MOVQ DX, CX + MOVQ R13, R14 + SHRQ CL, R14 + MOVWQZX (BX)(R14*2), CX + SHLQ CL, R13 + SHRQ $0x08, CX + MOVB CL, 2(R12) + + // stream 0, symbol 3 + MOVQ DX, CX + MOVQ DI, R14 + SHRQ CL, R14 + MOVWQZX (BX)(R14*2), CX + SHLQ CL, DI + SHRQ $0x08, CX + MOVB CL, 3(SI) + + // stream 1, symbol 3 + MOVQ DX, CX + MOVQ R9, R14 + SHRQ CL, R14 + MOVWQZX (BX)(R14*2), CX + SHLQ CL, R9 + SHRQ $0x08, CX + MOVB CL, 3(R8) + + // stream 2, symbol 3 + MOVQ DX, CX + MOVQ R11, R14 + SHRQ CL, R14 + MOVWQZX (BX)(R14*2), CX + SHLQ CL, R11 + SHRQ $0x08, CX + MOVB CL, 3(R10) + + // stream 3, symbol 3 + MOVQ DX, CX + MOVQ R13, R14 + SHRQ CL, R14 + MOVWQZX (BX)(R14*2), CX + SHLQ CL, R13 + SHRQ $0x08, CX + MOVB CL, 3(R12) + + // stream 0, symbol 4 + MOVQ DX, CX + MOVQ DI, R14 + SHRQ CL, R14 + MOVWQZX (BX)(R14*2), CX + SHLQ CL, DI + SHRQ $0x08, CX + MOVB CL, 4(SI) + + // stream 1, symbol 4 + MOVQ DX, CX + MOVQ R9, R14 + SHRQ CL, R14 + MOVWQZX (BX)(R14*2), CX + SHLQ CL, R9 + SHRQ $0x08, CX + MOVB CL, 4(R8) + + // stream 2, symbol 4 + MOVQ DX, CX + MOVQ R11, R14 + SHRQ CL, R14 + MOVWQZX (BX)(R14*2), CX + SHLQ CL, R11 + SHRQ $0x08, CX + MOVB CL, 4(R10) + + // stream 3, symbol 4 + MOVQ DX, CX + MOVQ R13, R14 + SHRQ CL, R14 + MOVWQZX (BX)(R14*2), CX + SHLQ CL, R13 + SHRQ $0x08, CX + MOVB CL, 4(R12) + + // stream 0, symbol 5 + MOVQ DX, CX + MOVQ DI, R14 + SHRQ CL, R14 + MOVWQZX (BX)(R14*2), CX + SHLQ CL, DI + SHRQ $0x08, CX + MOVB CL, 5(SI) + + // stream 1, symbol 5 + MOVQ DX, CX + MOVQ R9, R14 + SHRQ CL, R14 + MOVWQZX (BX)(R14*2), CX + SHLQ CL, R9 + SHRQ $0x08, CX + MOVB CL, 5(R8) + + // stream 2, symbol 5 + MOVQ DX, CX + MOVQ R11, R14 + SHRQ CL, R14 + MOVWQZX (BX)(R14*2), CX + SHLQ CL, R11 + SHRQ $0x08, CX + MOVB CL, 5(R10) + + // stream 3, symbol 5 + MOVQ DX, CX + MOVQ R13, R14 + SHRQ CL, R14 + MOVWQZX (BX)(R14*2), CX + SHLQ CL, R13 + SHRQ $0x08, CX + MOVB CL, 5(R12) + + // stream 0, symbol 6 + MOVQ DX, CX + MOVQ DI, R14 + SHRQ CL, R14 + MOVWQZX (BX)(R14*2), CX + SHLQ CL, DI + SHRQ $0x08, CX + MOVB CL, 6(SI) + + // stream 1, symbol 6 + MOVQ DX, CX + MOVQ R9, R14 + SHRQ CL, R14 + MOVWQZX (BX)(R14*2), CX + SHLQ CL, R9 + SHRQ $0x08, CX + MOVB CL, 6(R8) + + // stream 2, symbol 6 + MOVQ DX, CX + MOVQ R11, R14 + SHRQ CL, R14 + MOVWQZX (BX)(R14*2), CX + SHLQ CL, R11 + SHRQ $0x08, CX + MOVB CL, 6(R10) + + // stream 3, symbol 6 + MOVQ DX, CX + MOVQ R13, R14 + SHRQ CL, R14 + MOVWQZX (BX)(R14*2), CX + SHLQ CL, R13 + SHRQ $0x08, CX + MOVB CL, 6(R12) + + // Reload the four bit containers + TZCNTQ DI, R14 + MOVQ R14, CX + ANDQ $0x07, CX + SHRQ $0x03, R14 + MOVQ 64(AX), DI + SUBQ R14, DI + MOVQ DI, 64(AX) + MOVQ (DI), DI + ORQ $0x01, DI + SHLQ CL, DI + TZCNTQ R9, R14 + MOVQ R14, CX + ANDQ $0x07, CX + SHRQ $0x03, R14 + MOVQ 72(AX), R9 + SUBQ R14, R9 + MOVQ R9, 72(AX) + MOVQ (R9), R9 + ORQ $0x01, R9 + SHLQ CL, R9 + TZCNTQ R11, R14 + MOVQ R14, CX + ANDQ $0x07, CX + SHRQ $0x03, R14 + MOVQ 80(AX), R11 + SUBQ R14, R11 + MOVQ R11, 80(AX) + MOVQ (R11), R11 + ORQ $0x01, R11 SHLQ CL, R11 - ADDB CL, R12 - BSWAPL AX - - // these four writes get coalesced - // out[id * dstEvery + 0] = uint8(v0.entry >> 8) - // out[id * dstEvery + 1] = uint8(v1.entry >> 8) - // out[id * dstEvery + 3] = uint8(v2.entry >> 8) - // out[id * dstEvery + 4] = uint8(v3.entry >> 8) - MOVL AX, (BX)(R8*1) - - // update the bitreader structure - MOVQ R11, 80(R10) - MOVB R12, 88(R10) - - // br2.fillFast32() - MOVQ 128(R10), R11 - MOVBQZX 136(R10), R12 - CMPQ R12, $0x20 - JBE skip_fill2 - MOVQ 120(R10), AX - SUBQ $0x20, R12 - SUBQ $0x04, AX - MOVQ 96(R10), R13 - - // b.value |= uint64(low) << (b.bitsRead & 63) - MOVL (AX)(R13*1), R13 - MOVQ R12, CX - SHLQ CL, R13 - MOVQ AX, 120(R10) - ORQ R13, R11 - - // exhausted += (br2.off < 4) - CMPQ AX, $0x04 - ADCB $+0, DL - -skip_fill2: - // val0 := br2.peekTopBits(peekBits) - MOVQ R11, AX - MOVQ DI, CX - SHRQ CL, AX + TZCNTQ R13, R14 + MOVQ R14, CX + ANDQ $0x07, CX + SHRQ $0x03, R14 + MOVQ 88(AX), R13 + SUBQ R14, R13 + MOVQ R13, 88(AX) + MOVQ (R13), R13 + ORQ $0x01, R13 + SHLQ CL, R13 + ADDQ $0x07, SI + ADDQ $0x07, R8 + ADDQ $0x07, R10 + ADDQ $0x07, R12 + CMPQ SI, 96(AX) + JB inner_loop + JMP outer_loop + +done: + // Hand the state back: off = ip - in (negative when the window reached below the stream start), + // value = the sentinel-form container. bitReaderShifted.restoreFromAsm normalizes both. + MOVQ (AX), CX + MOVQ 64(AX), DX + SUBQ (CX), DX + MOVQ DX, 24(CX) + MOVQ DI, 32(CX) + MOVQ 72(AX), DX + SUBQ 48(CX), DX + MOVQ DX, 72(CX) + MOVQ R9, 80(CX) + MOVQ 80(AX), DX + SUBQ 96(CX), DX + MOVQ DX, 120(CX) + MOVQ R11, 128(CX) + MOVQ 88(AX), DX + SUBQ 144(CX), DX + MOVQ DX, 168(CX) + MOVQ R13, 176(CX) + SUBQ 16(AX), SI + SHLQ $0x02, SI + MOVQ SI, 40(AX) + RET - // v0 := table[val0&mask] - MOVWQZX (R9)(AX*2), CX +// func decompress4x_4b_main_loop_amd64(ctx *decompress4xContext) +// Requires: BMI, CMOV +TEXT ·decompress4x_4b_main_loop_amd64(SB), $0-8 + MOVQ ctx+0(FP), AX - // br2.advance(uint8(v0.entry) - MOVBLZX CH, AX + // Preload values + MOVBQZX 8(AX), DX + MOVQ 32(AX), BX + MOVQ 16(AX), SI + MOVQ 24(AX), CX + LEAQ (SI)(CX*1), R8 + LEAQ (SI)(CX*2), R10 + LEAQ (CX)(CX*2), R12 + ADDQ SI, R12 + + // Convert each bit reader to sentinel form: bits = value | 1<> 8) - // out[id * dstEvery + 1] = uint8(v1.entry >> 8) - // out[id * dstEvery + 3] = uint8(v2.entry >> 8) - // out[id * dstEvery + 4] = uint8(v3.entry >> 8) - MOVL AX, (BX)(R8*2) - - // update the bitreader structure - MOVQ R11, 128(R10) - MOVB R12, 136(R10) - - // br3.fillFast32() - MOVQ 176(R10), R11 - MOVBQZX 184(R10), R12 - CMPQ R12, $0x20 - JBE skip_fill3 - MOVQ 168(R10), AX - SUBQ $0x20, R12 - SUBQ $0x04, AX - MOVQ 144(R10), R13 - - // b.value |= uint64(low) << (b.bitsRead & 63) - MOVL (AX)(R13*1), R13 - MOVQ R12, CX - SHLQ CL, R13 - MOVQ AX, 168(R10) - ORQ R13, R11 - - // exhausted += (br3.off < 4) - CMPQ AX, $0x04 - ADCB $+0, DL - -skip_fill3: - // val0 := br3.peekTopBits(peekBits) - MOVQ R11, AX - MOVQ DI, CX - SHRQ CL, AX +// func decompress4x_4b_main_loop_bmi2(ctx *decompress4xContext) +// Requires: BMI, BMI2, CMOV +TEXT ·decompress4x_4b_main_loop_bmi2(SB), $0-8 + MOVQ ctx+0(FP), AX - // v0 := table[val0&mask] - MOVWQZX (R9)(AX*2), CX + // Preload values + MOVBQZX 8(AX), CX + MOVQ 32(AX), DX + MOVQ 16(AX), BX + MOVQ 24(AX), SI + LEAQ (BX)(SI*1), DI + LEAQ (BX)(SI*2), R9 + LEAQ (SI)(SI*2), R11 + ADDQ BX, R11 + + // Convert each bit reader to sentinel form: bits = value | 1<> 8) - // out[id * dstEvery + 1] = uint8(v1.entry >> 8) - // out[id * dstEvery + 3] = uint8(v2.entry >> 8) - // out[id * dstEvery + 4] = uint8(v3.entry >> 8) - LEAQ (R8)(R8*2), CX - MOVL AX, (BX)(CX*1) - - // update the bitreader structure - MOVQ R11, 176(R10) - MOVB R12, 184(R10) - ADDQ $0x04, BX - TESTB DL, DL - JZ main_loop - MOVQ ctx+0(FP), AX - SUBQ 16(AX), BX - SHLQ $0x02, BX - MOVQ BX, 40(AX) - RET - -// func decompress1x_main_loop_amd64(ctx *decompress1xContext) -TEXT ·decompress1x_main_loop_amd64(SB), $0-8 - MOVQ ctx+0(FP), CX - MOVQ 16(CX), DX - MOVQ 24(CX), BX - CMPQ BX, $0x04 - JB error_max_decoded_size_exceeded - LEAQ (DX)(BX*1), BX - MOVQ (CX), SI - MOVQ (SI), R8 - MOVQ 24(SI), R9 - MOVQ 32(SI), R10 - MOVBQZX 40(SI), R11 - MOVQ 32(CX), SI - MOVBQZX 8(CX), DI - JMP loop_condition - -main_loop: - // Check if we have room for 4 bytes in the output buffer - LEAQ 4(DX), CX - CMPQ CX, BX - JGE error_max_decoded_size_exceeded - - // Decode 4 values - CMPQ R11, $0x20 - JL bitReader_fillFast_1_end - SUBQ $0x20, R11 - SUBQ $0x04, R9 - MOVL (R8)(R9*1), R12 - MOVQ R11, CX - SHLQ CL, R12 - ORQ R12, R10 - -bitReader_fillFast_1_end: - MOVQ DI, CX - MOVQ R10, R12 - SHRQ CL, R12 - MOVWQZX (SI)(R12*2), CX - MOVB CH, AL - MOVBQZX CL, CX + SUBQ SI, CX + JLE done + SHRQ $0x03, CX + + // Iterations allowed by the input: a refill reads the 8 bytes below the window and + // moves it down by at most 7, so every read stays inside the stream while ip stays + // at least 8 above ilowest. + MOVQ R8, R12 + SUBQ R9, R12 + SHRQ $0x03, R12 + CMPQ R12, CX + CMOVQCS R12, CX + TESTQ CX, CX + JZ done + IMUL3Q $0x07, CX, R12 + ADDQ SI, R12 + +inner_loop: + // symbol 0 + MOVQ DX, CX + MOVQ R10, R13 + SHRQ CL, R13 + MOVWQZX (BX)(R13*2), CX + SHLQ CL, R10 ADDQ CX, R11 + SHRQ $0x08, CX + MOVB CL, (SI) + + // symbol 1 + MOVQ DX, CX + MOVQ R10, R13 + SHRQ CL, R13 + MOVWQZX (BX)(R13*2), CX SHLQ CL, R10 - MOVQ DI, CX - MOVQ R10, R12 - SHRQ CL, R12 - MOVWQZX (SI)(R12*2), CX - MOVB CH, AH - MOVBQZX CL, CX ADDQ CX, R11 + SHRQ $0x08, CX + MOVB CL, 1(SI) + + // symbol 2 + MOVQ DX, CX + MOVQ R10, R13 + SHRQ CL, R13 + MOVWQZX (BX)(R13*2), CX SHLQ CL, R10 - BSWAPL AX - CMPQ R11, $0x20 - JL bitReader_fillFast_2_end - SUBQ $0x20, R11 - SUBQ $0x04, R9 - MOVL (R8)(R9*1), R12 - MOVQ R11, CX - SHLQ CL, R12 - ORQ R12, R10 - -bitReader_fillFast_2_end: - MOVQ DI, CX - MOVQ R10, R12 - SHRQ CL, R12 - MOVWQZX (SI)(R12*2), CX - MOVB CH, AH - MOVBQZX CL, CX ADDQ CX, R11 + SHRQ $0x08, CX + MOVB CL, 2(SI) + + // symbol 3 + MOVQ DX, CX + MOVQ R10, R13 + SHRQ CL, R13 + MOVWQZX (BX)(R13*2), CX SHLQ CL, R10 - MOVQ DI, CX - MOVQ R10, R12 - SHRQ CL, R12 - MOVWQZX (SI)(R12*2), CX - MOVB CH, AL - MOVBQZX CL, CX ADDQ CX, R11 + SHRQ $0x08, CX + MOVB CL, 3(SI) + + // symbol 4 + MOVQ DX, CX + MOVQ R10, R13 + SHRQ CL, R13 + MOVWQZX (BX)(R13*2), CX SHLQ CL, R10 - BSWAPL AX - - // Store the decoded values - MOVL AX, (DX) - ADDQ $0x04, DX - -loop_condition: - CMPQ R9, $0x08 - JGE main_loop + ADDQ CX, R11 + SHRQ $0x08, CX + MOVB CL, 4(SI) + + // symbol 5 + MOVQ DX, CX + MOVQ R10, R13 + SHRQ CL, R13 + MOVWQZX (BX)(R13*2), CX + SHLQ CL, R10 + ADDQ CX, R11 + SHRQ $0x08, CX + MOVB CL, 5(SI) + + // symbol 6 + MOVQ DX, CX + MOVQ R10, R13 + SHRQ CL, R13 + MOVWQZX (BX)(R13*2), CX + SHLQ CL, R10 + ADDQ CX, R11 + SHRQ $0x08, CX + MOVB CL, 6(SI) - // Update ctx structure - MOVQ ctx+0(FP), AX - SUBQ 16(AX), DX - MOVQ DX, 40(AX) - MOVQ (AX), AX - MOVQ R9, 24(AX) - MOVQ R10, 32(AX) - MOVB R11, 40(AX) + // Refill the whole bytes consumed from below the window + MOVQ R11, R13 + ANDQ $0x38, R13 + MOVQ -8(R8), R14 + SHRQ $0x01, R14 + ANDQ $0x07, R11 + MOVQ R13, CX + XORQ $0x3f, CX + SHRQ CL, R14 + MOVQ R11, CX + SHLQ CL, R14 + ORQ R14, R10 + SHRQ $0x03, R13 + SUBQ R13, R8 + ADDQ $0x07, SI + CMPQ SI, R12 + JB inner_loop + JMP outer_loop + +done: + // Hand the state back in the Go bit reader's own form: off = ip - in, value, bitsRead. + MOVQ (AX), CX + SUBQ (CX), R8 + MOVQ R8, 24(CX) + MOVQ R10, 32(CX) + MOVB R11, 40(CX) + SUBQ 16(AX), SI + MOVQ SI, 40(AX) RET - // Report error -error_max_decoded_size_exceeded: +// func decompress1x_4b_main_loop_amd64(ctx *decompress1xContext) +// Requires: CMOV +TEXT ·decompress1x_4b_main_loop_amd64(SB), $0-8 MOVQ ctx+0(FP), AX - MOVQ $-1, CX - MOVQ CX, 40(AX) - RET -// func decompress1x_main_loop_bmi2(ctx *decompress1xContext) -// Requires: BMI2 -TEXT ·decompress1x_main_loop_bmi2(SB), $0-8 - MOVQ ctx+0(FP), CX - MOVQ 16(CX), DX - MOVQ 24(CX), BX - CMPQ BX, $0x04 - JB error_max_decoded_size_exceeded - LEAQ (DX)(BX*1), BX - MOVQ (CX), SI - MOVQ (SI), R8 - MOVQ 24(SI), R9 - MOVQ 32(SI), R10 - MOVBQZX 40(SI), R11 - MOVQ 32(CX), SI - MOVBQZX 8(CX), DI - JMP loop_condition - -main_loop: - // Check if we have room for 4 bytes in the output buffer - LEAQ 4(DX), CX - CMPQ CX, BX - JGE error_max_decoded_size_exceeded - - // Decode 4 values - CMPQ R11, $0x20 - JL bitReader_fillFast_1_end - SUBQ $0x20, R11 - SUBQ $0x04, R9 - MOVL (R8)(R9*1), CX - SHLXQ R11, CX, CX - ORQ CX, R10 - -bitReader_fillFast_1_end: - SHRXQ DI, R10, CX - MOVWQZX (SI)(CX*2), CX - MOVB CH, AL - MOVBQZX CL, CX + // Preload values + MOVBQZX 8(AX), DX + MOVQ 32(AX), BX + MOVQ 16(AX), SI + MOVQ 24(AX), DI + ADDQ SI, DI + MOVQ 56(AX), R8 + MOVQ 48(AX), R9 + MOVQ (AX), CX + MOVQ 32(CX), R10 + MOVBQZX 40(CX), R11 + +outer_loop: + // Iterations allowed by the output: (limit - op) / 16 + MOVQ DI, CX + SUBQ SI, CX + JLE done + SHRQ $0x04, CX + + // Iterations allowed by the input: a refill reads the 8 bytes below the window and + // moves it down by at most 7, so every read stays inside the stream while ip stays + // at least 8 above ilowest. + MOVQ R8, R12 + SUBQ R9, R12 + SHRQ $0x03, R12 + CMPQ R12, CX + CMOVQCS R12, CX + TESTQ CX, CX + JZ done + IMUL3Q $0x0e, CX, R12 + ADDQ SI, R12 + +inner_loop: + // symbol 0 + MOVQ DX, CX + MOVQ R10, R13 + SHRQ CL, R13 + MOVWQZX (BX)(R13*2), CX + SHLQ CL, R10 + ADDQ CX, R11 + SHRQ $0x08, CX + MOVB CL, (SI) + + // symbol 1 + MOVQ DX, CX + MOVQ R10, R13 + SHRQ CL, R13 + MOVWQZX (BX)(R13*2), CX + SHLQ CL, R10 + ADDQ CX, R11 + SHRQ $0x08, CX + MOVB CL, 1(SI) + + // symbol 2 + MOVQ DX, CX + MOVQ R10, R13 + SHRQ CL, R13 + MOVWQZX (BX)(R13*2), CX + SHLQ CL, R10 + ADDQ CX, R11 + SHRQ $0x08, CX + MOVB CL, 2(SI) + + // symbol 3 + MOVQ DX, CX + MOVQ R10, R13 + SHRQ CL, R13 + MOVWQZX (BX)(R13*2), CX + SHLQ CL, R10 + ADDQ CX, R11 + SHRQ $0x08, CX + MOVB CL, 3(SI) + + // symbol 4 + MOVQ DX, CX + MOVQ R10, R13 + SHRQ CL, R13 + MOVWQZX (BX)(R13*2), CX + SHLQ CL, R10 + ADDQ CX, R11 + SHRQ $0x08, CX + MOVB CL, 4(SI) + + // symbol 5 + MOVQ DX, CX + MOVQ R10, R13 + SHRQ CL, R13 + MOVWQZX (BX)(R13*2), CX + SHLQ CL, R10 + ADDQ CX, R11 + SHRQ $0x08, CX + MOVB CL, 5(SI) + + // symbol 6 + MOVQ DX, CX + MOVQ R10, R13 + SHRQ CL, R13 + MOVWQZX (BX)(R13*2), CX + SHLQ CL, R10 + ADDQ CX, R11 + SHRQ $0x08, CX + MOVB CL, 6(SI) + + // symbol 7 + MOVQ DX, CX + MOVQ R10, R13 + SHRQ CL, R13 + MOVWQZX (BX)(R13*2), CX + SHLQ CL, R10 + ADDQ CX, R11 + SHRQ $0x08, CX + MOVB CL, 7(SI) + + // symbol 8 + MOVQ DX, CX + MOVQ R10, R13 + SHRQ CL, R13 + MOVWQZX (BX)(R13*2), CX + SHLQ CL, R10 + ADDQ CX, R11 + SHRQ $0x08, CX + MOVB CL, 8(SI) + + // symbol 9 + MOVQ DX, CX + MOVQ R10, R13 + SHRQ CL, R13 + MOVWQZX (BX)(R13*2), CX + SHLQ CL, R10 ADDQ CX, R11 - SHLXQ CX, R10, R10 - SHRXQ DI, R10, CX - MOVWQZX (SI)(CX*2), CX - MOVB CH, AH - MOVBQZX CL, CX + SHRQ $0x08, CX + MOVB CL, 9(SI) + + // symbol 10 + MOVQ DX, CX + MOVQ R10, R13 + SHRQ CL, R13 + MOVWQZX (BX)(R13*2), CX + SHLQ CL, R10 ADDQ CX, R11 - SHLXQ CX, R10, R10 - BSWAPL AX - CMPQ R11, $0x20 - JL bitReader_fillFast_2_end - SUBQ $0x20, R11 - SUBQ $0x04, R9 - MOVL (R8)(R9*1), CX - SHLXQ R11, CX, CX - ORQ CX, R10 - -bitReader_fillFast_2_end: - SHRXQ DI, R10, CX - MOVWQZX (SI)(CX*2), CX - MOVB CH, AH - MOVBQZX CL, CX + SHRQ $0x08, CX + MOVB CL, 10(SI) + + // symbol 11 + MOVQ DX, CX + MOVQ R10, R13 + SHRQ CL, R13 + MOVWQZX (BX)(R13*2), CX + SHLQ CL, R10 + ADDQ CX, R11 + SHRQ $0x08, CX + MOVB CL, 11(SI) + + // symbol 12 + MOVQ DX, CX + MOVQ R10, R13 + SHRQ CL, R13 + MOVWQZX (BX)(R13*2), CX + SHLQ CL, R10 ADDQ CX, R11 - SHLXQ CX, R10, R10 - SHRXQ DI, R10, CX - MOVWQZX (SI)(CX*2), CX - MOVB CH, AL - MOVBQZX CL, CX + SHRQ $0x08, CX + MOVB CL, 12(SI) + + // symbol 13 + MOVQ DX, CX + MOVQ R10, R13 + SHRQ CL, R13 + MOVWQZX (BX)(R13*2), CX + SHLQ CL, R10 ADDQ CX, R11 - SHLXQ CX, R10, R10 - BSWAPL AX + SHRQ $0x08, CX + MOVB CL, 13(SI) - // Store the decoded values - MOVL AX, (DX) - ADDQ $0x04, DX + // Refill the whole bytes consumed from below the window + MOVQ R11, R13 + ANDQ $0x38, R13 + MOVQ -8(R8), R14 + SHRQ $0x01, R14 + ANDQ $0x07, R11 + MOVQ R13, CX + XORQ $0x3f, CX + SHRQ CL, R14 + MOVQ R11, CX + SHLQ CL, R14 + ORQ R14, R10 + SHRQ $0x03, R13 + SUBQ R13, R8 + ADDQ $0x0e, SI + CMPQ SI, R12 + JB inner_loop + JMP outer_loop + +done: + // Hand the state back in the Go bit reader's own form: off = ip - in, value, bitsRead. + MOVQ (AX), CX + SUBQ (CX), R8 + MOVQ R8, 24(CX) + MOVQ R10, 32(CX) + MOVB R11, 40(CX) + SUBQ 16(AX), SI + MOVQ SI, 40(AX) + RET -loop_condition: - CMPQ R9, $0x08 - JGE main_loop +// func decompress1x_main_loop_bmi2(ctx *decompress1xContext) +// Requires: BMI2, CMOV +TEXT ·decompress1x_main_loop_bmi2(SB), $0-8 + MOVQ ctx+0(FP), AX - // Update ctx structure + // Preload values + MOVBQZX 8(AX), CX + MOVQ 32(AX), DX + MOVQ 16(AX), BX + MOVQ 24(AX), SI + ADDQ BX, SI + MOVQ 56(AX), DI + MOVQ 48(AX), R8 + MOVQ (AX), R10 + MOVQ 32(R10), R9 + MOVBQZX 40(R10), R10 + +outer_loop: + // Iterations allowed by the output: (limit - op) / 8 + MOVQ SI, R11 + SUBQ BX, R11 + JLE done + SHRQ $0x03, R11 + + // Iterations allowed by the input: a refill reads the 8 bytes below the window and + // moves it down by at most 7, so every read stays inside the stream while ip stays + // at least 8 above ilowest. + MOVQ DI, R12 + SUBQ R8, R12 + SHRQ $0x03, R12 + CMPQ R12, R11 + CMOVQCS R12, R11 + TESTQ R11, R11 + JZ done + IMUL3Q $0x05, R11, R11 + ADDQ BX, R11 + +inner_loop: + // symbol 0 + SHRXQ CX, R9, R12 + MOVWQZX (DX)(R12*2), R12 + SHLXQ R12, R9, R9 + ADDQ R12, R10 + SHRQ $0x08, R12 + MOVB R12, (BX) + + // symbol 1 + SHRXQ CX, R9, R12 + MOVWQZX (DX)(R12*2), R12 + SHLXQ R12, R9, R9 + ADDQ R12, R10 + SHRQ $0x08, R12 + MOVB R12, 1(BX) + + // symbol 2 + SHRXQ CX, R9, R12 + MOVWQZX (DX)(R12*2), R12 + SHLXQ R12, R9, R9 + ADDQ R12, R10 + SHRQ $0x08, R12 + MOVB R12, 2(BX) + + // symbol 3 + SHRXQ CX, R9, R12 + MOVWQZX (DX)(R12*2), R12 + SHLXQ R12, R9, R9 + ADDQ R12, R10 + SHRQ $0x08, R12 + MOVB R12, 3(BX) + + // symbol 4 + SHRXQ CX, R9, R12 + MOVWQZX (DX)(R12*2), R12 + SHLXQ R12, R9, R9 + ADDQ R12, R10 + SHRQ $0x08, R12 + MOVB R12, 4(BX) + + // Refill the whole bytes consumed from below the window + MOVQ R10, R12 + ANDQ $0x38, R12 + MOVQ -8(DI), R13 + SHRQ $0x01, R13 + ANDQ $0x07, R10 + MOVQ R12, R14 + XORQ $0x3f, R14 + SHRXQ R14, R13, R13 + SHLXQ R10, R13, R13 + ORQ R13, R9 + SHRQ $0x03, R12 + SUBQ R12, DI + ADDQ $0x05, BX + CMPQ BX, R11 + JB inner_loop + JMP outer_loop + +done: + // Hand the state back in the Go bit reader's own form: off = ip - in, value, bitsRead. + MOVQ (AX), CX + SUBQ (CX), DI + MOVQ DI, 24(CX) + MOVQ R9, 32(CX) + MOVB R10, 40(CX) + SUBQ 16(AX), BX + MOVQ BX, 40(AX) + RET + +// func decompress1x_8b_main_loop_bmi2(ctx *decompress1xContext) +// Requires: BMI2, CMOV +TEXT ·decompress1x_8b_main_loop_bmi2(SB), $0-8 MOVQ ctx+0(FP), AX - SUBQ 16(AX), DX - MOVQ DX, 40(AX) - MOVQ (AX), AX - MOVQ R9, 24(AX) - MOVQ R10, 32(AX) - MOVB R11, 40(AX) + + // Preload values + MOVBQZX 8(AX), CX + MOVQ 32(AX), DX + MOVQ 16(AX), BX + MOVQ 24(AX), SI + ADDQ BX, SI + MOVQ 56(AX), DI + MOVQ 48(AX), R8 + MOVQ (AX), R10 + MOVQ 32(R10), R9 + MOVBQZX 40(R10), R10 + +outer_loop: + // Iterations allowed by the output: (limit - op) / 8 + MOVQ SI, R11 + SUBQ BX, R11 + JLE done + SHRQ $0x03, R11 + + // Iterations allowed by the input: a refill reads the 8 bytes below the window and + // moves it down by at most 7, so every read stays inside the stream while ip stays + // at least 8 above ilowest. + MOVQ DI, R12 + SUBQ R8, R12 + SHRQ $0x03, R12 + CMPQ R12, R11 + CMOVQCS R12, R11 + TESTQ R11, R11 + JZ done + IMUL3Q $0x07, R11, R11 + ADDQ BX, R11 + +inner_loop: + // symbol 0 + SHRXQ CX, R9, R12 + MOVWQZX (DX)(R12*2), R12 + SHLXQ R12, R9, R9 + ADDQ R12, R10 + SHRQ $0x08, R12 + MOVB R12, (BX) + + // symbol 1 + SHRXQ CX, R9, R12 + MOVWQZX (DX)(R12*2), R12 + SHLXQ R12, R9, R9 + ADDQ R12, R10 + SHRQ $0x08, R12 + MOVB R12, 1(BX) + + // symbol 2 + SHRXQ CX, R9, R12 + MOVWQZX (DX)(R12*2), R12 + SHLXQ R12, R9, R9 + ADDQ R12, R10 + SHRQ $0x08, R12 + MOVB R12, 2(BX) + + // symbol 3 + SHRXQ CX, R9, R12 + MOVWQZX (DX)(R12*2), R12 + SHLXQ R12, R9, R9 + ADDQ R12, R10 + SHRQ $0x08, R12 + MOVB R12, 3(BX) + + // symbol 4 + SHRXQ CX, R9, R12 + MOVWQZX (DX)(R12*2), R12 + SHLXQ R12, R9, R9 + ADDQ R12, R10 + SHRQ $0x08, R12 + MOVB R12, 4(BX) + + // symbol 5 + SHRXQ CX, R9, R12 + MOVWQZX (DX)(R12*2), R12 + SHLXQ R12, R9, R9 + ADDQ R12, R10 + SHRQ $0x08, R12 + MOVB R12, 5(BX) + + // symbol 6 + SHRXQ CX, R9, R12 + MOVWQZX (DX)(R12*2), R12 + SHLXQ R12, R9, R9 + ADDQ R12, R10 + SHRQ $0x08, R12 + MOVB R12, 6(BX) + + // Refill the whole bytes consumed from below the window + MOVQ R10, R12 + ANDQ $0x38, R12 + MOVQ -8(DI), R13 + SHRQ $0x01, R13 + ANDQ $0x07, R10 + MOVQ R12, R14 + XORQ $0x3f, R14 + SHRXQ R14, R13, R13 + SHLXQ R10, R13, R13 + ORQ R13, R9 + SHRQ $0x03, R12 + SUBQ R12, DI + ADDQ $0x07, BX + CMPQ BX, R11 + JB inner_loop + JMP outer_loop + +done: + // Hand the state back in the Go bit reader's own form: off = ip - in, value, bitsRead. + MOVQ (AX), CX + SUBQ (CX), DI + MOVQ DI, 24(CX) + MOVQ R9, 32(CX) + MOVB R10, 40(CX) + SUBQ 16(AX), BX + MOVQ BX, 40(AX) RET - // Report error -error_max_decoded_size_exceeded: +// func decompress1x_4b_main_loop_bmi2(ctx *decompress1xContext) +// Requires: BMI2, CMOV +TEXT ·decompress1x_4b_main_loop_bmi2(SB), $0-8 MOVQ ctx+0(FP), AX - MOVQ $-1, CX - MOVQ CX, 40(AX) + + // Preload values + MOVBQZX 8(AX), CX + MOVQ 32(AX), DX + MOVQ 16(AX), BX + MOVQ 24(AX), SI + ADDQ BX, SI + MOVQ 56(AX), DI + MOVQ 48(AX), R8 + MOVQ (AX), R10 + MOVQ 32(R10), R9 + MOVBQZX 40(R10), R10 + +outer_loop: + // Iterations allowed by the output: (limit - op) / 16 + MOVQ SI, R11 + SUBQ BX, R11 + JLE done + SHRQ $0x04, R11 + + // Iterations allowed by the input: a refill reads the 8 bytes below the window and + // moves it down by at most 7, so every read stays inside the stream while ip stays + // at least 8 above ilowest. + MOVQ DI, R12 + SUBQ R8, R12 + SHRQ $0x03, R12 + CMPQ R12, R11 + CMOVQCS R12, R11 + TESTQ R11, R11 + JZ done + IMUL3Q $0x0e, R11, R11 + ADDQ BX, R11 + +inner_loop: + // symbol 0 + SHRXQ CX, R9, R12 + MOVWQZX (DX)(R12*2), R12 + SHLXQ R12, R9, R9 + ADDQ R12, R10 + SHRQ $0x08, R12 + MOVB R12, (BX) + + // symbol 1 + SHRXQ CX, R9, R12 + MOVWQZX (DX)(R12*2), R12 + SHLXQ R12, R9, R9 + ADDQ R12, R10 + SHRQ $0x08, R12 + MOVB R12, 1(BX) + + // symbol 2 + SHRXQ CX, R9, R12 + MOVWQZX (DX)(R12*2), R12 + SHLXQ R12, R9, R9 + ADDQ R12, R10 + SHRQ $0x08, R12 + MOVB R12, 2(BX) + + // symbol 3 + SHRXQ CX, R9, R12 + MOVWQZX (DX)(R12*2), R12 + SHLXQ R12, R9, R9 + ADDQ R12, R10 + SHRQ $0x08, R12 + MOVB R12, 3(BX) + + // symbol 4 + SHRXQ CX, R9, R12 + MOVWQZX (DX)(R12*2), R12 + SHLXQ R12, R9, R9 + ADDQ R12, R10 + SHRQ $0x08, R12 + MOVB R12, 4(BX) + + // symbol 5 + SHRXQ CX, R9, R12 + MOVWQZX (DX)(R12*2), R12 + SHLXQ R12, R9, R9 + ADDQ R12, R10 + SHRQ $0x08, R12 + MOVB R12, 5(BX) + + // symbol 6 + SHRXQ CX, R9, R12 + MOVWQZX (DX)(R12*2), R12 + SHLXQ R12, R9, R9 + ADDQ R12, R10 + SHRQ $0x08, R12 + MOVB R12, 6(BX) + + // symbol 7 + SHRXQ CX, R9, R12 + MOVWQZX (DX)(R12*2), R12 + SHLXQ R12, R9, R9 + ADDQ R12, R10 + SHRQ $0x08, R12 + MOVB R12, 7(BX) + + // symbol 8 + SHRXQ CX, R9, R12 + MOVWQZX (DX)(R12*2), R12 + SHLXQ R12, R9, R9 + ADDQ R12, R10 + SHRQ $0x08, R12 + MOVB R12, 8(BX) + + // symbol 9 + SHRXQ CX, R9, R12 + MOVWQZX (DX)(R12*2), R12 + SHLXQ R12, R9, R9 + ADDQ R12, R10 + SHRQ $0x08, R12 + MOVB R12, 9(BX) + + // symbol 10 + SHRXQ CX, R9, R12 + MOVWQZX (DX)(R12*2), R12 + SHLXQ R12, R9, R9 + ADDQ R12, R10 + SHRQ $0x08, R12 + MOVB R12, 10(BX) + + // symbol 11 + SHRXQ CX, R9, R12 + MOVWQZX (DX)(R12*2), R12 + SHLXQ R12, R9, R9 + ADDQ R12, R10 + SHRQ $0x08, R12 + MOVB R12, 11(BX) + + // symbol 12 + SHRXQ CX, R9, R12 + MOVWQZX (DX)(R12*2), R12 + SHLXQ R12, R9, R9 + ADDQ R12, R10 + SHRQ $0x08, R12 + MOVB R12, 12(BX) + + // symbol 13 + SHRXQ CX, R9, R12 + MOVWQZX (DX)(R12*2), R12 + SHLXQ R12, R9, R9 + ADDQ R12, R10 + SHRQ $0x08, R12 + MOVB R12, 13(BX) + + // Refill the whole bytes consumed from below the window + MOVQ R10, R12 + ANDQ $0x38, R12 + MOVQ -8(DI), R13 + SHRQ $0x01, R13 + ANDQ $0x07, R10 + MOVQ R12, R14 + XORQ $0x3f, R14 + SHRXQ R14, R13, R13 + SHLXQ R10, R13, R13 + ORQ R13, R9 + SHRQ $0x03, R12 + SUBQ R12, DI + ADDQ $0x0e, BX + CMPQ BX, R11 + JB inner_loop + JMP outer_loop + +done: + // Hand the state back in the Go bit reader's own form: off = ip - in, value, bitsRead. + MOVQ (AX), CX + SUBQ (CX), DI + MOVQ DI, 24(CX) + MOVQ R9, 32(CX) + MOVB R10, 40(CX) + SUBQ 16(AX), BX + MOVQ BX, 40(AX) RET diff --git a/vendor/github.com/klauspost/compress/huff0/decompress_arm64.go b/vendor/github.com/klauspost/compress/huff0/decompress_arm64.go index 8ba3c810e60a..d1988ad54dc4 100644 --- a/vendor/github.com/klauspost/compress/huff0/decompress_arm64.go +++ b/vendor/github.com/klauspost/compress/huff0/decompress_arm64.go @@ -6,24 +6,47 @@ package huff0 // decompress4x_main_loop_arm64 is an arm64 assembler implementation -// of Decompress4X when tablelog > 8. +// of Decompress4X when tablelog > 8, decoding fastSymbols symbols per +// stream between bit container reloads. // //go:noescape func decompress4x_main_loop_arm64(ctx *decompress4xContext) // decompress4x_8b_main_loop_arm64 is an arm64 assembler implementation -// of Decompress4X when tablelog <= 8 which decodes 4 entries -// per loop. +// of Decompress4X when tablelog <= 8, decoding fast8bSymbols symbols +// per stream between bit container reloads. // //go:noescape func decompress4x_8b_main_loop_arm64(ctx *decompress4xContext) +// decompress4x_4b_main_loop_arm64 is an arm64 assembler implementation +// of Decompress4X when tablelog <= 4, decoding fast4bSymbols symbols +// per stream between bit container reloads. +// +//go:noescape +func decompress4x_4b_main_loop_arm64(ctx *decompress4xContext) + // decompress1x_main_loop_arm64 is an arm64 assembler implementation -// of Decompress1X when tablelog > 8. +// of Decompress1X when tablelog > 8, decoding fastSymbols symbols +// between bit container reloads. // //go:noescape func decompress1x_main_loop_arm64(ctx *decompress1xContext) +// decompress1x_8b_main_loop_arm64 is an arm64 assembler implementation +// of Decompress1X when tablelog <= 8, decoding fast8bSymbols symbols +// between bit container reloads. +// +//go:noescape +func decompress1x_8b_main_loop_arm64(ctx *decompress1xContext) + +// decompress1x_4b_main_loop_arm64 is an arm64 assembler implementation +// of Decompress1X when tablelog <= 4, decoding fast4bSymbols symbols +// between bit container reloads. +// +//go:noescape +func decompress1x_4b_main_loop_arm64(ctx *decompress1xContext) + func decompress4x_main_loop_asm(ctx *decompress4xContext) { decompress4x_main_loop_arm64(ctx) } @@ -32,6 +55,18 @@ func decompress4x_8b_main_loop_asm(ctx *decompress4xContext) { decompress4x_8b_main_loop_arm64(ctx) } +func decompress4x_4b_main_loop_asm(ctx *decompress4xContext) { + decompress4x_4b_main_loop_arm64(ctx) +} + func decompress1x_main_loop_asm(ctx *decompress1xContext) { decompress1x_main_loop_arm64(ctx) } + +func decompress1x_8b_main_loop_asm(ctx *decompress1xContext) { + decompress1x_8b_main_loop_arm64(ctx) +} + +func decompress1x_4b_main_loop_asm(ctx *decompress1xContext) { + decompress1x_4b_main_loop_arm64(ctx) +} diff --git a/vendor/github.com/klauspost/compress/huff0/decompress_arm64.s b/vendor/github.com/klauspost/compress/huff0/decompress_arm64.s index 357df28e9d6c..971b8eb8bdc0 100644 --- a/vendor/github.com/klauspost/compress/huff0/decompress_arm64.s +++ b/vendor/github.com/klauspost/compress/huff0/decompress_arm64.s @@ -4,793 +4,1654 @@ //go:build arm64 && (!appengine && !noasm && gc) // func decompress4x_main_loop_amd64(ctx *decompress4xContext) +// Requires: BMI, CMOV TEXT ·decompress4x_main_loop_arm64(SB), $0-8 + MOVD ctx+0(FP), R0 + // Preload values - MOVD ctx+0(FP), R0 - MOVBU 8(R0), R6 - MOVD 16(R0), R3 - MOVD 48(R0), R5 - MOVD 24(R0), R7 - MOVD 32(R0), R8 - MOVD (R0), R9 - - // Main loop -main_loop: - MOVD $0, R2 - CMP R5, R3 - CSET GE, R16 - BFI $0, R16, $8, R2 - - // br0.fillFast32() - MOVD 32(R9), R10 - MOVBU 40(R9), R11 - CMP $0x20, R11 - BLS skip_fill0 - MOVD 24(R9), R0 - SUB $0x20, R11, R11 - SUB $0x04, R0, R0 - MOVD (R9), R12 - - // b.value |= uint64(low) << (b.bitsRead & 63) - MOVWU (R0)(R12), R12 - MOVD R11, R1 + MOVBU 8(R0), R2 + MOVD 32(R0), R3 + MOVD 16(R0), R5 + MOVD 24(R0), R1 + ADD R1, R5, R7 + ADD R1<<1, R5, R9 + ADD R1<<1, R1, R11 + ADD R5, R11, R11 + + // Convert each bit reader to sentinel form: bits = value | 1<> 8) - // out[id * dstEvery + 1] = uint8(v1.entry >> 8) - MOVH R0, (R3) - - // update the bitreader structure - MOVD R10, 32(R9) - MOVB R11, 40(R9) - - // br1.fillFast32() - MOVD 80(R9), R10 - MOVBU 88(R9), R11 - CMP $0x20, R11 - BLS skip_fill1 - MOVD 72(R9), R0 - SUB $0x20, R11, R11 - SUB $0x04, R0, R0 - MOVD 48(R9), R12 - - // b.value |= uint64(low) << (b.bitsRead & 63) - MOVWU (R0)(R12), R12 - MOVD R11, R1 + LSR $0x08, R1, R1 + MOVB R1, (R11) + + // stream 0, symbol 1 + LSR R2, R6, R13 + MOVHU (R3)(R13<<1), R1 + LSL R1, R6, R6 + LSR $0x08, R1, R1 + MOVB R1, 1(R5) + + // stream 1, symbol 1 + LSR R2, R8, R13 + MOVHU (R3)(R13<<1), R1 + LSL R1, R8, R8 + LSR $0x08, R1, R1 + MOVB R1, 1(R7) + + // stream 2, symbol 1 + LSR R2, R10, R13 + MOVHU (R3)(R13<<1), R1 + LSL R1, R10, R10 + LSR $0x08, R1, R1 + MOVB R1, 1(R9) + + // stream 3, symbol 1 + LSR R2, R12, R13 + MOVHU (R3)(R13<<1), R1 LSL R1, R12, R12 - MOVD R0, 72(R9) - ORR R12, R10, R10 - - // exhausted += (br1.off < 4) - CMP $0x04, R0 - CSINC HS, R2, R2, R16 - BFI $0, R16, $8, R2 - -skip_fill1: - // val0 := br1.peekTopBits(peekBits) - MOVD R10, R12 - MOVD R6, R1 - LSR R1, R12, R12 - - // v0 := table[val0&mask] - MOVHU (R8)(R12<<1), R1 - - // br1.advance(uint8(v0.entry) - UBFX $8, R1, $8, R0 - LSL R1, R10, R10 - ADD R1, R11, R15 - BFI $0, R15, $8, R11 - - // val1 := br1.peekTopBits(peekBits) - MOVD R6, R1 - MOVD R10, R12 - LSR R1, R12, R12 - - // v1 := table[val1&mask] - MOVHU (R8)(R12<<1), R1 - - // br1.advance(uint8(v1.entry)) - UBFX $8, R1, $8, R16 - BFI $8, R16, $8, R0 - LSL R1, R10, R10 - ADD R1, R11, R15 - BFI $0, R15, $8, R11 - - // these two writes get coalesced - // out[id * dstEvery + 0] = uint8(v0.entry >> 8) - // out[id * dstEvery + 1] = uint8(v1.entry >> 8) - MOVH R0, (R3)(R7) - - // update the bitreader structure - MOVD R10, 80(R9) - MOVB R11, 88(R9) - - // br2.fillFast32() - MOVD 128(R9), R10 - MOVBU 136(R9), R11 - CMP $0x20, R11 - BLS skip_fill2 - MOVD 120(R9), R0 - SUB $0x20, R11, R11 - SUB $0x04, R0, R0 - MOVD 96(R9), R12 - - // b.value |= uint64(low) << (b.bitsRead & 63) - MOVWU (R0)(R12), R12 - MOVD R11, R1 + LSR $0x08, R1, R1 + MOVB R1, 1(R11) + + // stream 0, symbol 2 + LSR R2, R6, R13 + MOVHU (R3)(R13<<1), R1 + LSL R1, R6, R6 + LSR $0x08, R1, R1 + MOVB R1, 2(R5) + + // stream 1, symbol 2 + LSR R2, R8, R13 + MOVHU (R3)(R13<<1), R1 + LSL R1, R8, R8 + LSR $0x08, R1, R1 + MOVB R1, 2(R7) + + // stream 2, symbol 2 + LSR R2, R10, R13 + MOVHU (R3)(R13<<1), R1 + LSL R1, R10, R10 + LSR $0x08, R1, R1 + MOVB R1, 2(R9) + + // stream 3, symbol 2 + LSR R2, R12, R13 + MOVHU (R3)(R13<<1), R1 LSL R1, R12, R12 - MOVD R0, 120(R9) - ORR R12, R10, R10 - - // exhausted += (br2.off < 4) - CMP $0x04, R0 - CSINC HS, R2, R2, R16 - BFI $0, R16, $8, R2 - -skip_fill2: - // val0 := br2.peekTopBits(peekBits) - MOVD R10, R12 - MOVD R6, R1 - LSR R1, R12, R12 - - // v0 := table[val0&mask] - MOVHU (R8)(R12<<1), R1 - - // br2.advance(uint8(v0.entry) - UBFX $8, R1, $8, R0 - LSL R1, R10, R10 - ADD R1, R11, R15 - BFI $0, R15, $8, R11 - - // val1 := br2.peekTopBits(peekBits) - MOVD R6, R1 - MOVD R10, R12 - LSR R1, R12, R12 - - // v1 := table[val1&mask] - MOVHU (R8)(R12<<1), R1 - - // br2.advance(uint8(v1.entry)) - UBFX $8, R1, $8, R16 - BFI $8, R16, $8, R0 - LSL R1, R10, R10 - ADD R1, R11, R15 - BFI $0, R15, $8, R11 - - // these two writes get coalesced - // out[id * dstEvery + 0] = uint8(v0.entry >> 8) - // out[id * dstEvery + 1] = uint8(v1.entry >> 8) - MOVH R0, (R3)(R7<<1) - - // update the bitreader structure - MOVD R10, 128(R9) - MOVB R11, 136(R9) - - // br3.fillFast32() - MOVD 176(R9), R10 - MOVBU 184(R9), R11 - CMP $0x20, R11 - BLS skip_fill3 - MOVD 168(R9), R0 - SUB $0x20, R11, R11 - SUB $0x04, R0, R0 - MOVD 144(R9), R12 - - // b.value |= uint64(low) << (b.bitsRead & 63) - MOVWU (R0)(R12), R12 - MOVD R11, R1 + LSR $0x08, R1, R1 + MOVB R1, 2(R11) + + // stream 0, symbol 3 + LSR R2, R6, R13 + MOVHU (R3)(R13<<1), R1 + LSL R1, R6, R6 + LSR $0x08, R1, R1 + MOVB R1, 3(R5) + + // stream 1, symbol 3 + LSR R2, R8, R13 + MOVHU (R3)(R13<<1), R1 + LSL R1, R8, R8 + LSR $0x08, R1, R1 + MOVB R1, 3(R7) + + // stream 2, symbol 3 + LSR R2, R10, R13 + MOVHU (R3)(R13<<1), R1 + LSL R1, R10, R10 + LSR $0x08, R1, R1 + MOVB R1, 3(R9) + + // stream 3, symbol 3 + LSR R2, R12, R13 + MOVHU (R3)(R13<<1), R1 LSL R1, R12, R12 - MOVD R0, 168(R9) - ORR R12, R10, R10 - - // exhausted += (br3.off < 4) - CMP $0x04, R0 - CSINC HS, R2, R2, R16 - BFI $0, R16, $8, R2 - -skip_fill3: - // val0 := br3.peekTopBits(peekBits) - MOVD R10, R12 - MOVD R6, R1 - LSR R1, R12, R12 - - // v0 := table[val0&mask] - MOVHU (R8)(R12<<1), R1 - - // br3.advance(uint8(v0.entry) - UBFX $8, R1, $8, R0 - LSL R1, R10, R10 - ADD R1, R11, R15 - BFI $0, R15, $8, R11 - - // val1 := br3.peekTopBits(peekBits) - MOVD R6, R1 - MOVD R10, R12 - LSR R1, R12, R12 - - // v1 := table[val1&mask] - MOVHU (R8)(R12<<1), R1 - - // br3.advance(uint8(v1.entry)) - UBFX $8, R1, $8, R16 - BFI $8, R16, $8, R0 + LSR $0x08, R1, R1 + MOVB R1, 3(R11) + + // stream 0, symbol 4 + LSR R2, R6, R13 + MOVHU (R3)(R13<<1), R1 + LSL R1, R6, R6 + LSR $0x08, R1, R1 + MOVB R1, 4(R5) + + // stream 1, symbol 4 + LSR R2, R8, R13 + MOVHU (R3)(R13<<1), R1 + LSL R1, R8, R8 + LSR $0x08, R1, R1 + MOVB R1, 4(R7) + + // stream 2, symbol 4 + LSR R2, R10, R13 + MOVHU (R3)(R13<<1), R1 + LSL R1, R10, R10 + LSR $0x08, R1, R1 + MOVB R1, 4(R9) + + // stream 3, symbol 4 + LSR R2, R12, R13 + MOVHU (R3)(R13<<1), R1 + LSL R1, R12, R12 + LSR $0x08, R1, R1 + MOVB R1, 4(R11) + + // Reload the four bit containers + RBIT R6, R13 + CLZ R13, R13 + MOVD R13, R1 + AND $0x07, R1, R1 + LSR $0x03, R13, R13 + MOVD 64(R0), R6 + SUB R13, R6, R6 + MOVD R6, 64(R0) + MOVD (R6), R6 + ORR $0x01, R6, R6 + LSL R1, R6, R6 + RBIT R8, R13 + CLZ R13, R13 + MOVD R13, R1 + AND $0x07, R1, R1 + LSR $0x03, R13, R13 + MOVD 72(R0), R8 + SUB R13, R8, R8 + MOVD R8, 72(R0) + MOVD (R8), R8 + ORR $0x01, R8, R8 + LSL R1, R8, R8 + RBIT R10, R13 + CLZ R13, R13 + MOVD R13, R1 + AND $0x07, R1, R1 + LSR $0x03, R13, R13 + MOVD 80(R0), R10 + SUB R13, R10, R10 + MOVD R10, 80(R0) + MOVD (R10), R10 + ORR $0x01, R10, R10 LSL R1, R10, R10 - ADD R1, R11, R15 - BFI $0, R15, $8, R11 - - // these two writes get coalesced - // out[id * dstEvery + 0] = uint8(v0.entry >> 8) - // out[id * dstEvery + 1] = uint8(v1.entry >> 8) - ADD R7<<1, R7, R1 - MOVH R0, (R3)(R1) - - // update the bitreader structure - MOVD R10, 176(R9) - MOVB R11, 184(R9) - ADD $0x02, R3, R3 - TST $0xff, R2 - BEQ main_loop - MOVD ctx+0(FP), R0 + RBIT R12, R13 + CLZ R13, R13 + MOVD R13, R1 + AND $0x07, R1, R1 + LSR $0x03, R13, R13 + MOVD 88(R0), R12 + SUB R13, R12, R12 + MOVD R12, 88(R0) + MOVD (R12), R12 + ORR $0x01, R12, R12 + LSL R1, R12, R12 + ADD $0x05, R5, R5 + ADD $0x05, R7, R7 + ADD $0x05, R9, R9 + ADD $0x05, R11, R11 + MOVD 96(R0), R16 + CMP R16, R5 + BLO inner_loop + JMP outer_loop + +done: + // Hand the state back: off = ip - in (negative when the window reached below the stream start), + // value = the sentinel-form container. bitReaderShifted.restoreFromAsm normalizes both. + MOVD (R0), R1 + MOVD 64(R0), R2 + MOVD (R1), R16 + SUB R16, R2, R2 + MOVD R2, 24(R1) + MOVD R6, 32(R1) + MOVD 72(R0), R2 + MOVD 48(R1), R16 + SUB R16, R2, R2 + MOVD R2, 72(R1) + MOVD R8, 80(R1) + MOVD 80(R0), R2 + MOVD 96(R1), R16 + SUB R16, R2, R2 + MOVD R2, 120(R1) + MOVD R10, 128(R1) + MOVD 88(R0), R2 + MOVD 144(R1), R16 + SUB R16, R2, R2 + MOVD R2, 168(R1) + MOVD R12, 176(R1) MOVD 16(R0), R16 - SUB R16, R3, R3 - LSL $0x02, R3, R3 - MOVD R3, 40(R0) + SUB R16, R5, R5 + LSL $0x02, R5, R5 + MOVD R5, 40(R0) RET // func decompress4x_8b_main_loop_amd64(ctx *decompress4xContext) +// Requires: BMI, CMOV TEXT ·decompress4x_8b_main_loop_arm64(SB), $0-8 + MOVD ctx+0(FP), R0 + // Preload values - MOVD ctx+0(FP), R0 - MOVBU 8(R0), R6 - MOVD 16(R0), R3 - MOVD 48(R0), R5 - MOVD 24(R0), R7 - MOVD 32(R0), R8 - MOVD (R0), R9 - - // Main loop -main_loop: - MOVD $0, R2 - CMP R5, R3 - CSET GE, R16 - BFI $0, R16, $8, R2 - - // br0.fillFast32() - MOVD 32(R9), R10 - MOVBU 40(R9), R11 - CMP $0x20, R11 - BLS skip_fill0 - MOVD 24(R9), R0 - SUB $0x20, R11, R11 - SUB $0x04, R0, R0 - MOVD (R9), R12 - - // b.value |= uint64(low) << (b.bitsRead & 63) - MOVWU (R0)(R12), R12 - MOVD R11, R1 + MOVBU 8(R0), R2 + MOVD 32(R0), R3 + MOVD 16(R0), R5 + MOVD 24(R0), R1 + ADD R1, R5, R7 + ADD R1<<1, R5, R9 + ADD R1<<1, R1, R11 + ADD R5, R11, R11 + + // Convert each bit reader to sentinel form: bits = value | 1<> 8) - // out[id * dstEvery + 1] = uint8(v1.entry >> 8) - // out[id * dstEvery + 3] = uint8(v2.entry >> 8) - // out[id * dstEvery + 4] = uint8(v3.entry >> 8) - MOVW R0, (R3) - - // update the bitreader structure - MOVD R10, 32(R9) - MOVB R11, 40(R9) - - // br1.fillFast32() - MOVD 80(R9), R10 - MOVBU 88(R9), R11 - CMP $0x20, R11 - BLS skip_fill1 - MOVD 72(R9), R0 - SUB $0x20, R11, R11 - SUB $0x04, R0, R0 - MOVD 48(R9), R12 - - // b.value |= uint64(low) << (b.bitsRead & 63) - MOVWU (R0)(R12), R12 - MOVD R11, R1 + LSR $0x08, R1, R1 + MOVB R1, (R11) + + // stream 0, symbol 1 + LSR R2, R6, R13 + MOVHU (R3)(R13<<1), R1 + LSL R1, R6, R6 + LSR $0x08, R1, R1 + MOVB R1, 1(R5) + + // stream 1, symbol 1 + LSR R2, R8, R13 + MOVHU (R3)(R13<<1), R1 + LSL R1, R8, R8 + LSR $0x08, R1, R1 + MOVB R1, 1(R7) + + // stream 2, symbol 1 + LSR R2, R10, R13 + MOVHU (R3)(R13<<1), R1 + LSL R1, R10, R10 + LSR $0x08, R1, R1 + MOVB R1, 1(R9) + + // stream 3, symbol 1 + LSR R2, R12, R13 + MOVHU (R3)(R13<<1), R1 LSL R1, R12, R12 - MOVD R0, 72(R9) - ORR R12, R10, R10 - - // exhausted += (br1.off < 4) - CMP $0x04, R0 - CSINC HS, R2, R2, R16 - BFI $0, R16, $8, R2 - -skip_fill1: - // val0 := br1.peekTopBits(peekBits) - MOVD R10, R0 - MOVD R6, R1 - LSR R1, R0, R0 - - // v0 := table[val0&mask] - MOVHU (R8)(R0<<1), R1 - - // br1.advance(uint8(v0.entry) - UBFX $8, R1, $8, R0 - LSL R1, R10, R10 - ADD R1, R11, R15 - BFI $0, R15, $8, R11 - - // val1 := br1.peekTopBits(peekBits) - MOVD R10, R12 - MOVD R6, R1 - LSR R1, R12, R12 - - // v1 := table[val0&mask] - MOVHU (R8)(R12<<1), R1 - - // br1.advance(uint8(v1.entry) - UBFX $8, R1, $8, R16 - BFI $8, R16, $8, R0 + LSR $0x08, R1, R1 + MOVB R1, 1(R11) + + // stream 0, symbol 2 + LSR R2, R6, R13 + MOVHU (R3)(R13<<1), R1 + LSL R1, R6, R6 + LSR $0x08, R1, R1 + MOVB R1, 2(R5) + + // stream 1, symbol 2 + LSR R2, R8, R13 + MOVHU (R3)(R13<<1), R1 + LSL R1, R8, R8 + LSR $0x08, R1, R1 + MOVB R1, 2(R7) + + // stream 2, symbol 2 + LSR R2, R10, R13 + MOVHU (R3)(R13<<1), R1 + LSL R1, R10, R10 + LSR $0x08, R1, R1 + MOVB R1, 2(R9) + + // stream 3, symbol 2 + LSR R2, R12, R13 + MOVHU (R3)(R13<<1), R1 + LSL R1, R12, R12 + LSR $0x08, R1, R1 + MOVB R1, 2(R11) + + // stream 0, symbol 3 + LSR R2, R6, R13 + MOVHU (R3)(R13<<1), R1 + LSL R1, R6, R6 + LSR $0x08, R1, R1 + MOVB R1, 3(R5) + + // stream 1, symbol 3 + LSR R2, R8, R13 + MOVHU (R3)(R13<<1), R1 + LSL R1, R8, R8 + LSR $0x08, R1, R1 + MOVB R1, 3(R7) + + // stream 2, symbol 3 + LSR R2, R10, R13 + MOVHU (R3)(R13<<1), R1 + LSL R1, R10, R10 + LSR $0x08, R1, R1 + MOVB R1, 3(R9) + + // stream 3, symbol 3 + LSR R2, R12, R13 + MOVHU (R3)(R13<<1), R1 + LSL R1, R12, R12 + LSR $0x08, R1, R1 + MOVB R1, 3(R11) + + // stream 0, symbol 4 + LSR R2, R6, R13 + MOVHU (R3)(R13<<1), R1 + LSL R1, R6, R6 + LSR $0x08, R1, R1 + MOVB R1, 4(R5) + + // stream 1, symbol 4 + LSR R2, R8, R13 + MOVHU (R3)(R13<<1), R1 + LSL R1, R8, R8 + LSR $0x08, R1, R1 + MOVB R1, 4(R7) + + // stream 2, symbol 4 + LSR R2, R10, R13 + MOVHU (R3)(R13<<1), R1 + LSL R1, R10, R10 + LSR $0x08, R1, R1 + MOVB R1, 4(R9) + + // stream 3, symbol 4 + LSR R2, R12, R13 + MOVHU (R3)(R13<<1), R1 + LSL R1, R12, R12 + LSR $0x08, R1, R1 + MOVB R1, 4(R11) + + // stream 0, symbol 5 + LSR R2, R6, R13 + MOVHU (R3)(R13<<1), R1 + LSL R1, R6, R6 + LSR $0x08, R1, R1 + MOVB R1, 5(R5) + + // stream 1, symbol 5 + LSR R2, R8, R13 + MOVHU (R3)(R13<<1), R1 + LSL R1, R8, R8 + LSR $0x08, R1, R1 + MOVB R1, 5(R7) + + // stream 2, symbol 5 + LSR R2, R10, R13 + MOVHU (R3)(R13<<1), R1 + LSL R1, R10, R10 + LSR $0x08, R1, R1 + MOVB R1, 5(R9) + + // stream 3, symbol 5 + LSR R2, R12, R13 + MOVHU (R3)(R13<<1), R1 + LSL R1, R12, R12 + LSR $0x08, R1, R1 + MOVB R1, 5(R11) + + // stream 0, symbol 6 + LSR R2, R6, R13 + MOVHU (R3)(R13<<1), R1 + LSL R1, R6, R6 + LSR $0x08, R1, R1 + MOVB R1, 6(R5) + + // stream 1, symbol 6 + LSR R2, R8, R13 + MOVHU (R3)(R13<<1), R1 + LSL R1, R8, R8 + LSR $0x08, R1, R1 + MOVB R1, 6(R7) + + // stream 2, symbol 6 + LSR R2, R10, R13 + MOVHU (R3)(R13<<1), R1 + LSL R1, R10, R10 + LSR $0x08, R1, R1 + MOVB R1, 6(R9) + + // stream 3, symbol 6 + LSR R2, R12, R13 + MOVHU (R3)(R13<<1), R1 + LSL R1, R12, R12 + LSR $0x08, R1, R1 + MOVB R1, 6(R11) + + // Reload the four bit containers + RBIT R6, R13 + CLZ R13, R13 + MOVD R13, R1 + AND $0x07, R1, R1 + LSR $0x03, R13, R13 + MOVD 64(R0), R6 + SUB R13, R6, R6 + MOVD R6, 64(R0) + MOVD (R6), R6 + ORR $0x01, R6, R6 + LSL R1, R6, R6 + RBIT R8, R13 + CLZ R13, R13 + MOVD R13, R1 + AND $0x07, R1, R1 + LSR $0x03, R13, R13 + MOVD 72(R0), R8 + SUB R13, R8, R8 + MOVD R8, 72(R0) + MOVD (R8), R8 + ORR $0x01, R8, R8 + LSL R1, R8, R8 + RBIT R10, R13 + CLZ R13, R13 + MOVD R13, R1 + AND $0x07, R1, R1 + LSR $0x03, R13, R13 + MOVD 80(R0), R10 + SUB R13, R10, R10 + MOVD R10, 80(R0) + MOVD (R10), R10 + ORR $0x01, R10, R10 LSL R1, R10, R10 - ADD R1, R11, R15 - BFI $0, R15, $8, R11 - REVW R0, R0 - - // val2 := br1.peekTopBits(peekBits) - MOVD R10, R12 - MOVD R6, R1 - LSR R1, R12, R12 + RBIT R12, R13 + CLZ R13, R13 + MOVD R13, R1 + AND $0x07, R1, R1 + LSR $0x03, R13, R13 + MOVD 88(R0), R12 + SUB R13, R12, R12 + MOVD R12, 88(R0) + MOVD (R12), R12 + ORR $0x01, R12, R12 + LSL R1, R12, R12 + ADD $0x07, R5, R5 + ADD $0x07, R7, R7 + ADD $0x07, R9, R9 + ADD $0x07, R11, R11 + MOVD 96(R0), R16 + CMP R16, R5 + BLO inner_loop + JMP outer_loop + +done: + // Hand the state back: off = ip - in (negative when the window reached below the stream start), + // value = the sentinel-form container. bitReaderShifted.restoreFromAsm normalizes both. + MOVD (R0), R1 + MOVD 64(R0), R2 + MOVD (R1), R16 + SUB R16, R2, R2 + MOVD R2, 24(R1) + MOVD R6, 32(R1) + MOVD 72(R0), R2 + MOVD 48(R1), R16 + SUB R16, R2, R2 + MOVD R2, 72(R1) + MOVD R8, 80(R1) + MOVD 80(R0), R2 + MOVD 96(R1), R16 + SUB R16, R2, R2 + MOVD R2, 120(R1) + MOVD R10, 128(R1) + MOVD 88(R0), R2 + MOVD 144(R1), R16 + SUB R16, R2, R2 + MOVD R2, 168(R1) + MOVD R12, 176(R1) + MOVD 16(R0), R16 + SUB R16, R5, R5 + LSL $0x02, R5, R5 + MOVD R5, 40(R0) + RET - // v2 := table[val0&mask] - MOVHU (R8)(R12<<1), R1 +// func decompress4x_4b_main_loop_amd64(ctx *decompress4xContext) +// Requires: BMI, CMOV +TEXT ·decompress4x_4b_main_loop_arm64(SB), $0-8 + MOVD ctx+0(FP), R0 - // br1.advance(uint8(v2.entry) - UBFX $8, R1, $8, R16 - BFI $8, R16, $8, R0 + // Preload values + MOVBU 8(R0), R2 + MOVD 32(R0), R3 + MOVD 16(R0), R5 + MOVD 24(R0), R1 + ADD R1, R5, R7 + ADD R1<<1, R5, R9 + ADD R1<<1, R1, R11 + ADD R5, R11, R11 + + // Convert each bit reader to sentinel form: bits = value | 1<> 8) - // out[id * dstEvery + 1] = uint8(v1.entry >> 8) - // out[id * dstEvery + 3] = uint8(v2.entry >> 8) - // out[id * dstEvery + 4] = uint8(v3.entry >> 8) - MOVW R0, (R3)(R7) - - // update the bitreader structure - MOVD R10, 80(R9) - MOVB R11, 88(R9) - - // br2.fillFast32() - MOVD 128(R9), R10 - MOVBU 136(R9), R11 - CMP $0x20, R11 - BLS skip_fill2 - MOVD 120(R9), R0 - SUB $0x20, R11, R11 - SUB $0x04, R0, R0 - MOVD 96(R9), R12 - - // b.value |= uint64(low) << (b.bitsRead & 63) - MOVWU (R0)(R12), R12 - MOVD R11, R1 - LSL R1, R12, R12 - MOVD R0, 120(R9) - ORR R12, R10, R10 +// skipped decompress4x_4b_main_loop_bmi2 (generic twin preferred on arm64) - // exhausted += (br2.off < 4) - CMP $0x04, R0 - CSINC HS, R2, R2, R16 - BFI $0, R16, $8, R2 +// func decompress1x_main_loop_amd64(ctx *decompress1xContext) +// Requires: CMOV +TEXT ·decompress1x_main_loop_arm64(SB), $0-8 + MOVD ctx+0(FP), R0 -skip_fill2: - // val0 := br2.peekTopBits(peekBits) - MOVD R10, R0 + // Preload values + MOVBU 8(R0), R2 + MOVD 32(R0), R3 + MOVD 16(R0), R5 + MOVD 24(R0), R6 + ADD R5, R6, R6 + MOVD 56(R0), R7 + MOVD 48(R0), R8 + MOVD (R0), R1 + MOVD 32(R1), R9 + MOVBU 40(R1), R10 + +outer_loop: + // Iterations allowed by the output: (limit - op) / 8 MOVD R6, R1 - LSR R1, R0, R0 - - // v0 := table[val0&mask] - MOVHU (R8)(R0<<1), R1 + SUBS R5, R1, R1 + BLE done + LSR $0x03, R1, R1 + + // Iterations allowed by the input: a refill reads the 8 bytes below the window and + // moves it down by at most 7, so every read stays inside the stream while ip stays + // at least 8 above ilowest. + MOVD R7, R11 + SUB R8, R11, R11 + LSR $0x03, R11, R11 + CMP R1, R11 + CSEL LO, R11, R1, R1 + TST R1, R1 + BEQ done + MOVD $5, R16 + MUL R16, R1, R11 + ADD R5, R11, R11 + +inner_loop: + // symbol 0 + LSR R2, R9, R12 + MOVHU (R3)(R12<<1), R1 + LSL R1, R9, R9 + ADD R1, R10, R10 + LSR $0x08, R1, R1 + MOVB R1, (R5) - // br2.advance(uint8(v0.entry) - UBFX $8, R1, $8, R0 - LSL R1, R10, R10 - ADD R1, R11, R15 - BFI $0, R15, $8, R11 + // symbol 1 + LSR R2, R9, R12 + MOVHU (R3)(R12<<1), R1 + LSL R1, R9, R9 + ADD R1, R10, R10 + LSR $0x08, R1, R1 + MOVB R1, 1(R5) - // val1 := br2.peekTopBits(peekBits) - MOVD R10, R12 - MOVD R6, R1 - LSR R1, R12, R12 + // symbol 2 + LSR R2, R9, R12 + MOVHU (R3)(R12<<1), R1 + LSL R1, R9, R9 + ADD R1, R10, R10 + LSR $0x08, R1, R1 + MOVB R1, 2(R5) - // v1 := table[val0&mask] - MOVHU (R8)(R12<<1), R1 + // symbol 3 + LSR R2, R9, R12 + MOVHU (R3)(R12<<1), R1 + LSL R1, R9, R9 + ADD R1, R10, R10 + LSR $0x08, R1, R1 + MOVB R1, 3(R5) - // br2.advance(uint8(v1.entry) - UBFX $8, R1, $8, R16 - BFI $8, R16, $8, R0 - LSL R1, R10, R10 - ADD R1, R11, R15 - BFI $0, R15, $8, R11 - REVW R0, R0 + // symbol 4 + LSR R2, R9, R12 + MOVHU (R3)(R12<<1), R1 + LSL R1, R9, R9 + ADD R1, R10, R10 + LSR $0x08, R1, R1 + MOVB R1, 4(R5) - // val2 := br2.peekTopBits(peekBits) + // Refill the whole bytes consumed from below the window MOVD R10, R12 - MOVD R6, R1 - LSR R1, R12, R12 - - // v2 := table[val0&mask] - MOVHU (R8)(R12<<1), R1 + AND $0x38, R12, R12 + MOVD -8(R7), R13 + LSR $0x01, R13, R13 + AND $0x07, R10, R10 + MOVD R12, R1 + EOR $0x3f, R1, R1 + LSR R1, R13, R13 + MOVD R10, R1 + LSL R1, R13, R13 + ORR R13, R9, R9 + LSR $0x03, R12, R12 + SUB R12, R7, R7 + ADD $0x05, R5, R5 + CMP R11, R5 + BLO inner_loop + JMP outer_loop + +done: + // Hand the state back in the Go bit reader's own form: off = ip - in, value, bitsRead. + MOVD (R0), R1 + MOVD (R1), R16 + SUB R16, R7, R7 + MOVD R7, 24(R1) + MOVD R9, 32(R1) + MOVB R10, 40(R1) + MOVD 16(R0), R16 + SUB R16, R5, R5 + MOVD R5, 40(R0) + RET - // br2.advance(uint8(v2.entry) - UBFX $8, R1, $8, R16 - BFI $8, R16, $8, R0 - LSL R1, R10, R10 - ADD R1, R11, R15 - BFI $0, R15, $8, R11 +// func decompress1x_8b_main_loop_amd64(ctx *decompress1xContext) +// Requires: CMOV +TEXT ·decompress1x_8b_main_loop_arm64(SB), $0-8 + MOVD ctx+0(FP), R0 - // val3 := br2.peekTopBits(peekBits) - MOVD R10, R12 + // Preload values + MOVBU 8(R0), R2 + MOVD 32(R0), R3 + MOVD 16(R0), R5 + MOVD 24(R0), R6 + ADD R5, R6, R6 + MOVD 56(R0), R7 + MOVD 48(R0), R8 + MOVD (R0), R1 + MOVD 32(R1), R9 + MOVBU 40(R1), R10 + +outer_loop: + // Iterations allowed by the output: (limit - op) / 8 MOVD R6, R1 - LSR R1, R12, R12 + SUBS R5, R1, R1 + BLE done + LSR $0x03, R1, R1 + + // Iterations allowed by the input: a refill reads the 8 bytes below the window and + // moves it down by at most 7, so every read stays inside the stream while ip stays + // at least 8 above ilowest. + MOVD R7, R11 + SUB R8, R11, R11 + LSR $0x03, R11, R11 + CMP R1, R11 + CSEL LO, R11, R1, R1 + TST R1, R1 + BEQ done + MOVD $7, R16 + MUL R16, R1, R11 + ADD R5, R11, R11 + +inner_loop: + // symbol 0 + LSR R2, R9, R12 + MOVHU (R3)(R12<<1), R1 + LSL R1, R9, R9 + ADD R1, R10, R10 + LSR $0x08, R1, R1 + MOVB R1, (R5) - // v3 := table[val0&mask] - MOVHU (R8)(R12<<1), R1 + // symbol 1 + LSR R2, R9, R12 + MOVHU (R3)(R12<<1), R1 + LSL R1, R9, R9 + ADD R1, R10, R10 + LSR $0x08, R1, R1 + MOVB R1, 1(R5) - // br2.advance(uint8(v3.entry) - UBFX $8, R1, $8, R16 - BFI $0, R16, $8, R0 - LSL R1, R10, R10 - ADD R1, R11, R15 - BFI $0, R15, $8, R11 - REVW R0, R0 - - // these four writes get coalesced - // out[id * dstEvery + 0] = uint8(v0.entry >> 8) - // out[id * dstEvery + 1] = uint8(v1.entry >> 8) - // out[id * dstEvery + 3] = uint8(v2.entry >> 8) - // out[id * dstEvery + 4] = uint8(v3.entry >> 8) - ADD R7<<1, R3, R15 - MOVW R0, (R15) - - // update the bitreader structure - MOVD R10, 128(R9) - MOVB R11, 136(R9) - - // br3.fillFast32() - MOVD 176(R9), R10 - MOVBU 184(R9), R11 - CMP $0x20, R11 - BLS skip_fill3 - MOVD 168(R9), R0 - SUB $0x20, R11, R11 - SUB $0x04, R0, R0 - MOVD 144(R9), R12 - - // b.value |= uint64(low) << (b.bitsRead & 63) - MOVWU (R0)(R12), R12 - MOVD R11, R1 - LSL R1, R12, R12 - MOVD R0, 168(R9) - ORR R12, R10, R10 + // symbol 2 + LSR R2, R9, R12 + MOVHU (R3)(R12<<1), R1 + LSL R1, R9, R9 + ADD R1, R10, R10 + LSR $0x08, R1, R1 + MOVB R1, 2(R5) - // exhausted += (br3.off < 4) - CMP $0x04, R0 - CSINC HS, R2, R2, R16 - BFI $0, R16, $8, R2 + // symbol 3 + LSR R2, R9, R12 + MOVHU (R3)(R12<<1), R1 + LSL R1, R9, R9 + ADD R1, R10, R10 + LSR $0x08, R1, R1 + MOVB R1, 3(R5) -skip_fill3: - // val0 := br3.peekTopBits(peekBits) - MOVD R10, R0 - MOVD R6, R1 - LSR R1, R0, R0 + // symbol 4 + LSR R2, R9, R12 + MOVHU (R3)(R12<<1), R1 + LSL R1, R9, R9 + ADD R1, R10, R10 + LSR $0x08, R1, R1 + MOVB R1, 4(R5) - // v0 := table[val0&mask] - MOVHU (R8)(R0<<1), R1 + // symbol 5 + LSR R2, R9, R12 + MOVHU (R3)(R12<<1), R1 + LSL R1, R9, R9 + ADD R1, R10, R10 + LSR $0x08, R1, R1 + MOVB R1, 5(R5) - // br3.advance(uint8(v0.entry) - UBFX $8, R1, $8, R0 - LSL R1, R10, R10 - ADD R1, R11, R15 - BFI $0, R15, $8, R11 + // symbol 6 + LSR R2, R9, R12 + MOVHU (R3)(R12<<1), R1 + LSL R1, R9, R9 + ADD R1, R10, R10 + LSR $0x08, R1, R1 + MOVB R1, 6(R5) - // val1 := br3.peekTopBits(peekBits) + // Refill the whole bytes consumed from below the window MOVD R10, R12 - MOVD R6, R1 - LSR R1, R12, R12 - - // v1 := table[val0&mask] - MOVHU (R8)(R12<<1), R1 + AND $0x38, R12, R12 + MOVD -8(R7), R13 + LSR $0x01, R13, R13 + AND $0x07, R10, R10 + MOVD R12, R1 + EOR $0x3f, R1, R1 + LSR R1, R13, R13 + MOVD R10, R1 + LSL R1, R13, R13 + ORR R13, R9, R9 + LSR $0x03, R12, R12 + SUB R12, R7, R7 + ADD $0x07, R5, R5 + CMP R11, R5 + BLO inner_loop + JMP outer_loop + +done: + // Hand the state back in the Go bit reader's own form: off = ip - in, value, bitsRead. + MOVD (R0), R1 + MOVD (R1), R16 + SUB R16, R7, R7 + MOVD R7, 24(R1) + MOVD R9, 32(R1) + MOVB R10, 40(R1) + MOVD 16(R0), R16 + SUB R16, R5, R5 + MOVD R5, 40(R0) + RET - // br3.advance(uint8(v1.entry) - UBFX $8, R1, $8, R16 - BFI $8, R16, $8, R0 - LSL R1, R10, R10 - ADD R1, R11, R15 - BFI $0, R15, $8, R11 - REVW R0, R0 +// func decompress1x_4b_main_loop_amd64(ctx *decompress1xContext) +// Requires: CMOV +TEXT ·decompress1x_4b_main_loop_arm64(SB), $0-8 + MOVD ctx+0(FP), R0 - // val2 := br3.peekTopBits(peekBits) - MOVD R10, R12 + // Preload values + MOVBU 8(R0), R2 + MOVD 32(R0), R3 + MOVD 16(R0), R5 + MOVD 24(R0), R6 + ADD R5, R6, R6 + MOVD 56(R0), R7 + MOVD 48(R0), R8 + MOVD (R0), R1 + MOVD 32(R1), R9 + MOVBU 40(R1), R10 + +outer_loop: + // Iterations allowed by the output: (limit - op) / 16 MOVD R6, R1 - LSR R1, R12, R12 + SUBS R5, R1, R1 + BLE done + LSR $0x04, R1, R1 + + // Iterations allowed by the input: a refill reads the 8 bytes below the window and + // moves it down by at most 7, so every read stays inside the stream while ip stays + // at least 8 above ilowest. + MOVD R7, R11 + SUB R8, R11, R11 + LSR $0x03, R11, R11 + CMP R1, R11 + CSEL LO, R11, R1, R1 + TST R1, R1 + BEQ done + MOVD $14, R16 + MUL R16, R1, R11 + ADD R5, R11, R11 + +inner_loop: + // symbol 0 + LSR R2, R9, R12 + MOVHU (R3)(R12<<1), R1 + LSL R1, R9, R9 + ADD R1, R10, R10 + LSR $0x08, R1, R1 + MOVB R1, (R5) - // v2 := table[val0&mask] - MOVHU (R8)(R12<<1), R1 + // symbol 1 + LSR R2, R9, R12 + MOVHU (R3)(R12<<1), R1 + LSL R1, R9, R9 + ADD R1, R10, R10 + LSR $0x08, R1, R1 + MOVB R1, 1(R5) - // br3.advance(uint8(v2.entry) - UBFX $8, R1, $8, R16 - BFI $8, R16, $8, R0 - LSL R1, R10, R10 - ADD R1, R11, R15 - BFI $0, R15, $8, R11 + // symbol 2 + LSR R2, R9, R12 + MOVHU (R3)(R12<<1), R1 + LSL R1, R9, R9 + ADD R1, R10, R10 + LSR $0x08, R1, R1 + MOVB R1, 2(R5) - // val3 := br3.peekTopBits(peekBits) - MOVD R10, R12 - MOVD R6, R1 - LSR R1, R12, R12 + // symbol 3 + LSR R2, R9, R12 + MOVHU (R3)(R12<<1), R1 + LSL R1, R9, R9 + ADD R1, R10, R10 + LSR $0x08, R1, R1 + MOVB R1, 3(R5) - // v3 := table[val0&mask] - MOVHU (R8)(R12<<1), R1 + // symbol 4 + LSR R2, R9, R12 + MOVHU (R3)(R12<<1), R1 + LSL R1, R9, R9 + ADD R1, R10, R10 + LSR $0x08, R1, R1 + MOVB R1, 4(R5) - // br3.advance(uint8(v3.entry) - UBFX $8, R1, $8, R16 - BFI $0, R16, $8, R0 - LSL R1, R10, R10 - ADD R1, R11, R15 - BFI $0, R15, $8, R11 - REVW R0, R0 - - // these four writes get coalesced - // out[id * dstEvery + 0] = uint8(v0.entry >> 8) - // out[id * dstEvery + 1] = uint8(v1.entry >> 8) - // out[id * dstEvery + 3] = uint8(v2.entry >> 8) - // out[id * dstEvery + 4] = uint8(v3.entry >> 8) - ADD R7<<1, R7, R1 - MOVW R0, (R3)(R1) - - // update the bitreader structure - MOVD R10, 176(R9) - MOVB R11, 184(R9) - ADD $0x04, R3, R3 - TST $0xff, R2 - BEQ main_loop - MOVD ctx+0(FP), R0 - MOVD 16(R0), R16 - SUB R16, R3, R3 - LSL $0x02, R3, R3 - MOVD R3, 40(R0) - RET + // symbol 5 + LSR R2, R9, R12 + MOVHU (R3)(R12<<1), R1 + LSL R1, R9, R9 + ADD R1, R10, R10 + LSR $0x08, R1, R1 + MOVB R1, 5(R5) -// func decompress1x_main_loop_amd64(ctx *decompress1xContext) -TEXT ·decompress1x_main_loop_arm64(SB), $0-8 - MOVD ctx+0(FP), R1 - MOVD 16(R1), R2 - MOVD 24(R1), R3 - CMP $0x04, R3 - BLO error_max_decoded_size_exceeded - ADD R3, R2, R3 - MOVD (R1), R5 - MOVD (R5), R7 - MOVD 24(R5), R8 - MOVD 32(R5), R9 - MOVBU 40(R5), R10 - MOVD 32(R1), R5 - MOVBU 8(R1), R6 - JMP loop_condition - -main_loop: - // Check if we have room for 4 bytes in the output buffer - ADD $4, R2, R1 - CMP R3, R1 - BGE error_max_decoded_size_exceeded - - // Decode 4 values - CMP $0x20, R10 - BLT bitReader_fillFast_1_end - SUB $0x20, R10, R10 - SUB $0x04, R8, R8 - MOVWU (R7)(R8), R11 - MOVD R10, R1 - LSL R1, R11, R11 - ORR R11, R9, R9 - -bitReader_fillFast_1_end: - MOVD R6, R1 - MOVD R9, R11 - LSR R1, R11, R11 - MOVHU (R5)(R11<<1), R1 - UBFX $8, R1, $8, R16 - BFI $0, R16, $8, R0 - MOVBU R1, R1 + // symbol 6 + LSR R2, R9, R12 + MOVHU (R3)(R12<<1), R1 + LSL R1, R9, R9 ADD R1, R10, R10 + LSR $0x08, R1, R1 + MOVB R1, 6(R5) + + // symbol 7 + LSR R2, R9, R12 + MOVHU (R3)(R12<<1), R1 LSL R1, R9, R9 - MOVD R6, R1 - MOVD R9, R11 - LSR R1, R11, R11 - MOVHU (R5)(R11<<1), R1 - UBFX $8, R1, $8, R16 - BFI $8, R16, $8, R0 - MOVBU R1, R1 ADD R1, R10, R10 + LSR $0x08, R1, R1 + MOVB R1, 7(R5) + + // symbol 8 + LSR R2, R9, R12 + MOVHU (R3)(R12<<1), R1 LSL R1, R9, R9 - REVW R0, R0 - CMP $0x20, R10 - BLT bitReader_fillFast_2_end - SUB $0x20, R10, R10 - SUB $0x04, R8, R8 - MOVWU (R7)(R8), R11 - MOVD R10, R1 - LSL R1, R11, R11 - ORR R11, R9, R9 - -bitReader_fillFast_2_end: - MOVD R6, R1 - MOVD R9, R11 - LSR R1, R11, R11 - MOVHU (R5)(R11<<1), R1 - UBFX $8, R1, $8, R16 - BFI $8, R16, $8, R0 - MOVBU R1, R1 ADD R1, R10, R10 + LSR $0x08, R1, R1 + MOVB R1, 8(R5) + + // symbol 9 + LSR R2, R9, R12 + MOVHU (R3)(R12<<1), R1 + LSL R1, R9, R9 + ADD R1, R10, R10 + LSR $0x08, R1, R1 + MOVB R1, 9(R5) + + // symbol 10 + LSR R2, R9, R12 + MOVHU (R3)(R12<<1), R1 LSL R1, R9, R9 - MOVD R6, R1 - MOVD R9, R11 - LSR R1, R11, R11 - MOVHU (R5)(R11<<1), R1 - UBFX $8, R1, $8, R16 - BFI $0, R16, $8, R0 - MOVBU R1, R1 ADD R1, R10, R10 + LSR $0x08, R1, R1 + MOVB R1, 10(R5) + + // symbol 11 + LSR R2, R9, R12 + MOVHU (R3)(R12<<1), R1 LSL R1, R9, R9 - REVW R0, R0 + ADD R1, R10, R10 + LSR $0x08, R1, R1 + MOVB R1, 11(R5) - // Store the decoded values - MOVW R0, (R2) - ADD $0x04, R2, R2 + // symbol 12 + LSR R2, R9, R12 + MOVHU (R3)(R12<<1), R1 + LSL R1, R9, R9 + ADD R1, R10, R10 + LSR $0x08, R1, R1 + MOVB R1, 12(R5) -loop_condition: - CMP $0x08, R8 - BGE main_loop + // symbol 13 + LSR R2, R9, R12 + MOVHU (R3)(R12<<1), R1 + LSL R1, R9, R9 + ADD R1, R10, R10 + LSR $0x08, R1, R1 + MOVB R1, 13(R5) - // Update ctx structure - MOVD ctx+0(FP), R0 + // Refill the whole bytes consumed from below the window + MOVD R10, R12 + AND $0x38, R12, R12 + MOVD -8(R7), R13 + LSR $0x01, R13, R13 + AND $0x07, R10, R10 + MOVD R12, R1 + EOR $0x3f, R1, R1 + LSR R1, R13, R13 + MOVD R10, R1 + LSL R1, R13, R13 + ORR R13, R9, R9 + LSR $0x03, R12, R12 + SUB R12, R7, R7 + ADD $0x0e, R5, R5 + CMP R11, R5 + BLO inner_loop + JMP outer_loop + +done: + // Hand the state back in the Go bit reader's own form: off = ip - in, value, bitsRead. + MOVD (R0), R1 + MOVD (R1), R16 + SUB R16, R7, R7 + MOVD R7, 24(R1) + MOVD R9, 32(R1) + MOVB R10, 40(R1) MOVD 16(R0), R16 - SUB R16, R2, R2 - MOVD R2, 40(R0) - MOVD (R0), R0 - MOVD R8, 24(R0) - MOVD R9, 32(R0) - MOVB R10, 40(R0) - RET - - // Report error -error_max_decoded_size_exceeded: - MOVD ctx+0(FP), R0 - MOVD $-1, R1 - MOVD R1, 40(R0) + SUB R16, R5, R5 + MOVD R5, 40(R0) RET // skipped decompress1x_main_loop_bmi2 (generic twin preferred on arm64) + +// skipped decompress1x_8b_main_loop_bmi2 (generic twin preferred on arm64) + +// skipped decompress1x_4b_main_loop_bmi2 (generic twin preferred on arm64) diff --git a/vendor/github.com/klauspost/compress/huff0/decompress_asm.go b/vendor/github.com/klauspost/compress/huff0/decompress_asm.go index 4854dd43653e..71a80bbd5572 100644 --- a/vendor/github.com/klauspost/compress/huff0/decompress_asm.go +++ b/vendor/github.com/klauspost/compress/huff0/decompress_asm.go @@ -14,6 +14,10 @@ import ( // fallback8BitSize is the size where using Go version is faster. const fallback8BitSize = 800 +// decompress4xContext is the argument block of the Decompress4X asm loops. +// Go fills every field but decoded and inner; the asm advances ip, and on +// return leaves each bit reader in the form bitReaderShifted.restoreFromAsm +// expects. type decompress4xContext struct { pbr *[4]bitReaderShifted peekBits uint8 @@ -21,9 +25,20 @@ type decompress4xContext struct { dstEvery int tbl *dEntrySingle decoded int - limit *byte + limit *byte // stream 0's output pointer must stay below this + ilowest *byte // start of the input block; no read goes below it + ip [4]*byte // each stream's 8-byte input window, advanced by the asm + inner *byte // scratch for the asm: the current inner-loop limit } +// Symbols decoded per stream between reloads by the 4X asm loops; must +// match the constants of the same names in _generate/gen.go. +const ( + fastSymbols = 5 // tablelog 9..11 + fast8bSymbols = 7 // tablelog 5..8 + fast4bSymbols = 14 // tablelog <= 4 +) + // Decompress4X will decompress a 4X encoded stream. // The length of the supplied input must match the end of a block exactly. // The *capacity* of the dst slice must match the destination size of @@ -72,23 +87,55 @@ func (d *Decoder) Decompress4X(dst, src []byte) ([]byte, error) { var decoded int - if len(out) > 4*4 && !(br[0].off < 4 || br[1].off < 4 || br[2].off < 4 || br[3].off < 4) { + nSyms := fastSymbols + if d.actualTableLog <= 4 { + nSyms = fast4bSymbols + } else if use8BitTables { + nSyms = fast8bSymbols + } + // The asm writes nSyms bytes per stream per iteration and only re-checks + // its bounds between batches of iterations (the batch size is derived + // from limit in _generate/gen.go), so stream 0 must stop early enough + // that stream 3 (which may be up to 3 bytes shorter than dstEvery) + // never writes past the end of out. Every stream needs a full 8-byte + // window ahead of its read pointer to enter the loop. + if limit := dstEvery - nSyms - 2; limit > 0 && br[0].canUseAsm() && br[1].canUseAsm() && br[2].canUseAsm() && br[3].canUseAsm() { + for i := range br { + br[i].prepareForAsm() + } ctx := decompress4xContext{ pbr: &br, peekBits: uint8((64 - d.actualTableLog) & 63), // see: bitReaderShifted.peekBitsFast() out: &out[0], dstEvery: dstEvery, tbl: &single[0], - limit: &out[dstEvery-4], // Always stop decoding when first buffer gets here to avoid writing OOB on last. + limit: &out[limit], + // The 6-byte jump table sits below stream 0 inside src, so the + // lowest window may reach into it; restoreFromAsm accounts for + // bytes below a stream's start. Bounding by src rather than by + // stream 0 keeps the asm running about six bytes longer, which + // matters for streams with very short codes. + ilowest: &src[0], } - if use8BitTables { + for i := range br { + ctx.ip[i] = &br[i].in[br[i].off] + } + switch nSyms { + case fast4bSymbols: + decompress4x_4b_main_loop_asm(&ctx) + case fast8bSymbols: decompress4x_8b_main_loop_asm(&ctx) - } else { + default: decompress4x_main_loop_asm(&ctx) } decoded = ctx.decoded out = out[decoded/4:] + for i := range br { + if err := br[i].restoreFromAsm(); err != nil { + return nil, err + } + } } // Decode remaining. @@ -128,18 +175,20 @@ func (d *Decoder) Decompress4X(dst, src []byte) ([]byte, error) { return dst, nil } +// decompress1xContext is the argument block of the Decompress1X asm loops. +// Go fills every field but decoded; the asm advances ip, and on return +// leaves the bit reader in the form bitReaderShifted.restoreFromAsm expects. type decompress1xContext struct { pbr *bitReaderShifted peekBits uint8 out *byte - outCap int + outCap int // no write reaches out[outCap] tbl *dEntrySingle decoded int + ilowest *byte // start of the stream; no read goes below it + ip *byte // the 8-byte input window, advanced by the asm } -// Error reported by asm implementations -const error_max_decoded_size_exeeded = -1 - // Decompress1X will decompress a 1X encoded stream. // The cap of the output buffer will be the maximum decompressed size. // The length of the supplied input must match the end of a block exactly. @@ -158,25 +207,36 @@ func (d *Decoder) Decompress1X(dst, src []byte) ([]byte, error) { const tlSize = 1 << tableLogMax const tlMask = tlSize - 1 - if maxDecodedSize >= 4 { + // The asm decodes whole batches of symbols and only re-checks its + // bounds between them, so it needs at least one batch of room (the + // output bound rounds nSyms up to 16, see _generate/gen.go) and a full + // 8-byte window ahead of the read pointer to enter the loop. It also + // needs at most 7 bits consumed on entry so that a batch cannot run + // the container dry, which prepareForAsm establishes. + decoded := 0 + if maxDecodedSize >= 16 && br.canUseAsm() { + br.prepareForAsm() ctx := decompress1xContext{ pbr: &br, out: &dst[0], outCap: maxDecodedSize, peekBits: uint8((64 - d.actualTableLog) & 63), // see: bitReaderShifted.peekBitsFast() tbl: &d.dt.single[0], + ilowest: &src[0], + ip: &br.in[br.off], } - - decompress1x_main_loop_asm(&ctx) - if ctx.decoded == error_max_decoded_size_exeeded { - return nil, ErrMaxDecodedSizeExceeded + if d.actualTableLog <= 4 { + decompress1x_4b_main_loop_asm(&ctx) + } else if d.actualTableLog <= 8 { + decompress1x_8b_main_loop_asm(&ctx) + } else { + decompress1x_main_loop_asm(&ctx) } - - dst = dst[:ctx.decoded] + decoded = ctx.decoded } + dst = dst[:decoded] - // br < 8, so uint8 is fine - bitsLeft := uint8(br.off)*8 + 64 - br.bitsRead + bitsLeft := br.remaining() for bitsLeft > 0 { br.fill() if len(dst) >= maxDecodedSize { @@ -186,7 +246,7 @@ func (d *Decoder) Decompress1X(dst, src []byte) ([]byte, error) { v := d.dt.single[br.peekBitsFast(d.actualTableLog)&tlMask] nBits := uint8(v.entry) br.advance(nBits) - bitsLeft -= nBits + bitsLeft -= uint(nBits) dst = append(dst, uint8(v.entry>>8)) } return dst, br.close() diff --git a/vendor/github.com/klauspost/compress/zstd/blockdec.go b/vendor/github.com/klauspost/compress/zstd/blockdec.go index 51f9da03875b..89b20d6a8421 100644 --- a/vendor/github.com/klauspost/compress/zstd/blockdec.go +++ b/vendor/github.com/klauspost/compress/zstd/blockdec.go @@ -240,9 +240,11 @@ func (b *blockDec) decodeBuf(hist *history) error { b.dst[i] = v } hist.appendKeep(b.dst) + hist.decoders.consumeSyncLen(len(b.dst)) return nil case blockTypeRaw: hist.appendKeep(b.data) + hist.decoders.consumeSyncLen(len(b.data)) return nil case blockTypeCompressed: saved := b.dst @@ -487,6 +489,7 @@ func (b *blockDec) decodeCompressed(hist *history) error { } if hist.decoders.nSeqs == 0 { b.dst = append(b.dst, hist.decoders.literals...) + hist.decoders.consumeSyncLen(len(hist.decoders.literals)) return nil } before := len(hist.decoders.out) diff --git a/vendor/github.com/klauspost/compress/zstd/decoder.go b/vendor/github.com/klauspost/compress/zstd/decoder.go index c7e500f02a95..90ef3bb03f4a 100644 --- a/vendor/github.com/klauspost/compress/zstd/decoder.go +++ b/vendor/github.com/klauspost/compress/zstd/decoder.go @@ -7,8 +7,11 @@ package zstd import ( "context" "encoding/binary" + "errors" "io" + "maps" "sync" + "sync/atomic" "github.com/klauspost/compress/zstd/internal/xxhash" ) @@ -20,7 +23,8 @@ import ( // A decoder can safely be re-used even if the previous stream failed. // To release the resources, you must call the Close() function on a decoder. type Decoder struct { - o decoderOptions + // publishedOptions contains the current immutable decoder options. + publishedOptions atomic.Pointer[decoderOptions] // Unreferenced decoders, ready for use. decoders chan *blockDec @@ -84,9 +88,10 @@ var ( func NewReader(r io.Reader, opts ...DOption) (*Decoder, error) { initPredefined() var d Decoder - d.o.setDefault() - for _, o := range opts { - err := o(&d.o) + var o decoderOptions + o.setDefault() + for _, opt := range opts { + err := opt(&o) if err != nil { return nil, err } @@ -99,17 +104,19 @@ func NewReader(r io.Reader, opts ...DOption) (*Decoder, error) { } // Initialize dict map if needed. - if d.o.dicts == nil { - d.o.dicts = make(map[uint32]*dict) + if o.dicts == nil { + o.dicts = make(map[uint32]*dict) } // Create decoders - d.decoders = make(chan *blockDec, d.o.concurrent) - for i := 0; i < d.o.concurrent; i++ { - dec := newBlockDec(d.o.lowMem) - dec.localFrame = newFrameDec(d.o) + d.decoders = make(chan *blockDec, o.concurrent) + for i := 0; i < o.concurrent; i++ { + dec := newBlockDec(o.lowMem) + dec.localFrame = newFrameDec(o) + dec.localFrame.o.dicts = nil d.decoders <- dec } + d.publishedOptions.Store(&o) if r == nil { return &d, nil @@ -171,6 +178,9 @@ func (d *Decoder) Reset(r io.Reader) error { d.drainOutput() d.syncStream.br.r = nil + if d.frame != nil { + d.frame.o.dicts = nil + } if r == nil { d.current.err = ErrDecoderNilInput if len(d.current.b) > 0 { @@ -180,8 +190,10 @@ func (d *Decoder) Reset(r io.Reader) error { return nil } + o := *d.publishedOptions.Load() + // If bytes buffer and < 5MB, do sync decoding anyway. - if bb, ok := r.(byter); ok && bb.Len() < d.o.decodeBufsBelow && !d.o.limitToCap { + if bb, ok := r.(byter); ok && bb.Len() < o.decodeBufsBelow && !o.limitToCap { bb2 := bb if debugDecoder { println("*bytes.Buffer detected, doing sync decode, len:", bb.Len()) @@ -193,18 +205,30 @@ func (d *Decoder) Reset(r io.Reader) error { } dst, err := d.DecodeAll(b, dst) - if err == nil { - err = io.EOF + if !errors.Is(err, ErrDecoderSizeExceeded) { + if err == nil { + err = io.EOF + } + // Save output buffer + d.syncStream.dstBuf = dst + d.current.b = dst + d.current.err = err + d.current.flushed = true + if debugDecoder { + println("sync decode to", len(dst), "bytes, err:", err) + } + return nil } - // Save output buffer - d.syncStream.dstBuf = dst - d.current.b = dst - d.current.err = err - d.current.flushed = true + // The output does not fit WithDecoderMaxMemory, which caps the total + // decoded size when decoding in memory. A Reader follows the + // streaming contract, where the option caps the window instead, so + // decode this input as a stream after all; the shortcut has not + // consumed the reader. Drop the oversized buffer rather than keep it + // for reuse. if debugDecoder { - println("sync decode to", len(dst), "bytes, err:", err) + println("sync decode exceeded max memory after", len(dst), "bytes; streaming instead") } - return nil + d.syncStream.dstBuf = nil } // Remove current block. d.stashDecoder() @@ -217,14 +241,16 @@ func (d *Decoder) Reset(r io.Reader) error { // Ensure no-one else is still running... d.streamWg.Wait() if d.frame == nil { - d.frame = newFrameDec(d.o) + d.frame = newFrameDec(o) + } else { + d.frame.setOptions(o) } - if d.o.concurrent == 1 { + if o.concurrent == 1 { return d.startSyncDecoder(r) } - d.current.output = make(chan decodeOutput, d.o.concurrent) + d.current.output = make(chan decodeOutput, o.concurrent) ctx, cancel := context.WithCancel(context.Background()) d.current.cancel = cancel d.streamWg.Add(1) @@ -238,13 +264,21 @@ func (d *Decoder) Reset(r io.Reader) error { // Options are applied on top of the existing options. // Some options cannot be changed on reset and will return an error. func (d *Decoder) ResetWithOptions(r io.Reader, opts ...DOption) error { - d.o.resetOpt = true - defer func() { d.o.resetOpt = false }() + if d.current.err == ErrDecoderClosed { + return d.current.err + } + next := *d.publishedOptions.Load() + next.dicts = maps.Clone(next.dicts) + next.resetOpt = true for _, o := range opts { - if err := o(&d.o); err != nil { + if err := o(&next); err != nil { + next.resetOpt = false + d.publishedOptions.Store(&next) return err } } + next.resetOpt = false + d.publishedOptions.Store(&next) return d.Reset(r) } @@ -324,6 +358,8 @@ func (d *Decoder) DecodeAll(input, dst []byte) ([]byte, error) { // Grab a block decoder and frame decoder. block := <-d.decoders frame := block.localFrame + o := *d.publishedOptions.Load() + frame.setOptions(o) initialSize := len(dst) defer func() { if debugDecoder { @@ -335,6 +371,7 @@ func (d *Decoder) DecodeAll(input, dst []byte) ([]byte, error) { frame.history.decoders.br.in = nil frame.history.decoders.br.cursor = 0 } + frame.o.dicts = nil d.decoders <- block }() frame.bBuf = input @@ -354,20 +391,20 @@ func (d *Decoder) DecodeAll(input, dst []byte) ([]byte, error) { if err = d.setDict(frame); err != nil { return nil, err } - if frame.WindowSize > d.o.maxWindowSize { + if frame.WindowSize > frame.o.maxWindowSize { if debugDecoder { - println("window size exceeded:", frame.WindowSize, ">", d.o.maxWindowSize) + println("window size exceeded:", frame.WindowSize, ">", frame.o.maxWindowSize) } return dst, ErrWindowSizeExceeded } if frame.FrameContentSize != fcsUnknown { - if frame.FrameContentSize > d.o.maxDecodedSize-uint64(len(dst)-initialSize) { + if frame.FrameContentSize > frame.o.maxDecodedSize-uint64(len(dst)-initialSize) { if debugDecoder { - println("decoder size exceeded; fcs:", frame.FrameContentSize, "> mcs:", d.o.maxDecodedSize-uint64(len(dst)-initialSize), "len:", len(dst)) + println("decoder size exceeded; fcs:", frame.FrameContentSize, "> mcs:", frame.o.maxDecodedSize-uint64(len(dst)-initialSize), "len:", len(dst)) } return dst, ErrDecoderSizeExceeded } - if d.o.limitToCap && frame.FrameContentSize > uint64(cap(dst)-len(dst)) { + if frame.o.limitToCap && frame.FrameContentSize > uint64(cap(dst)-len(dst)) { if debugDecoder { println("decoder size exceeded; fcs:", frame.FrameContentSize, "> (cap-len)", cap(dst)-len(dst)) } @@ -380,14 +417,14 @@ func (d *Decoder) DecodeAll(input, dst []byte) ([]byte, error) { } } - if cap(dst) == 0 && !d.o.limitToCap { + if cap(dst) == 0 && !frame.o.limitToCap { // Allocate len(input) * 2 by default if nothing is provided // and we didn't get frame content size. size := min( // Cap to 1 MB. len(input)*2, 1<<20) - if uint64(size) > d.o.maxDecodedSize { - size = int(d.o.maxDecodedSize) + if uint64(size) > frame.o.maxDecodedSize { + size = int(frame.o.maxDecodedSize) } dst = make([]byte, 0, size) } @@ -396,7 +433,7 @@ func (d *Decoder) DecodeAll(input, dst []byte) ([]byte, error) { if err != nil { return dst, err } - if uint64(len(dst)-initialSize) > d.o.maxDecodedSize { + if uint64(len(dst)-initialSize) > frame.o.maxDecodedSize { return dst, ErrDecoderSizeExceeded } if len(frame.bBuf) == 0 { @@ -459,7 +496,7 @@ func (d *Decoder) nextBlock(blocking bool) (ok bool) { println("got", len(d.current.b), "bytes, error:", d.current.err, "data crc:", tmp) } - if d.o.ignoreChecksum { + if d.frame.o.ignoreChecksum { return true } @@ -497,7 +534,7 @@ func (d *Decoder) nextBlockSync() (ok bool) { if d.current.err != nil { return false } - if d.frame.WindowSize > d.o.maxDecodedSize || d.frame.WindowSize > d.o.maxWindowSize { + if d.frame.WindowSize > d.frame.o.maxDecodedSize || d.frame.WindowSize > d.frame.o.maxWindowSize { d.current.err = ErrDecoderSizeExceeded return false } @@ -546,11 +583,11 @@ func (d *Decoder) nextBlockSync() (ok bool) { // Update/Check CRC if d.frame.HasCheckSum { - if !d.o.ignoreChecksum { + if !d.frame.o.ignoreChecksum { d.frame.crc.Write(d.current.b) } if d.current.d.Last { - if !d.o.ignoreChecksum { + if !d.frame.o.ignoreChecksum { d.current.err = d.frame.checkCRC() } else { d.current.err = d.frame.consumeCRC() @@ -656,8 +693,8 @@ func (d *Decoder) startStreamDecoder(ctx context.Context, r io.Reader, output ch defer d.streamWg.Done() br := readerWrapper{r: r} - var seqDecode = make(chan *blockDec, d.o.concurrent) - var seqExecute = make(chan *blockDec, d.o.concurrent) + var seqDecode = make(chan *blockDec, d.frame.o.concurrent) + var seqExecute = make(chan *blockDec, d.frame.o.concurrent) // Async 1: Decode sequences... go func() { @@ -858,9 +895,9 @@ decodeStream: if err == nil { err = d.setDict(frame) } - if err == nil && d.frame.WindowSize > d.o.maxWindowSize { + if err == nil && d.frame.WindowSize > d.frame.o.maxWindowSize { if debugDecoder { - println("decoder size exceeded, fws:", d.frame.WindowSize, "> mws:", d.o.maxWindowSize) + println("decoder size exceeded, fws:", d.frame.WindowSize, "> mws:", d.frame.o.maxWindowSize) } err = ErrDecoderSizeExceeded @@ -940,7 +977,7 @@ decodeStream: } func (d *Decoder) setDict(frame *frameDec) (err error) { - dict, ok := d.o.dicts[frame.DictionaryID] + dict, ok := frame.o.dicts[frame.DictionaryID] if ok { if debugDecoder { println("setting dict", frame.DictionaryID) diff --git a/vendor/github.com/klauspost/compress/zstd/framedec.go b/vendor/github.com/klauspost/compress/zstd/framedec.go index d88f067e5c2e..81d8b9c922a5 100644 --- a/vendor/github.com/klauspost/compress/zstd/framedec.go +++ b/vendor/github.com/klauspost/compress/zstd/framedec.go @@ -49,13 +49,16 @@ const ( ) func newFrameDec(o decoderOptions) *frameDec { + d := frameDec{} + d.setOptions(o) + return &d +} + +func (d *frameDec) setOptions(o decoderOptions) { if o.maxWindowSize > o.maxDecodedSize { o.maxWindowSize = o.maxDecodedSize } - d := frameDec{ - o: o, - } - return &d + d.o = o } // reset will read the frame header and prepare for block decoding. diff --git a/vendor/github.com/klauspost/compress/zstd/fse_decoder_arm64.s b/vendor/github.com/klauspost/compress/zstd/fse_decoder_arm64.s index b6b460787803..8ca913eaec69 100644 --- a/vendor/github.com/klauspost/compress/zstd/fse_decoder_arm64.s +++ b/vendor/github.com/klauspost/compress/zstd/fse_decoder_arm64.s @@ -47,12 +47,10 @@ init_main_loop_condition: // Spread symbols // Calculate table step - MOVD R0, R8 - LSR $0x01, R8, R8 - MOVD R0, R9 - LSR $0x03, R9, R9 - ADD R9, R8, R8 - ADD $3, R8, R8 + LSR $0x01, R0, R8 + LSR $0x03, R0, R9 + ADD R9, R8, R8 + ADD $3, R8, R8 // Fill add bits values SUB $1, R0, R9 diff --git a/vendor/github.com/klauspost/compress/zstd/seqdec.go b/vendor/github.com/klauspost/compress/zstd/seqdec.go index 0bfb0e43c7b6..d1a7732fcc64 100644 --- a/vendor/github.com/klauspost/compress/zstd/seqdec.go +++ b/vendor/github.com/klauspost/compress/zstd/seqdec.go @@ -99,6 +99,29 @@ func (s *sequenceDecs) initialize(br *bitReader, hist *history, out []byte) erro return nil } +// consumeSyncLen accounts for n bytes of frame output produced outside +// decodeSync. maxSyncLen bounds how much the frame may still produce and +// decodeSyncSimple compares it with the output buffer's slack to decide +// whether the extended 16-byte copies are safe. Blocks with sequences +// subtracted their output; raw blocks, RLE blocks and compressed blocks +// holding only literals did not, so a frame that opened with a raw block +// overstated its remaining size for every block after it and ran the +// bounds-exact copies for the rest of the frame. +// +// Zero means "no bound" and selects the conservative paths, so a block that +// meets or exceeds the bound lands there; the frame-size checks in +// runDecoder reject the excess afterwards. +func (s *sequenceDecs) consumeSyncLen(n int) { + if s.maxSyncLen == 0 { + return + } + if uint64(n) >= s.maxSyncLen { + s.maxSyncLen = 0 + return + } + s.maxSyncLen -= uint64(n) +} + func (s *sequenceDecs) freeDecoders() { if f := s.litLengths.fse; f != nil && !f.preDefined { fseDecoderPool.Put(f) diff --git a/vendor/github.com/klauspost/compress/zstd/seqdec_amd64.s b/vendor/github.com/klauspost/compress/zstd/seqdec_amd64.s index 9c4f8d3528dd..798d940231fe 100644 --- a/vendor/github.com/klauspost/compress/zstd/seqdec_amd64.s +++ b/vendor/github.com/klauspost/compress/zstd/seqdec_amd64.s @@ -4,7 +4,7 @@ // func sequenceDecs_decode_amd64(s *sequenceDecs, br *bitReader, ctx *decodeAsmContext) int // Requires: CMOV -TEXT ·sequenceDecs_decode_amd64(SB), $8-32 +TEXT ·sequenceDecs_decode_amd64(SB), $40-32 MOVQ br+8(FP), CX MOVQ 24(CX), DX MOVBQZX 40(CX), BX @@ -16,6 +16,14 @@ TEXT ·sequenceDecs_decode_amd64(SB), $8-32 MOVQ 72(AX), DI MOVQ 80(AX), R8 MOVQ 88(AX), R9 + MOVQ (AX), CX + MOVQ CX, 8(SP) + MOVQ 24(AX), CX + MOVQ CX, 16(SP) + MOVQ 48(AX), CX + MOVQ CX, 24(SP) + MOVQ 96(AX), CX + MOVQ CX, 32(SP) MOVQ 104(AX), R10 MOVQ s+0(FP), AX MOVQ 144(AX), R11 @@ -59,42 +67,28 @@ sequenceDecs_decode_amd64_fill_end: MOVQ BX, CX MOVQ DX, R15 SHLQ CL, R15 + SHRQ $0x01, R15 MOVBLZX AH, CX SHRQ $0x20, AX - TESTQ CX, CX - JZ sequenceDecs_decode_amd64_of_update_zero ADDQ CX, BX - CMPQ BX, $0x40 - JA sequenceDecs_decode_amd64_of_update_zero - CMPQ CX, $0x40 - JAE sequenceDecs_decode_amd64_of_update_zero - NEGQ CX + XORQ $0x3f, CX SHRQ CL, R15 ADDQ R15, AX - -sequenceDecs_decode_amd64_of_update_zero: - MOVQ AX, 16(R10) + MOVQ AX, 16(R10) // Update match length MOVQ R8, AX MOVQ BX, CX MOVQ DX, R15 SHLQ CL, R15 + SHRQ $0x01, R15 MOVBLZX AH, CX SHRQ $0x20, AX - TESTQ CX, CX - JZ sequenceDecs_decode_amd64_ml_update_zero ADDQ CX, BX - CMPQ BX, $0x40 - JA sequenceDecs_decode_amd64_ml_update_zero - CMPQ CX, $0x40 - JAE sequenceDecs_decode_amd64_ml_update_zero - NEGQ CX + XORQ $0x3f, CX SHRQ CL, R15 ADDQ R15, AX - -sequenceDecs_decode_amd64_ml_update_zero: - MOVQ AX, 8(R10) + MOVQ AX, 8(R10) // Fill bitreader to have enough for the remaining CMPQ SI, $0x08 @@ -130,86 +124,72 @@ sequenceDecs_decode_amd64_fill_2_end: MOVQ BX, CX MOVQ DX, R15 SHLQ CL, R15 + SHRQ $0x01, R15 MOVBLZX AH, CX SHRQ $0x20, AX - TESTQ CX, CX - JZ sequenceDecs_decode_amd64_ll_update_zero ADDQ CX, BX - CMPQ BX, $0x40 - JA sequenceDecs_decode_amd64_ll_update_zero - CMPQ CX, $0x40 - JAE sequenceDecs_decode_amd64_ll_update_zero - NEGQ CX + XORQ $0x3f, CX SHRQ CL, R15 ADDQ R15, AX - -sequenceDecs_decode_amd64_ll_update_zero: - MOVQ AX, (R10) + MOVQ AX, (R10) // Fill bitreader for state updates MOVQ R14, (SP) MOVQ R9, AX SHRQ $0x08, AX MOVBQZX AL, AX - MOVQ ctx+16(FP), CX - CMPQ 96(CX), $0x00 + CMPQ 32(SP), $0x00 JZ sequenceDecs_decode_amd64_skip_update // Update Literal Length State MOVBQZX DI, R14 SHRL $0x10, DI - LEAQ (BX)(R14*1), CX + MOVQ BX, CX MOVQ DX, R15 - MOVQ CX, BX - ROLQ CL, R15 - MOVL $0x00000001, BP - MOVBLZX R14, CX - SHLL CL, BP - DECL BP - ANDQ BP, R15 + SHLQ CL, R15 + SHRQ $0x01, R15 + ADDQ R14, BX + XORQ $0x3f, R14 + MOVQ R14, CX + SHRQ CL, R15 ADDQ R15, DI // Load ctx.llTable - MOVQ ctx+16(FP), CX - MOVQ (CX), CX + MOVQ 8(SP), CX MOVQ (CX)(DI*8), DI // Update Match Length State MOVBQZX R8, R14 SHRL $0x10, R8 - LEAQ (BX)(R14*1), CX + MOVQ BX, CX MOVQ DX, R15 - MOVQ CX, BX - ROLQ CL, R15 - MOVL $0x00000001, BP - MOVBLZX R14, CX - SHLL CL, BP - DECL BP - ANDQ BP, R15 + SHLQ CL, R15 + SHRQ $0x01, R15 + ADDQ R14, BX + XORQ $0x3f, R14 + MOVQ R14, CX + SHRQ CL, R15 ADDQ R15, R8 // Load ctx.mlTable - MOVQ ctx+16(FP), CX - MOVQ 24(CX), CX + MOVQ 16(SP), CX MOVQ (CX)(R8*8), R8 // Update Offset State MOVBQZX R9, R14 SHRL $0x10, R9 - LEAQ (BX)(R14*1), CX + MOVQ BX, CX MOVQ DX, R15 - MOVQ CX, BX - ROLQ CL, R15 - MOVL $0x00000001, BP - MOVBLZX R14, CX - SHLL CL, BP - DECL BP - ANDQ BP, R15 + SHLQ CL, R15 + SHRQ $0x01, R15 + ADDQ R14, BX + XORQ $0x3f, R14 + MOVQ R14, CX + SHRQ CL, R15 ADDQ R15, R9 // Load ctx.ofTable - MOVQ ctx+16(FP), CX - MOVQ 48(CX), CX + MOVQ 24(SP), CX MOVQ (CX)(R9*8), R9 sequenceDecs_decode_amd64_skip_update: @@ -290,8 +270,7 @@ sequenceDecs_decode_amd64_after_adjust: sequenceDecs_decode_amd64_match_len_ofs_ok: ADDQ $0x18, R10 - MOVQ ctx+16(FP), AX - DECQ 96(AX) + DECQ 32(SP) JNS sequenceDecs_decode_amd64_main_loop MOVQ s+0(FP), AX MOVQ R11, 144(AX) @@ -303,36 +282,54 @@ sequenceDecs_decode_amd64_match_len_ofs_ok: MOVQ SI, 32(AX) // Return success + MOVQ 32(SP), AX + MOVQ ctx+16(FP), CX + MOVQ AX, 96(CX) MOVQ $0x00000000, ret+24(FP) RET // Return with match length error sequenceDecs_decode_amd64_error_match_len_ofs_mismatch: + MOVQ 32(SP), AX + MOVQ ctx+16(FP), CX + MOVQ AX, 96(CX) MOVQ $0x00000001, ret+24(FP) RET // Return with match too long error sequenceDecs_decode_amd64_error_match_len_too_big: + MOVQ 32(SP), AX + MOVQ ctx+16(FP), CX + MOVQ AX, 96(CX) MOVQ $0x00000002, ret+24(FP) RET // Return with match offset too long error + MOVQ 32(SP), AX + MOVQ ctx+16(FP), CX + MOVQ AX, 96(CX) MOVQ $0x00000003, ret+24(FP) RET // Return with not enough literals error error_not_enough_literals: + MOVQ 32(SP), AX + MOVQ ctx+16(FP), CX + MOVQ AX, 96(CX) MOVQ $0x00000004, ret+24(FP) RET // Return with overread error error_overread: + MOVQ 32(SP), AX + MOVQ ctx+16(FP), CX + MOVQ AX, 96(CX) MOVQ $0x00000006, ret+24(FP) RET // func sequenceDecs_decode_56_amd64(s *sequenceDecs, br *bitReader, ctx *decodeAsmContext) int // Requires: CMOV -TEXT ·sequenceDecs_decode_56_amd64(SB), $8-32 +TEXT ·sequenceDecs_decode_56_amd64(SB), $40-32 MOVQ br+8(FP), CX MOVQ 24(CX), DX MOVBQZX 40(CX), BX @@ -344,6 +341,14 @@ TEXT ·sequenceDecs_decode_56_amd64(SB), $8-32 MOVQ 72(AX), DI MOVQ 80(AX), R8 MOVQ 88(AX), R9 + MOVQ (AX), CX + MOVQ CX, 8(SP) + MOVQ 24(AX), CX + MOVQ CX, 16(SP) + MOVQ 48(AX), CX + MOVQ CX, 24(SP) + MOVQ 96(AX), CX + MOVQ CX, 32(SP) MOVQ 104(AX), R10 MOVQ s+0(FP), AX MOVQ 144(AX), R11 @@ -387,128 +392,100 @@ sequenceDecs_decode_56_amd64_fill_end: MOVQ BX, CX MOVQ DX, R15 SHLQ CL, R15 + SHRQ $0x01, R15 MOVBLZX AH, CX SHRQ $0x20, AX - TESTQ CX, CX - JZ sequenceDecs_decode_56_amd64_of_update_zero ADDQ CX, BX - CMPQ BX, $0x40 - JA sequenceDecs_decode_56_amd64_of_update_zero - CMPQ CX, $0x40 - JAE sequenceDecs_decode_56_amd64_of_update_zero - NEGQ CX + XORQ $0x3f, CX SHRQ CL, R15 ADDQ R15, AX - -sequenceDecs_decode_56_amd64_of_update_zero: - MOVQ AX, 16(R10) + MOVQ AX, 16(R10) // Update match length MOVQ R8, AX MOVQ BX, CX MOVQ DX, R15 SHLQ CL, R15 + SHRQ $0x01, R15 MOVBLZX AH, CX SHRQ $0x20, AX - TESTQ CX, CX - JZ sequenceDecs_decode_56_amd64_ml_update_zero ADDQ CX, BX - CMPQ BX, $0x40 - JA sequenceDecs_decode_56_amd64_ml_update_zero - CMPQ CX, $0x40 - JAE sequenceDecs_decode_56_amd64_ml_update_zero - NEGQ CX + XORQ $0x3f, CX SHRQ CL, R15 ADDQ R15, AX - -sequenceDecs_decode_56_amd64_ml_update_zero: - MOVQ AX, 8(R10) + MOVQ AX, 8(R10) // Update literal length MOVQ DI, AX MOVQ BX, CX MOVQ DX, R15 SHLQ CL, R15 + SHRQ $0x01, R15 MOVBLZX AH, CX SHRQ $0x20, AX - TESTQ CX, CX - JZ sequenceDecs_decode_56_amd64_ll_update_zero ADDQ CX, BX - CMPQ BX, $0x40 - JA sequenceDecs_decode_56_amd64_ll_update_zero - CMPQ CX, $0x40 - JAE sequenceDecs_decode_56_amd64_ll_update_zero - NEGQ CX + XORQ $0x3f, CX SHRQ CL, R15 ADDQ R15, AX - -sequenceDecs_decode_56_amd64_ll_update_zero: - MOVQ AX, (R10) + MOVQ AX, (R10) // Fill bitreader for state updates MOVQ R14, (SP) MOVQ R9, AX SHRQ $0x08, AX MOVBQZX AL, AX - MOVQ ctx+16(FP), CX - CMPQ 96(CX), $0x00 + CMPQ 32(SP), $0x00 JZ sequenceDecs_decode_56_amd64_skip_update // Update Literal Length State MOVBQZX DI, R14 SHRL $0x10, DI - LEAQ (BX)(R14*1), CX + MOVQ BX, CX MOVQ DX, R15 - MOVQ CX, BX - ROLQ CL, R15 - MOVL $0x00000001, BP - MOVBLZX R14, CX - SHLL CL, BP - DECL BP - ANDQ BP, R15 + SHLQ CL, R15 + SHRQ $0x01, R15 + ADDQ R14, BX + XORQ $0x3f, R14 + MOVQ R14, CX + SHRQ CL, R15 ADDQ R15, DI // Load ctx.llTable - MOVQ ctx+16(FP), CX - MOVQ (CX), CX + MOVQ 8(SP), CX MOVQ (CX)(DI*8), DI // Update Match Length State MOVBQZX R8, R14 SHRL $0x10, R8 - LEAQ (BX)(R14*1), CX + MOVQ BX, CX MOVQ DX, R15 - MOVQ CX, BX - ROLQ CL, R15 - MOVL $0x00000001, BP - MOVBLZX R14, CX - SHLL CL, BP - DECL BP - ANDQ BP, R15 + SHLQ CL, R15 + SHRQ $0x01, R15 + ADDQ R14, BX + XORQ $0x3f, R14 + MOVQ R14, CX + SHRQ CL, R15 ADDQ R15, R8 // Load ctx.mlTable - MOVQ ctx+16(FP), CX - MOVQ 24(CX), CX + MOVQ 16(SP), CX MOVQ (CX)(R8*8), R8 // Update Offset State MOVBQZX R9, R14 SHRL $0x10, R9 - LEAQ (BX)(R14*1), CX + MOVQ BX, CX MOVQ DX, R15 - MOVQ CX, BX - ROLQ CL, R15 - MOVL $0x00000001, BP - MOVBLZX R14, CX - SHLL CL, BP - DECL BP - ANDQ BP, R15 + SHLQ CL, R15 + SHRQ $0x01, R15 + ADDQ R14, BX + XORQ $0x3f, R14 + MOVQ R14, CX + SHRQ CL, R15 ADDQ R15, R9 // Load ctx.ofTable - MOVQ ctx+16(FP), CX - MOVQ 48(CX), CX + MOVQ 24(SP), CX MOVQ (CX)(R9*8), R9 sequenceDecs_decode_56_amd64_skip_update: @@ -589,8 +566,7 @@ sequenceDecs_decode_56_amd64_after_adjust: sequenceDecs_decode_56_amd64_match_len_ofs_ok: ADDQ $0x18, R10 - MOVQ ctx+16(FP), AX - DECQ 96(AX) + DECQ 32(SP) JNS sequenceDecs_decode_56_amd64_main_loop MOVQ s+0(FP), AX MOVQ R11, 144(AX) @@ -602,36 +578,54 @@ sequenceDecs_decode_56_amd64_match_len_ofs_ok: MOVQ SI, 32(AX) // Return success + MOVQ 32(SP), AX + MOVQ ctx+16(FP), CX + MOVQ AX, 96(CX) MOVQ $0x00000000, ret+24(FP) RET // Return with match length error sequenceDecs_decode_56_amd64_error_match_len_ofs_mismatch: + MOVQ 32(SP), AX + MOVQ ctx+16(FP), CX + MOVQ AX, 96(CX) MOVQ $0x00000001, ret+24(FP) RET // Return with match too long error sequenceDecs_decode_56_amd64_error_match_len_too_big: + MOVQ 32(SP), AX + MOVQ ctx+16(FP), CX + MOVQ AX, 96(CX) MOVQ $0x00000002, ret+24(FP) RET // Return with match offset too long error + MOVQ 32(SP), AX + MOVQ ctx+16(FP), CX + MOVQ AX, 96(CX) MOVQ $0x00000003, ret+24(FP) RET // Return with not enough literals error error_not_enough_literals: + MOVQ 32(SP), AX + MOVQ ctx+16(FP), CX + MOVQ AX, 96(CX) MOVQ $0x00000004, ret+24(FP) RET // Return with overread error error_overread: + MOVQ 32(SP), AX + MOVQ ctx+16(FP), CX + MOVQ AX, 96(CX) MOVQ $0x00000006, ret+24(FP) RET // func sequenceDecs_decode_bmi2(s *sequenceDecs, br *bitReader, ctx *decodeAsmContext) int // Requires: BMI, BMI2, CMOV -TEXT ·sequenceDecs_decode_bmi2(SB), $8-32 +TEXT ·sequenceDecs_decode_bmi2(SB), $40-32 MOVQ br+8(FP), BX MOVQ 24(BX), AX MOVBQZX 40(BX), DX @@ -643,6 +637,14 @@ TEXT ·sequenceDecs_decode_bmi2(SB), $8-32 MOVQ 72(CX), SI MOVQ 80(CX), DI MOVQ 88(CX), R8 + MOVQ (CX), R9 + MOVQ R9, 8(SP) + MOVQ 24(CX), R9 + MOVQ R9, 16(SP) + MOVQ 48(CX), R9 + MOVQ R9, 24(SP) + MOVQ 96(CX), R9 + MOVQ R9, 32(SP) MOVQ 104(CX), R9 MOVQ s+0(FP), CX MOVQ 144(CX), R10 @@ -753,8 +755,7 @@ sequenceDecs_decode_bmi2_fill_2_end: MOVQ R13, (SP) MOVQ $0x00000808, CX BEXTRQ CX, R8, R13 - MOVQ ctx+16(FP), CX - CMPQ 96(CX), $0x00 + CMPQ 32(SP), $0x00 JZ sequenceDecs_decode_bmi2_skip_update LEAQ (SI)(DI*1), R14 ADDQ R8, R14 @@ -772,8 +773,7 @@ sequenceDecs_decode_bmi2_fill_2_end: ADDQ CX, R8 // Load ctx.ofTable - MOVQ ctx+16(FP), CX - MOVQ 48(CX), CX + MOVQ 24(SP), CX MOVQ (CX)(R8*8), R8 // Update Match Length State @@ -783,8 +783,7 @@ sequenceDecs_decode_bmi2_fill_2_end: ADDQ CX, DI // Load ctx.mlTable - MOVQ ctx+16(FP), CX - MOVQ 24(CX), CX + MOVQ 16(SP), CX MOVQ (CX)(DI*8), DI // Update Literal Length State @@ -793,8 +792,7 @@ sequenceDecs_decode_bmi2_fill_2_end: ADDQ CX, SI // Load ctx.llTable - MOVQ ctx+16(FP), CX - MOVQ (CX), CX + MOVQ 8(SP), CX MOVQ (CX)(SI*8), SI sequenceDecs_decode_bmi2_skip_update: @@ -875,8 +873,7 @@ sequenceDecs_decode_bmi2_after_adjust: sequenceDecs_decode_bmi2_match_len_ofs_ok: ADDQ $0x18, R9 - MOVQ ctx+16(FP), CX - DECQ 96(CX) + DECQ 32(SP) JNS sequenceDecs_decode_bmi2_main_loop MOVQ s+0(FP), CX MOVQ R10, 144(CX) @@ -888,36 +885,54 @@ sequenceDecs_decode_bmi2_match_len_ofs_ok: MOVQ BX, 32(CX) // Return success + MOVQ 32(SP), AX + MOVQ ctx+16(FP), CX + MOVQ AX, 96(CX) MOVQ $0x00000000, ret+24(FP) RET // Return with match length error sequenceDecs_decode_bmi2_error_match_len_ofs_mismatch: + MOVQ 32(SP), AX + MOVQ ctx+16(FP), CX + MOVQ AX, 96(CX) MOVQ $0x00000001, ret+24(FP) RET // Return with match too long error sequenceDecs_decode_bmi2_error_match_len_too_big: + MOVQ 32(SP), AX + MOVQ ctx+16(FP), CX + MOVQ AX, 96(CX) MOVQ $0x00000002, ret+24(FP) RET // Return with match offset too long error + MOVQ 32(SP), AX + MOVQ ctx+16(FP), CX + MOVQ AX, 96(CX) MOVQ $0x00000003, ret+24(FP) RET // Return with not enough literals error error_not_enough_literals: + MOVQ 32(SP), AX + MOVQ ctx+16(FP), CX + MOVQ AX, 96(CX) MOVQ $0x00000004, ret+24(FP) RET // Return with overread error error_overread: + MOVQ 32(SP), AX + MOVQ ctx+16(FP), CX + MOVQ AX, 96(CX) MOVQ $0x00000006, ret+24(FP) RET // func sequenceDecs_decode_56_bmi2(s *sequenceDecs, br *bitReader, ctx *decodeAsmContext) int // Requires: BMI, BMI2, CMOV -TEXT ·sequenceDecs_decode_56_bmi2(SB), $8-32 +TEXT ·sequenceDecs_decode_56_bmi2(SB), $40-32 MOVQ br+8(FP), BX MOVQ 24(BX), AX MOVBQZX 40(BX), DX @@ -929,6 +944,14 @@ TEXT ·sequenceDecs_decode_56_bmi2(SB), $8-32 MOVQ 72(CX), SI MOVQ 80(CX), DI MOVQ 88(CX), R8 + MOVQ (CX), R9 + MOVQ R9, 8(SP) + MOVQ 24(CX), R9 + MOVQ R9, 16(SP) + MOVQ 48(CX), R9 + MOVQ R9, 24(SP) + MOVQ 96(CX), R9 + MOVQ R9, 32(SP) MOVQ 104(CX), R9 MOVQ s+0(FP), CX MOVQ 144(CX), R10 @@ -1010,8 +1033,7 @@ sequenceDecs_decode_56_bmi2_fill_end: MOVQ R13, (SP) MOVQ $0x00000808, CX BEXTRQ CX, R8, R13 - MOVQ ctx+16(FP), CX - CMPQ 96(CX), $0x00 + CMPQ 32(SP), $0x00 JZ sequenceDecs_decode_56_bmi2_skip_update LEAQ (SI)(DI*1), R14 ADDQ R8, R14 @@ -1029,8 +1051,7 @@ sequenceDecs_decode_56_bmi2_fill_end: ADDQ CX, R8 // Load ctx.ofTable - MOVQ ctx+16(FP), CX - MOVQ 48(CX), CX + MOVQ 24(SP), CX MOVQ (CX)(R8*8), R8 // Update Match Length State @@ -1040,8 +1061,7 @@ sequenceDecs_decode_56_bmi2_fill_end: ADDQ CX, DI // Load ctx.mlTable - MOVQ ctx+16(FP), CX - MOVQ 24(CX), CX + MOVQ 16(SP), CX MOVQ (CX)(DI*8), DI // Update Literal Length State @@ -1050,8 +1070,7 @@ sequenceDecs_decode_56_bmi2_fill_end: ADDQ CX, SI // Load ctx.llTable - MOVQ ctx+16(FP), CX - MOVQ (CX), CX + MOVQ 8(SP), CX MOVQ (CX)(SI*8), SI sequenceDecs_decode_56_bmi2_skip_update: @@ -1132,8 +1151,7 @@ sequenceDecs_decode_56_bmi2_after_adjust: sequenceDecs_decode_56_bmi2_match_len_ofs_ok: ADDQ $0x18, R9 - MOVQ ctx+16(FP), CX - DECQ 96(CX) + DECQ 32(SP) JNS sequenceDecs_decode_56_bmi2_main_loop MOVQ s+0(FP), CX MOVQ R10, 144(CX) @@ -1145,30 +1163,48 @@ sequenceDecs_decode_56_bmi2_match_len_ofs_ok: MOVQ BX, 32(CX) // Return success + MOVQ 32(SP), AX + MOVQ ctx+16(FP), CX + MOVQ AX, 96(CX) MOVQ $0x00000000, ret+24(FP) RET // Return with match length error sequenceDecs_decode_56_bmi2_error_match_len_ofs_mismatch: + MOVQ 32(SP), AX + MOVQ ctx+16(FP), CX + MOVQ AX, 96(CX) MOVQ $0x00000001, ret+24(FP) RET // Return with match too long error sequenceDecs_decode_56_bmi2_error_match_len_too_big: + MOVQ 32(SP), AX + MOVQ ctx+16(FP), CX + MOVQ AX, 96(CX) MOVQ $0x00000002, ret+24(FP) RET // Return with match offset too long error + MOVQ 32(SP), AX + MOVQ ctx+16(FP), CX + MOVQ AX, 96(CX) MOVQ $0x00000003, ret+24(FP) RET // Return with not enough literals error error_not_enough_literals: + MOVQ 32(SP), AX + MOVQ ctx+16(FP), CX + MOVQ AX, 96(CX) MOVQ $0x00000004, ret+24(FP) RET // Return with overread error error_overread: + MOVQ 32(SP), AX + MOVQ ctx+16(FP), CX + MOVQ AX, 96(CX) MOVQ $0x00000006, ret+24(FP) RET @@ -1203,9 +1239,11 @@ main_loop: MOVQ 8(AX), R13 // Copy literals - TESTQ R11, R11 - JZ check_offset - XORQ R14, R14 + MOVUPS (SI), X0 + MOVUPS X0, (BX) + CMPQ R11, $0x10 + JBE copy_1_end + MOVQ $0x00000010, R14 copy_1: MOVUPS (SI)(R14*1), X0 @@ -1213,12 +1251,13 @@ copy_1: ADDQ $0x10, R14 CMPQ R14, R11 JB copy_1 - ADDQ R11, SI - ADDQ R11, BX - ADDQ R11, DI + +copy_1_end: + ADDQ R11, SI + ADDQ R11, BX + ADDQ R11, DI // Malformed input if seq.mo > t+len(hist) || seq.mo > s.windowSize) -check_offset: LEAQ (DI)(R10*1), R11 CMPQ R12, R11 JG error_match_off_too_big @@ -1366,9 +1405,19 @@ copy_match: JA copy_overlapping_match // Copy non-overlapping match - ADDQ R13, DI - MOVQ BX, R12 + ADDQ R13, DI + MOVUPS (R11), X0 + MOVUPS X0, (BX) + CMPQ R13, $0x10 + JA copy_2_long + ADDQ R13, BX + JMP handle_loop + +copy_2_long: + LEAQ 16(BX), R12 ADDQ R13, BX + ADDQ $0x10, R11 + SUBQ $0x10, R13 copy_2: MOVUPS (R11), X0 @@ -1784,7 +1833,7 @@ empty_seqs: // func sequenceDecs_decodeSync_amd64(s *sequenceDecs, br *bitReader, ctx *decodeSyncAsmContext) int // Requires: CMOV, SSE -TEXT ·sequenceDecs_decodeSync_amd64(SB), $64-32 +TEXT ·sequenceDecs_decodeSync_amd64(SB), $96-32 MOVQ br+8(FP), CX MOVQ 24(CX), DX MOVBQZX 40(CX), BX @@ -1796,26 +1845,34 @@ TEXT ·sequenceDecs_decodeSync_amd64(SB), $64-32 MOVQ 72(AX), DI MOVQ 80(AX), R8 MOVQ 88(AX), R9 - XORQ CX, CX + MOVQ (AX), CX MOVQ CX, 8(SP) + MOVQ 24(AX), CX MOVQ CX, 16(SP) + MOVQ 48(AX), CX MOVQ CX, 24(SP) + MOVQ 96(AX), CX + MOVQ CX, 32(SP) + XORQ CX, CX + MOVQ CX, 40(SP) + MOVQ CX, 48(SP) + MOVQ CX, 56(SP) MOVQ 112(AX), R10 MOVQ 128(AX), CX - MOVQ CX, 32(SP) + MOVQ CX, 64(SP) MOVQ 144(AX), R11 MOVQ 136(AX), R12 MOVQ 200(AX), CX - MOVQ CX, 56(SP) + MOVQ CX, 88(SP) MOVQ 176(AX), CX - MOVQ CX, 48(SP) + MOVQ CX, 80(SP) MOVQ 184(AX), AX - MOVQ AX, 40(SP) - MOVQ 40(SP), AX - ADDQ AX, 48(SP) + MOVQ AX, 72(SP) + MOVQ 72(SP), AX + ADDQ AX, 80(SP) // Calculate pointer to s.out[cap(s.out)] (a past-end pointer) - ADDQ R10, 32(SP) + ADDQ R10, 64(SP) // outBase += outPosition ADDQ R12, R10 @@ -1857,42 +1914,28 @@ sequenceDecs_decodeSync_amd64_fill_end: MOVQ BX, CX MOVQ DX, R14 SHLQ CL, R14 + SHRQ $0x01, R14 MOVBLZX AH, CX SHRQ $0x20, AX - TESTQ CX, CX - JZ sequenceDecs_decodeSync_amd64_of_update_zero ADDQ CX, BX - CMPQ BX, $0x40 - JA sequenceDecs_decodeSync_amd64_of_update_zero - CMPQ CX, $0x40 - JAE sequenceDecs_decodeSync_amd64_of_update_zero - NEGQ CX + XORQ $0x3f, CX SHRQ CL, R14 ADDQ R14, AX - -sequenceDecs_decodeSync_amd64_of_update_zero: - MOVQ AX, 8(SP) + MOVQ AX, 40(SP) // Update match length MOVQ R8, AX MOVQ BX, CX MOVQ DX, R14 SHLQ CL, R14 + SHRQ $0x01, R14 MOVBLZX AH, CX SHRQ $0x20, AX - TESTQ CX, CX - JZ sequenceDecs_decodeSync_amd64_ml_update_zero ADDQ CX, BX - CMPQ BX, $0x40 - JA sequenceDecs_decodeSync_amd64_ml_update_zero - CMPQ CX, $0x40 - JAE sequenceDecs_decodeSync_amd64_ml_update_zero - NEGQ CX + XORQ $0x3f, CX SHRQ CL, R14 ADDQ R14, AX - -sequenceDecs_decodeSync_amd64_ml_update_zero: - MOVQ AX, 16(SP) + MOVQ AX, 48(SP) // Fill bitreader to have enough for the remaining CMPQ SI, $0x08 @@ -1928,92 +1971,78 @@ sequenceDecs_decodeSync_amd64_fill_2_end: MOVQ BX, CX MOVQ DX, R14 SHLQ CL, R14 + SHRQ $0x01, R14 MOVBLZX AH, CX SHRQ $0x20, AX - TESTQ CX, CX - JZ sequenceDecs_decodeSync_amd64_ll_update_zero ADDQ CX, BX - CMPQ BX, $0x40 - JA sequenceDecs_decodeSync_amd64_ll_update_zero - CMPQ CX, $0x40 - JAE sequenceDecs_decodeSync_amd64_ll_update_zero - NEGQ CX + XORQ $0x3f, CX SHRQ CL, R14 ADDQ R14, AX - -sequenceDecs_decodeSync_amd64_ll_update_zero: - MOVQ AX, 24(SP) + MOVQ AX, 56(SP) // Fill bitreader for state updates MOVQ R13, (SP) MOVQ R9, AX SHRQ $0x08, AX MOVBQZX AL, AX - MOVQ ctx+16(FP), CX - CMPQ 96(CX), $0x00 + CMPQ 32(SP), $0x00 JZ sequenceDecs_decodeSync_amd64_skip_update // Update Literal Length State MOVBQZX DI, R13 SHRL $0x10, DI - LEAQ (BX)(R13*1), CX + MOVQ BX, CX MOVQ DX, R14 - MOVQ CX, BX - ROLQ CL, R14 - MOVL $0x00000001, R15 - MOVBLZX R13, CX - SHLL CL, R15 - DECL R15 - ANDQ R15, R14 + SHLQ CL, R14 + SHRQ $0x01, R14 + ADDQ R13, BX + XORQ $0x3f, R13 + MOVQ R13, CX + SHRQ CL, R14 ADDQ R14, DI // Load ctx.llTable - MOVQ ctx+16(FP), CX - MOVQ (CX), CX + MOVQ 8(SP), CX MOVQ (CX)(DI*8), DI // Update Match Length State MOVBQZX R8, R13 SHRL $0x10, R8 - LEAQ (BX)(R13*1), CX + MOVQ BX, CX MOVQ DX, R14 - MOVQ CX, BX - ROLQ CL, R14 - MOVL $0x00000001, R15 - MOVBLZX R13, CX - SHLL CL, R15 - DECL R15 - ANDQ R15, R14 + SHLQ CL, R14 + SHRQ $0x01, R14 + ADDQ R13, BX + XORQ $0x3f, R13 + MOVQ R13, CX + SHRQ CL, R14 ADDQ R14, R8 // Load ctx.mlTable - MOVQ ctx+16(FP), CX - MOVQ 24(CX), CX + MOVQ 16(SP), CX MOVQ (CX)(R8*8), R8 // Update Offset State MOVBQZX R9, R13 SHRL $0x10, R9 - LEAQ (BX)(R13*1), CX + MOVQ BX, CX MOVQ DX, R14 - MOVQ CX, BX - ROLQ CL, R14 - MOVL $0x00000001, R15 - MOVBLZX R13, CX - SHLL CL, R15 - DECL R15 - ANDQ R15, R14 + SHLQ CL, R14 + SHRQ $0x01, R14 + ADDQ R13, BX + XORQ $0x3f, R13 + MOVQ R13, CX + SHRQ CL, R14 ADDQ R14, R9 // Load ctx.ofTable - MOVQ ctx+16(FP), CX - MOVQ 48(CX), CX + MOVQ 24(SP), CX MOVQ (CX)(R9*8), R9 sequenceDecs_decodeSync_amd64_skip_update: // Adjust offset MOVQ s+0(FP), CX - MOVQ 8(SP), R13 + MOVQ 40(SP), R13 CMPQ AX, $0x01 JBE sequenceDecs_decodeSync_amd64_adjust_offsetB_1_or_0 MOVUPS 144(CX), X0 @@ -2022,7 +2051,7 @@ sequenceDecs_decodeSync_amd64_skip_update: JMP sequenceDecs_decodeSync_amd64_after_adjust sequenceDecs_decodeSync_amd64_adjust_offsetB_1_or_0: - CMPQ 24(SP), $0x00000000 + CMPQ 56(SP), $0x00000000 JNE sequenceDecs_decodeSync_amd64_adjust_offset_maybezero INCQ R13 JMP sequenceDecs_decodeSync_amd64_adjust_offset_nonzero @@ -2057,14 +2086,11 @@ sequenceDecs_decodeSync_amd64_adjust_skip: MOVQ R14, R13 sequenceDecs_decodeSync_amd64_after_adjust: - MOVQ R13, 8(SP) + MOVQ R13, 40(SP) // Check values - MOVQ 16(SP), AX - MOVQ 24(SP), CX - LEAQ (AX)(CX*1), R14 - MOVQ s+0(FP), R15 - ADDQ R14, 256(R15) + MOVQ 48(SP), AX + MOVQ 56(SP), CX MOVQ ctx+16(FP), R14 SUBQ CX, 104(R14) JS error_not_enough_literals @@ -2076,50 +2102,49 @@ sequenceDecs_decodeSync_amd64_after_adjust: JNZ sequenceDecs_decodeSync_amd64_error_match_len_ofs_mismatch sequenceDecs_decodeSync_amd64_match_len_ofs_ok: - MOVQ 24(SP), AX - MOVQ 8(SP), CX - MOVQ 16(SP), R13 - // Check if we have enough space in s.out - LEAQ 16(AX)(R13*1), R14 + LEAQ 16(CX)(AX*1), R14 ADDQ R10, R14 - CMPQ R14, 32(SP) + CMPQ R14, 64(SP) JA error_not_enough_space // Copy literals - TESTQ AX, AX - JZ check_offset - XORQ R14, R14 + MOVUPS (R11), X0 + MOVUPS X0, (R10) + CMPQ CX, $0x10 + JBE copy_1_end + MOVQ $0x00000010, R14 copy_1: MOVUPS (R11)(R14*1), X0 MOVUPS X0, (R10)(R14*1) ADDQ $0x10, R14 - CMPQ R14, AX + CMPQ R14, CX JB copy_1 - ADDQ AX, R11 - ADDQ AX, R10 - ADDQ AX, R12 + +copy_1_end: + ADDQ CX, R11 + ADDQ CX, R10 + ADDQ CX, R12 // Malformed input if seq.mo > t+len(hist) || seq.mo > s.windowSize) -check_offset: - MOVQ R12, AX - ADDQ 40(SP), AX - CMPQ CX, AX + MOVQ R12, CX + ADDQ 72(SP), CX + CMPQ R13, CX JG error_match_off_too_big - CMPQ CX, 56(SP) + CMPQ R13, 88(SP) JG error_match_off_too_big // Copy match from history - MOVQ CX, AX - SUBQ R12, AX + MOVQ R13, CX + SUBQ R12, CX JLS copy_match - MOVQ 48(SP), R14 - SUBQ AX, R14 - CMPQ R13, AX + MOVQ 80(SP), R14 + SUBQ CX, R14 + CMPQ AX, CX JG copy_all_from_history - MOVQ R13, AX - SUBQ $0x10, AX + MOVQ AX, CX + SUBQ $0x10, CX JB copy_4_small copy_4_loop: @@ -2127,54 +2152,54 @@ copy_4_loop: MOVUPS X0, (R10) ADDQ $0x10, R14 ADDQ $0x10, R10 - SUBQ $0x10, AX + SUBQ $0x10, CX JAE copy_4_loop - LEAQ 16(R14)(AX*1), R14 - LEAQ 16(R10)(AX*1), R10 + LEAQ 16(R14)(CX*1), R14 + LEAQ 16(R10)(CX*1), R10 MOVUPS -16(R14), X0 MOVUPS X0, -16(R10) JMP copy_4_end copy_4_small: - CMPQ R13, $0x03 + CMPQ AX, $0x03 JE copy_4_move_3 - CMPQ R13, $0x08 + CMPQ AX, $0x08 JB copy_4_move_4through7 JMP copy_4_move_8through16 copy_4_move_3: - MOVWQZX (R14), AX - MOVB 2(R14), CL - MOVW AX, (R10) - MOVB CL, 2(R10) - ADDQ R13, R14 - ADDQ R13, R10 + MOVWQZX (R14), CX + MOVB 2(R14), R13 + MOVW CX, (R10) + MOVB R13, 2(R10) + ADDQ AX, R14 + ADDQ AX, R10 JMP copy_4_end copy_4_move_4through7: - MOVL (R14), AX - MOVL -4(R14)(R13*1), CX - MOVL AX, (R10) - MOVL CX, -4(R10)(R13*1) - ADDQ R13, R14 - ADDQ R13, R10 + MOVL (R14), CX + MOVL -4(R14)(AX*1), R13 + MOVL CX, (R10) + MOVL R13, -4(R10)(AX*1) + ADDQ AX, R14 + ADDQ AX, R10 JMP copy_4_end copy_4_move_8through16: - MOVQ (R14), AX - MOVQ -8(R14)(R13*1), CX - MOVQ AX, (R10) - MOVQ CX, -8(R10)(R13*1) - ADDQ R13, R14 - ADDQ R13, R10 + MOVQ (R14), CX + MOVQ -8(R14)(AX*1), R13 + MOVQ CX, (R10) + MOVQ R13, -8(R10)(AX*1) + ADDQ AX, R14 + ADDQ AX, R10 copy_4_end: - ADDQ R13, R12 + ADDQ AX, R12 JMP handle_loop JMP loop_finished copy_all_from_history: - MOVQ AX, R15 + MOVQ CX, R15 SUBQ $0x10, R15 JB copy_5_small @@ -2192,20 +2217,20 @@ copy_5_loop: JMP copy_5_end copy_5_small: - CMPQ AX, $0x03 + CMPQ CX, $0x03 JE copy_5_move_3 JB copy_5_move_1or2 - CMPQ AX, $0x08 + CMPQ CX, $0x08 JB copy_5_move_4through7 JMP copy_5_move_8through16 copy_5_move_1or2: MOVB (R14), R15 - MOVB -1(R14)(AX*1), BP + MOVB -1(R14)(CX*1), BP MOVB R15, (R10) - MOVB BP, -1(R10)(AX*1) - ADDQ AX, R14 - ADDQ AX, R10 + MOVB BP, -1(R10)(CX*1) + ADDQ CX, R14 + ADDQ CX, R10 JMP copy_5_end copy_5_move_3: @@ -2213,69 +2238,78 @@ copy_5_move_3: MOVB 2(R14), BP MOVW R15, (R10) MOVB BP, 2(R10) - ADDQ AX, R14 - ADDQ AX, R10 + ADDQ CX, R14 + ADDQ CX, R10 JMP copy_5_end copy_5_move_4through7: MOVL (R14), R15 - MOVL -4(R14)(AX*1), BP + MOVL -4(R14)(CX*1), BP MOVL R15, (R10) - MOVL BP, -4(R10)(AX*1) - ADDQ AX, R14 - ADDQ AX, R10 + MOVL BP, -4(R10)(CX*1) + ADDQ CX, R14 + ADDQ CX, R10 JMP copy_5_end copy_5_move_8through16: MOVQ (R14), R15 - MOVQ -8(R14)(AX*1), BP + MOVQ -8(R14)(CX*1), BP MOVQ R15, (R10) - MOVQ BP, -8(R10)(AX*1) - ADDQ AX, R14 - ADDQ AX, R10 + MOVQ BP, -8(R10)(CX*1) + ADDQ CX, R14 + ADDQ CX, R10 copy_5_end: - ADDQ AX, R12 - SUBQ AX, R13 + ADDQ CX, R12 + SUBQ CX, AX // Copy match from the current buffer copy_match: - MOVQ R10, AX - SUBQ CX, AX + MOVQ R10, CX + SUBQ R13, CX // ml <= mo - CMPQ R13, CX + CMPQ AX, R13 JA copy_overlapping_match // Copy non-overlapping match - ADDQ R13, R12 - MOVQ R10, CX - ADDQ R13, R10 + ADDQ AX, R12 + MOVUPS (CX), X0 + MOVUPS X0, (R10) + CMPQ AX, $0x10 + JA copy_2_long + ADDQ AX, R10 + JMP handle_loop + +copy_2_long: + LEAQ 16(R10), R13 + ADDQ AX, R10 + ADDQ $0x10, CX + SUBQ $0x10, AX copy_2: - MOVUPS (AX), X0 - MOVUPS X0, (CX) - ADDQ $0x10, AX + MOVUPS (CX), X0 + MOVUPS X0, (R13) ADDQ $0x10, CX - SUBQ $0x10, R13 + ADDQ $0x10, R13 + SUBQ $0x10, AX JHI copy_2 JMP handle_loop // Copy overlapping match copy_overlapping_match: - ADDQ R13, R12 + ADDQ AX, R12 copy_slow_3: - MOVBQZX (AX), CX - MOVB CL, (R10) - INCQ AX + MOVBQZX (CX), R13 + MOVB R13, (R10) + INCQ CX INCQ R10 - DECQ R13 + DECQ AX JNZ copy_slow_3 handle_loop: - MOVQ ctx+16(FP), AX - DECQ 96(AX) + DECQ 32(SP) JNS sequenceDecs_decodeSync_amd64_main_loop loop_finished: @@ -2284,6 +2318,13 @@ loop_finished: MOVB BL, 40(AX) MOVQ SI, 32(AX) + // s.seqSize += outPosition - ctx.outPosition + MOVQ ctx+16(FP), AX + MOVQ s+0(FP), CX + MOVQ R12, DX + SUBQ 136(AX), DX + ADDQ DX, 256(CX) + // Update the context MOVQ ctx+16(FP), AX MOVQ R12, 136(AX) @@ -2297,7 +2338,7 @@ loop_finished: // Return with match length error sequenceDecs_decodeSync_amd64_error_match_len_ofs_mismatch: - MOVQ 16(SP), AX + MOVQ 48(SP), AX MOVQ ctx+16(FP), CX MOVQ AX, 216(CX) MOVQ $0x00000001, ret+24(FP) @@ -2306,7 +2347,7 @@ sequenceDecs_decodeSync_amd64_error_match_len_ofs_mismatch: // Return with match too long error sequenceDecs_decodeSync_amd64_error_match_len_too_big: MOVQ ctx+16(FP), AX - MOVQ 16(SP), CX + MOVQ 48(SP), CX MOVQ CX, 216(AX) MOVQ $0x00000002, ret+24(FP) RET @@ -2314,7 +2355,7 @@ sequenceDecs_decodeSync_amd64_error_match_len_too_big: // Return with match offset too long error error_match_off_too_big: MOVQ ctx+16(FP), AX - MOVQ 8(SP), CX + MOVQ 40(SP), CX MOVQ CX, 224(AX) MOVQ R12, 136(AX) MOVQ $0x00000003, ret+24(FP) @@ -2323,7 +2364,7 @@ error_match_off_too_big: // Return with not enough literals error error_not_enough_literals: MOVQ ctx+16(FP), AX - MOVQ 24(SP), CX + MOVQ 56(SP), CX MOVQ CX, 208(AX) MOVQ $0x00000004, ret+24(FP) RET @@ -2336,9 +2377,9 @@ error_overread: // Return with not enough output space error error_not_enough_space: MOVQ ctx+16(FP), AX - MOVQ 24(SP), CX + MOVQ 56(SP), CX MOVQ CX, 208(AX) - MOVQ 16(SP), CX + MOVQ 48(SP), CX MOVQ CX, 216(AX) MOVQ R12, 136(AX) MOVQ $0x00000005, ret+24(FP) @@ -2346,7 +2387,7 @@ error_not_enough_space: // func sequenceDecs_decodeSync_bmi2(s *sequenceDecs, br *bitReader, ctx *decodeSyncAsmContext) int // Requires: BMI, BMI2, CMOV, SSE -TEXT ·sequenceDecs_decodeSync_bmi2(SB), $64-32 +TEXT ·sequenceDecs_decodeSync_bmi2(SB), $96-32 MOVQ br+8(FP), BX MOVQ 24(BX), AX MOVBQZX 40(BX), DX @@ -2358,26 +2399,34 @@ TEXT ·sequenceDecs_decodeSync_bmi2(SB), $64-32 MOVQ 72(CX), SI MOVQ 80(CX), DI MOVQ 88(CX), R8 - XORQ R9, R9 + MOVQ (CX), R9 MOVQ R9, 8(SP) + MOVQ 24(CX), R9 MOVQ R9, 16(SP) + MOVQ 48(CX), R9 MOVQ R9, 24(SP) + MOVQ 96(CX), R9 + MOVQ R9, 32(SP) + XORQ R9, R9 + MOVQ R9, 40(SP) + MOVQ R9, 48(SP) + MOVQ R9, 56(SP) MOVQ 112(CX), R9 MOVQ 128(CX), R10 - MOVQ R10, 32(SP) + MOVQ R10, 64(SP) MOVQ 144(CX), R10 MOVQ 136(CX), R11 MOVQ 200(CX), R12 - MOVQ R12, 56(SP) + MOVQ R12, 88(SP) MOVQ 176(CX), R12 - MOVQ R12, 48(SP) + MOVQ R12, 80(SP) MOVQ 184(CX), CX - MOVQ CX, 40(SP) - MOVQ 40(SP), CX - ADDQ CX, 48(SP) + MOVQ CX, 72(SP) + MOVQ 72(SP), CX + ADDQ CX, 80(SP) // Calculate pointer to s.out[cap(s.out)] (a past-end pointer) - ADDQ R9, 32(SP) + ADDQ R9, 64(SP) // outBase += outPosition ADDQ R11, R9 @@ -2425,7 +2474,7 @@ sequenceDecs_decodeSync_bmi2_fill_end: MOVQ R8, CX SHRQ $0x20, CX ADDQ R14, CX - MOVQ CX, 8(SP) + MOVQ CX, 40(SP) // Update match length MOVQ $0x00000808, CX @@ -2438,7 +2487,7 @@ sequenceDecs_decodeSync_bmi2_fill_end: MOVQ DI, CX SHRQ $0x20, CX ADDQ R14, CX - MOVQ CX, 16(SP) + MOVQ CX, 48(SP) // Fill bitreader to have enough for the remaining CMPQ BX, $0x08 @@ -2480,14 +2529,13 @@ sequenceDecs_decodeSync_bmi2_fill_2_end: MOVQ SI, CX SHRQ $0x20, CX ADDQ R14, CX - MOVQ CX, 24(SP) + MOVQ CX, 56(SP) // Fill bitreader for state updates MOVQ R12, (SP) MOVQ $0x00000808, CX BEXTRQ CX, R8, R12 - MOVQ ctx+16(FP), CX - CMPQ 96(CX), $0x00 + CMPQ 32(SP), $0x00 JZ sequenceDecs_decodeSync_bmi2_skip_update LEAQ (SI)(DI*1), R13 ADDQ R8, R13 @@ -2505,8 +2553,7 @@ sequenceDecs_decodeSync_bmi2_fill_2_end: ADDQ CX, R8 // Load ctx.ofTable - MOVQ ctx+16(FP), CX - MOVQ 48(CX), CX + MOVQ 24(SP), CX MOVQ (CX)(R8*8), R8 // Update Match Length State @@ -2516,8 +2563,7 @@ sequenceDecs_decodeSync_bmi2_fill_2_end: ADDQ CX, DI // Load ctx.mlTable - MOVQ ctx+16(FP), CX - MOVQ 24(CX), CX + MOVQ 16(SP), CX MOVQ (CX)(DI*8), DI // Update Literal Length State @@ -2526,14 +2572,13 @@ sequenceDecs_decodeSync_bmi2_fill_2_end: ADDQ CX, SI // Load ctx.llTable - MOVQ ctx+16(FP), CX - MOVQ (CX), CX + MOVQ 8(SP), CX MOVQ (CX)(SI*8), SI sequenceDecs_decodeSync_bmi2_skip_update: // Adjust offset MOVQ s+0(FP), CX - MOVQ 8(SP), R13 + MOVQ 40(SP), R13 CMPQ R12, $0x01 JBE sequenceDecs_decodeSync_bmi2_adjust_offsetB_1_or_0 MOVUPS 144(CX), X0 @@ -2542,7 +2587,7 @@ sequenceDecs_decodeSync_bmi2_skip_update: JMP sequenceDecs_decodeSync_bmi2_after_adjust sequenceDecs_decodeSync_bmi2_adjust_offsetB_1_or_0: - CMPQ 24(SP), $0x00000000 + CMPQ 56(SP), $0x00000000 JNE sequenceDecs_decodeSync_bmi2_adjust_offset_maybezero INCQ R13 JMP sequenceDecs_decodeSync_bmi2_adjust_offset_nonzero @@ -2577,14 +2622,11 @@ sequenceDecs_decodeSync_bmi2_adjust_skip: MOVQ R14, R13 sequenceDecs_decodeSync_bmi2_after_adjust: - MOVQ R13, 8(SP) + MOVQ R13, 40(SP) // Check values - MOVQ 16(SP), CX - MOVQ 24(SP), R12 - LEAQ (CX)(R12*1), R14 - MOVQ s+0(FP), R15 - ADDQ R14, 256(R15) + MOVQ 48(SP), CX + MOVQ 56(SP), R12 MOVQ ctx+16(FP), R14 SUBQ R12, 104(R14) JS error_not_enough_literals @@ -2596,50 +2638,49 @@ sequenceDecs_decodeSync_bmi2_after_adjust: JNZ sequenceDecs_decodeSync_bmi2_error_match_len_ofs_mismatch sequenceDecs_decodeSync_bmi2_match_len_ofs_ok: - MOVQ 24(SP), CX - MOVQ 8(SP), R12 - MOVQ 16(SP), R13 - // Check if we have enough space in s.out - LEAQ 16(CX)(R13*1), R14 + LEAQ 16(R12)(CX*1), R14 ADDQ R9, R14 - CMPQ R14, 32(SP) + CMPQ R14, 64(SP) JA error_not_enough_space // Copy literals - TESTQ CX, CX - JZ check_offset - XORQ R14, R14 + MOVUPS (R10), X0 + MOVUPS X0, (R9) + CMPQ R12, $0x10 + JBE copy_1_end + MOVQ $0x00000010, R14 copy_1: MOVUPS (R10)(R14*1), X0 MOVUPS X0, (R9)(R14*1) ADDQ $0x10, R14 - CMPQ R14, CX + CMPQ R14, R12 JB copy_1 - ADDQ CX, R10 - ADDQ CX, R9 - ADDQ CX, R11 + +copy_1_end: + ADDQ R12, R10 + ADDQ R12, R9 + ADDQ R12, R11 // Malformed input if seq.mo > t+len(hist) || seq.mo > s.windowSize) -check_offset: - MOVQ R11, CX - ADDQ 40(SP), CX - CMPQ R12, CX + MOVQ R11, R12 + ADDQ 72(SP), R12 + CMPQ R13, R12 JG error_match_off_too_big - CMPQ R12, 56(SP) + CMPQ R13, 88(SP) JG error_match_off_too_big // Copy match from history - MOVQ R12, CX - SUBQ R11, CX + MOVQ R13, R12 + SUBQ R11, R12 JLS copy_match - MOVQ 48(SP), R14 - SUBQ CX, R14 - CMPQ R13, CX + MOVQ 80(SP), R14 + SUBQ R12, R14 + CMPQ CX, R12 JG copy_all_from_history - MOVQ R13, CX - SUBQ $0x10, CX + MOVQ CX, R12 + SUBQ $0x10, R12 JB copy_4_small copy_4_loop: @@ -2647,54 +2688,54 @@ copy_4_loop: MOVUPS X0, (R9) ADDQ $0x10, R14 ADDQ $0x10, R9 - SUBQ $0x10, CX + SUBQ $0x10, R12 JAE copy_4_loop - LEAQ 16(R14)(CX*1), R14 - LEAQ 16(R9)(CX*1), R9 + LEAQ 16(R14)(R12*1), R14 + LEAQ 16(R9)(R12*1), R9 MOVUPS -16(R14), X0 MOVUPS X0, -16(R9) JMP copy_4_end copy_4_small: - CMPQ R13, $0x03 + CMPQ CX, $0x03 JE copy_4_move_3 - CMPQ R13, $0x08 + CMPQ CX, $0x08 JB copy_4_move_4through7 JMP copy_4_move_8through16 copy_4_move_3: - MOVWQZX (R14), CX - MOVB 2(R14), R12 - MOVW CX, (R9) - MOVB R12, 2(R9) - ADDQ R13, R14 - ADDQ R13, R9 + MOVWQZX (R14), R12 + MOVB 2(R14), R13 + MOVW R12, (R9) + MOVB R13, 2(R9) + ADDQ CX, R14 + ADDQ CX, R9 JMP copy_4_end copy_4_move_4through7: - MOVL (R14), CX - MOVL -4(R14)(R13*1), R12 - MOVL CX, (R9) - MOVL R12, -4(R9)(R13*1) - ADDQ R13, R14 - ADDQ R13, R9 + MOVL (R14), R12 + MOVL -4(R14)(CX*1), R13 + MOVL R12, (R9) + MOVL R13, -4(R9)(CX*1) + ADDQ CX, R14 + ADDQ CX, R9 JMP copy_4_end copy_4_move_8through16: - MOVQ (R14), CX - MOVQ -8(R14)(R13*1), R12 - MOVQ CX, (R9) - MOVQ R12, -8(R9)(R13*1) - ADDQ R13, R14 - ADDQ R13, R9 + MOVQ (R14), R12 + MOVQ -8(R14)(CX*1), R13 + MOVQ R12, (R9) + MOVQ R13, -8(R9)(CX*1) + ADDQ CX, R14 + ADDQ CX, R9 copy_4_end: - ADDQ R13, R11 + ADDQ CX, R11 JMP handle_loop JMP loop_finished copy_all_from_history: - MOVQ CX, R15 + MOVQ R12, R15 SUBQ $0x10, R15 JB copy_5_small @@ -2712,20 +2753,20 @@ copy_5_loop: JMP copy_5_end copy_5_small: - CMPQ CX, $0x03 + CMPQ R12, $0x03 JE copy_5_move_3 JB copy_5_move_1or2 - CMPQ CX, $0x08 + CMPQ R12, $0x08 JB copy_5_move_4through7 JMP copy_5_move_8through16 copy_5_move_1or2: MOVB (R14), R15 - MOVB -1(R14)(CX*1), BP + MOVB -1(R14)(R12*1), BP MOVB R15, (R9) - MOVB BP, -1(R9)(CX*1) - ADDQ CX, R14 - ADDQ CX, R9 + MOVB BP, -1(R9)(R12*1) + ADDQ R12, R14 + ADDQ R12, R9 JMP copy_5_end copy_5_move_3: @@ -2733,69 +2774,78 @@ copy_5_move_3: MOVB 2(R14), BP MOVW R15, (R9) MOVB BP, 2(R9) - ADDQ CX, R14 - ADDQ CX, R9 + ADDQ R12, R14 + ADDQ R12, R9 JMP copy_5_end copy_5_move_4through7: MOVL (R14), R15 - MOVL -4(R14)(CX*1), BP + MOVL -4(R14)(R12*1), BP MOVL R15, (R9) - MOVL BP, -4(R9)(CX*1) - ADDQ CX, R14 - ADDQ CX, R9 + MOVL BP, -4(R9)(R12*1) + ADDQ R12, R14 + ADDQ R12, R9 JMP copy_5_end copy_5_move_8through16: MOVQ (R14), R15 - MOVQ -8(R14)(CX*1), BP + MOVQ -8(R14)(R12*1), BP MOVQ R15, (R9) - MOVQ BP, -8(R9)(CX*1) - ADDQ CX, R14 - ADDQ CX, R9 + MOVQ BP, -8(R9)(R12*1) + ADDQ R12, R14 + ADDQ R12, R9 copy_5_end: - ADDQ CX, R11 - SUBQ CX, R13 + ADDQ R12, R11 + SUBQ R12, CX // Copy match from the current buffer copy_match: - MOVQ R9, CX - SUBQ R12, CX + MOVQ R9, R12 + SUBQ R13, R12 // ml <= mo - CMPQ R13, R12 + CMPQ CX, R13 JA copy_overlapping_match // Copy non-overlapping match - ADDQ R13, R11 - MOVQ R9, R12 - ADDQ R13, R9 + ADDQ CX, R11 + MOVUPS (R12), X0 + MOVUPS X0, (R9) + CMPQ CX, $0x10 + JA copy_2_long + ADDQ CX, R9 + JMP handle_loop + +copy_2_long: + LEAQ 16(R9), R13 + ADDQ CX, R9 + ADDQ $0x10, R12 + SUBQ $0x10, CX copy_2: - MOVUPS (CX), X0 - MOVUPS X0, (R12) - ADDQ $0x10, CX + MOVUPS (R12), X0 + MOVUPS X0, (R13) ADDQ $0x10, R12 - SUBQ $0x10, R13 + ADDQ $0x10, R13 + SUBQ $0x10, CX JHI copy_2 JMP handle_loop // Copy overlapping match copy_overlapping_match: - ADDQ R13, R11 + ADDQ CX, R11 copy_slow_3: - MOVBQZX (CX), R12 - MOVB R12, (R9) - INCQ CX + MOVBQZX (R12), R13 + MOVB R13, (R9) + INCQ R12 INCQ R9 - DECQ R13 + DECQ CX JNZ copy_slow_3 handle_loop: - MOVQ ctx+16(FP), CX - DECQ 96(CX) + DECQ 32(SP) JNS sequenceDecs_decodeSync_bmi2_main_loop loop_finished: @@ -2804,6 +2854,13 @@ loop_finished: MOVB DL, 40(CX) MOVQ BX, 32(CX) + // s.seqSize += outPosition - ctx.outPosition + MOVQ ctx+16(FP), AX + MOVQ s+0(FP), CX + MOVQ R11, DX + SUBQ 136(AX), DX + ADDQ DX, 256(CX) + // Update the context MOVQ ctx+16(FP), AX MOVQ R11, 136(AX) @@ -2817,7 +2874,7 @@ loop_finished: // Return with match length error sequenceDecs_decodeSync_bmi2_error_match_len_ofs_mismatch: - MOVQ 16(SP), AX + MOVQ 48(SP), AX MOVQ ctx+16(FP), CX MOVQ AX, 216(CX) MOVQ $0x00000001, ret+24(FP) @@ -2826,7 +2883,7 @@ sequenceDecs_decodeSync_bmi2_error_match_len_ofs_mismatch: // Return with match too long error sequenceDecs_decodeSync_bmi2_error_match_len_too_big: MOVQ ctx+16(FP), AX - MOVQ 16(SP), CX + MOVQ 48(SP), CX MOVQ CX, 216(AX) MOVQ $0x00000002, ret+24(FP) RET @@ -2834,7 +2891,7 @@ sequenceDecs_decodeSync_bmi2_error_match_len_too_big: // Return with match offset too long error error_match_off_too_big: MOVQ ctx+16(FP), AX - MOVQ 8(SP), CX + MOVQ 40(SP), CX MOVQ CX, 224(AX) MOVQ R11, 136(AX) MOVQ $0x00000003, ret+24(FP) @@ -2843,7 +2900,7 @@ error_match_off_too_big: // Return with not enough literals error error_not_enough_literals: MOVQ ctx+16(FP), AX - MOVQ 24(SP), CX + MOVQ 56(SP), CX MOVQ CX, 208(AX) MOVQ $0x00000004, ret+24(FP) RET @@ -2856,9 +2913,9 @@ error_overread: // Return with not enough output space error error_not_enough_space: MOVQ ctx+16(FP), AX - MOVQ 24(SP), CX + MOVQ 56(SP), CX MOVQ CX, 208(AX) - MOVQ 16(SP), CX + MOVQ 48(SP), CX MOVQ CX, 216(AX) MOVQ R11, 136(AX) MOVQ $0x00000005, ret+24(FP) @@ -2866,7 +2923,7 @@ error_not_enough_space: // func sequenceDecs_decodeSync_safe_amd64(s *sequenceDecs, br *bitReader, ctx *decodeSyncAsmContext) int // Requires: CMOV, SSE -TEXT ·sequenceDecs_decodeSync_safe_amd64(SB), $64-32 +TEXT ·sequenceDecs_decodeSync_safe_amd64(SB), $96-32 MOVQ br+8(FP), CX MOVQ 24(CX), DX MOVBQZX 40(CX), BX @@ -2878,26 +2935,34 @@ TEXT ·sequenceDecs_decodeSync_safe_amd64(SB), $64-32 MOVQ 72(AX), DI MOVQ 80(AX), R8 MOVQ 88(AX), R9 - XORQ CX, CX + MOVQ (AX), CX MOVQ CX, 8(SP) + MOVQ 24(AX), CX MOVQ CX, 16(SP) + MOVQ 48(AX), CX MOVQ CX, 24(SP) + MOVQ 96(AX), CX + MOVQ CX, 32(SP) + XORQ CX, CX + MOVQ CX, 40(SP) + MOVQ CX, 48(SP) + MOVQ CX, 56(SP) MOVQ 112(AX), R10 MOVQ 128(AX), CX - MOVQ CX, 32(SP) + MOVQ CX, 64(SP) MOVQ 144(AX), R11 MOVQ 136(AX), R12 MOVQ 200(AX), CX - MOVQ CX, 56(SP) + MOVQ CX, 88(SP) MOVQ 176(AX), CX - MOVQ CX, 48(SP) + MOVQ CX, 80(SP) MOVQ 184(AX), AX - MOVQ AX, 40(SP) - MOVQ 40(SP), AX - ADDQ AX, 48(SP) + MOVQ AX, 72(SP) + MOVQ 72(SP), AX + ADDQ AX, 80(SP) // Calculate pointer to s.out[cap(s.out)] (a past-end pointer) - ADDQ R10, 32(SP) + ADDQ R10, 64(SP) // outBase += outPosition ADDQ R12, R10 @@ -2939,42 +3004,28 @@ sequenceDecs_decodeSync_safe_amd64_fill_end: MOVQ BX, CX MOVQ DX, R14 SHLQ CL, R14 + SHRQ $0x01, R14 MOVBLZX AH, CX SHRQ $0x20, AX - TESTQ CX, CX - JZ sequenceDecs_decodeSync_safe_amd64_of_update_zero ADDQ CX, BX - CMPQ BX, $0x40 - JA sequenceDecs_decodeSync_safe_amd64_of_update_zero - CMPQ CX, $0x40 - JAE sequenceDecs_decodeSync_safe_amd64_of_update_zero - NEGQ CX + XORQ $0x3f, CX SHRQ CL, R14 ADDQ R14, AX - -sequenceDecs_decodeSync_safe_amd64_of_update_zero: - MOVQ AX, 8(SP) + MOVQ AX, 40(SP) // Update match length MOVQ R8, AX MOVQ BX, CX MOVQ DX, R14 SHLQ CL, R14 + SHRQ $0x01, R14 MOVBLZX AH, CX SHRQ $0x20, AX - TESTQ CX, CX - JZ sequenceDecs_decodeSync_safe_amd64_ml_update_zero ADDQ CX, BX - CMPQ BX, $0x40 - JA sequenceDecs_decodeSync_safe_amd64_ml_update_zero - CMPQ CX, $0x40 - JAE sequenceDecs_decodeSync_safe_amd64_ml_update_zero - NEGQ CX + XORQ $0x3f, CX SHRQ CL, R14 ADDQ R14, AX - -sequenceDecs_decodeSync_safe_amd64_ml_update_zero: - MOVQ AX, 16(SP) + MOVQ AX, 48(SP) // Fill bitreader to have enough for the remaining CMPQ SI, $0x08 @@ -3010,92 +3061,78 @@ sequenceDecs_decodeSync_safe_amd64_fill_2_end: MOVQ BX, CX MOVQ DX, R14 SHLQ CL, R14 + SHRQ $0x01, R14 MOVBLZX AH, CX SHRQ $0x20, AX - TESTQ CX, CX - JZ sequenceDecs_decodeSync_safe_amd64_ll_update_zero ADDQ CX, BX - CMPQ BX, $0x40 - JA sequenceDecs_decodeSync_safe_amd64_ll_update_zero - CMPQ CX, $0x40 - JAE sequenceDecs_decodeSync_safe_amd64_ll_update_zero - NEGQ CX + XORQ $0x3f, CX SHRQ CL, R14 ADDQ R14, AX - -sequenceDecs_decodeSync_safe_amd64_ll_update_zero: - MOVQ AX, 24(SP) + MOVQ AX, 56(SP) // Fill bitreader for state updates MOVQ R13, (SP) MOVQ R9, AX SHRQ $0x08, AX MOVBQZX AL, AX - MOVQ ctx+16(FP), CX - CMPQ 96(CX), $0x00 + CMPQ 32(SP), $0x00 JZ sequenceDecs_decodeSync_safe_amd64_skip_update // Update Literal Length State MOVBQZX DI, R13 SHRL $0x10, DI - LEAQ (BX)(R13*1), CX + MOVQ BX, CX MOVQ DX, R14 - MOVQ CX, BX - ROLQ CL, R14 - MOVL $0x00000001, R15 - MOVBLZX R13, CX - SHLL CL, R15 - DECL R15 - ANDQ R15, R14 + SHLQ CL, R14 + SHRQ $0x01, R14 + ADDQ R13, BX + XORQ $0x3f, R13 + MOVQ R13, CX + SHRQ CL, R14 ADDQ R14, DI // Load ctx.llTable - MOVQ ctx+16(FP), CX - MOVQ (CX), CX + MOVQ 8(SP), CX MOVQ (CX)(DI*8), DI // Update Match Length State MOVBQZX R8, R13 SHRL $0x10, R8 - LEAQ (BX)(R13*1), CX + MOVQ BX, CX MOVQ DX, R14 - MOVQ CX, BX - ROLQ CL, R14 - MOVL $0x00000001, R15 - MOVBLZX R13, CX - SHLL CL, R15 - DECL R15 - ANDQ R15, R14 + SHLQ CL, R14 + SHRQ $0x01, R14 + ADDQ R13, BX + XORQ $0x3f, R13 + MOVQ R13, CX + SHRQ CL, R14 ADDQ R14, R8 // Load ctx.mlTable - MOVQ ctx+16(FP), CX - MOVQ 24(CX), CX + MOVQ 16(SP), CX MOVQ (CX)(R8*8), R8 // Update Offset State MOVBQZX R9, R13 SHRL $0x10, R9 - LEAQ (BX)(R13*1), CX + MOVQ BX, CX MOVQ DX, R14 - MOVQ CX, BX - ROLQ CL, R14 - MOVL $0x00000001, R15 - MOVBLZX R13, CX - SHLL CL, R15 - DECL R15 - ANDQ R15, R14 + SHLQ CL, R14 + SHRQ $0x01, R14 + ADDQ R13, BX + XORQ $0x3f, R13 + MOVQ R13, CX + SHRQ CL, R14 ADDQ R14, R9 // Load ctx.ofTable - MOVQ ctx+16(FP), CX - MOVQ 48(CX), CX + MOVQ 24(SP), CX MOVQ (CX)(R9*8), R9 sequenceDecs_decodeSync_safe_amd64_skip_update: // Adjust offset MOVQ s+0(FP), CX - MOVQ 8(SP), R13 + MOVQ 40(SP), R13 CMPQ AX, $0x01 JBE sequenceDecs_decodeSync_safe_amd64_adjust_offsetB_1_or_0 MOVUPS 144(CX), X0 @@ -3104,7 +3141,7 @@ sequenceDecs_decodeSync_safe_amd64_skip_update: JMP sequenceDecs_decodeSync_safe_amd64_after_adjust sequenceDecs_decodeSync_safe_amd64_adjust_offsetB_1_or_0: - CMPQ 24(SP), $0x00000000 + CMPQ 56(SP), $0x00000000 JNE sequenceDecs_decodeSync_safe_amd64_adjust_offset_maybezero INCQ R13 JMP sequenceDecs_decodeSync_safe_amd64_adjust_offset_nonzero @@ -3139,14 +3176,11 @@ sequenceDecs_decodeSync_safe_amd64_adjust_skip: MOVQ R14, R13 sequenceDecs_decodeSync_safe_amd64_after_adjust: - MOVQ R13, 8(SP) + MOVQ R13, 40(SP) // Check values - MOVQ 16(SP), AX - MOVQ 24(SP), CX - LEAQ (AX)(CX*1), R14 - MOVQ s+0(FP), R15 - ADDQ R14, 256(R15) + MOVQ 48(SP), AX + MOVQ 56(SP), CX MOVQ ctx+16(FP), R14 SUBQ CX, 104(R14) JS error_not_enough_literals @@ -3158,20 +3192,16 @@ sequenceDecs_decodeSync_safe_amd64_after_adjust: JNZ sequenceDecs_decodeSync_safe_amd64_error_match_len_ofs_mismatch sequenceDecs_decodeSync_safe_amd64_match_len_ofs_ok: - MOVQ 24(SP), AX - MOVQ 8(SP), CX - MOVQ 16(SP), R13 - // Check if we have enough space in s.out - LEAQ (AX)(R13*1), R14 + LEAQ (CX)(AX*1), R14 ADDQ R10, R14 - CMPQ R14, 32(SP) + CMPQ R14, 64(SP) JA error_not_enough_space // Copy literals - TESTQ AX, AX + TESTQ CX, CX JZ check_offset - MOVQ AX, R14 + MOVQ CX, R14 SUBQ $0x10, R14 JB copy_1_small @@ -3189,20 +3219,20 @@ copy_1_loop: JMP copy_1_end copy_1_small: - CMPQ AX, $0x03 + CMPQ CX, $0x03 JE copy_1_move_3 JB copy_1_move_1or2 - CMPQ AX, $0x08 + CMPQ CX, $0x08 JB copy_1_move_4through7 JMP copy_1_move_8through16 copy_1_move_1or2: MOVB (R11), R14 - MOVB -1(R11)(AX*1), R15 + MOVB -1(R11)(CX*1), R15 MOVB R14, (R10) - MOVB R15, -1(R10)(AX*1) - ADDQ AX, R11 - ADDQ AX, R10 + MOVB R15, -1(R10)(CX*1) + ADDQ CX, R11 + ADDQ CX, R10 JMP copy_1_end copy_1_move_3: @@ -3210,49 +3240,49 @@ copy_1_move_3: MOVB 2(R11), R15 MOVW R14, (R10) MOVB R15, 2(R10) - ADDQ AX, R11 - ADDQ AX, R10 + ADDQ CX, R11 + ADDQ CX, R10 JMP copy_1_end copy_1_move_4through7: MOVL (R11), R14 - MOVL -4(R11)(AX*1), R15 + MOVL -4(R11)(CX*1), R15 MOVL R14, (R10) - MOVL R15, -4(R10)(AX*1) - ADDQ AX, R11 - ADDQ AX, R10 + MOVL R15, -4(R10)(CX*1) + ADDQ CX, R11 + ADDQ CX, R10 JMP copy_1_end copy_1_move_8through16: MOVQ (R11), R14 - MOVQ -8(R11)(AX*1), R15 + MOVQ -8(R11)(CX*1), R15 MOVQ R14, (R10) - MOVQ R15, -8(R10)(AX*1) - ADDQ AX, R11 - ADDQ AX, R10 + MOVQ R15, -8(R10)(CX*1) + ADDQ CX, R11 + ADDQ CX, R10 copy_1_end: - ADDQ AX, R12 + ADDQ CX, R12 // Malformed input if seq.mo > t+len(hist) || seq.mo > s.windowSize) check_offset: - MOVQ R12, AX - ADDQ 40(SP), AX - CMPQ CX, AX + MOVQ R12, CX + ADDQ 72(SP), CX + CMPQ R13, CX JG error_match_off_too_big - CMPQ CX, 56(SP) + CMPQ R13, 88(SP) JG error_match_off_too_big // Copy match from history - MOVQ CX, AX - SUBQ R12, AX + MOVQ R13, CX + SUBQ R12, CX JLS copy_match - MOVQ 48(SP), R14 - SUBQ AX, R14 - CMPQ R13, AX + MOVQ 80(SP), R14 + SUBQ CX, R14 + CMPQ AX, CX JG copy_all_from_history - MOVQ R13, AX - SUBQ $0x10, AX + MOVQ AX, CX + SUBQ $0x10, CX JB copy_4_small copy_4_loop: @@ -3260,54 +3290,54 @@ copy_4_loop: MOVUPS X0, (R10) ADDQ $0x10, R14 ADDQ $0x10, R10 - SUBQ $0x10, AX + SUBQ $0x10, CX JAE copy_4_loop - LEAQ 16(R14)(AX*1), R14 - LEAQ 16(R10)(AX*1), R10 + LEAQ 16(R14)(CX*1), R14 + LEAQ 16(R10)(CX*1), R10 MOVUPS -16(R14), X0 MOVUPS X0, -16(R10) JMP copy_4_end copy_4_small: - CMPQ R13, $0x03 + CMPQ AX, $0x03 JE copy_4_move_3 - CMPQ R13, $0x08 + CMPQ AX, $0x08 JB copy_4_move_4through7 JMP copy_4_move_8through16 copy_4_move_3: - MOVWQZX (R14), AX - MOVB 2(R14), CL - MOVW AX, (R10) - MOVB CL, 2(R10) - ADDQ R13, R14 - ADDQ R13, R10 + MOVWQZX (R14), CX + MOVB 2(R14), R13 + MOVW CX, (R10) + MOVB R13, 2(R10) + ADDQ AX, R14 + ADDQ AX, R10 JMP copy_4_end copy_4_move_4through7: - MOVL (R14), AX - MOVL -4(R14)(R13*1), CX - MOVL AX, (R10) - MOVL CX, -4(R10)(R13*1) - ADDQ R13, R14 - ADDQ R13, R10 + MOVL (R14), CX + MOVL -4(R14)(AX*1), R13 + MOVL CX, (R10) + MOVL R13, -4(R10)(AX*1) + ADDQ AX, R14 + ADDQ AX, R10 JMP copy_4_end copy_4_move_8through16: - MOVQ (R14), AX - MOVQ -8(R14)(R13*1), CX - MOVQ AX, (R10) - MOVQ CX, -8(R10)(R13*1) - ADDQ R13, R14 - ADDQ R13, R10 + MOVQ (R14), CX + MOVQ -8(R14)(AX*1), R13 + MOVQ CX, (R10) + MOVQ R13, -8(R10)(AX*1) + ADDQ AX, R14 + ADDQ AX, R10 copy_4_end: - ADDQ R13, R12 + ADDQ AX, R12 JMP handle_loop JMP loop_finished copy_all_from_history: - MOVQ AX, R15 + MOVQ CX, R15 SUBQ $0x10, R15 JB copy_5_small @@ -3325,20 +3355,20 @@ copy_5_loop: JMP copy_5_end copy_5_small: - CMPQ AX, $0x03 + CMPQ CX, $0x03 JE copy_5_move_3 JB copy_5_move_1or2 - CMPQ AX, $0x08 + CMPQ CX, $0x08 JB copy_5_move_4through7 JMP copy_5_move_8through16 copy_5_move_1or2: MOVB (R14), R15 - MOVB -1(R14)(AX*1), BP + MOVB -1(R14)(CX*1), BP MOVB R15, (R10) - MOVB BP, -1(R10)(AX*1) - ADDQ AX, R14 - ADDQ AX, R10 + MOVB BP, -1(R10)(CX*1) + ADDQ CX, R14 + ADDQ CX, R10 JMP copy_5_end copy_5_move_3: @@ -3346,120 +3376,119 @@ copy_5_move_3: MOVB 2(R14), BP MOVW R15, (R10) MOVB BP, 2(R10) - ADDQ AX, R14 - ADDQ AX, R10 + ADDQ CX, R14 + ADDQ CX, R10 JMP copy_5_end copy_5_move_4through7: MOVL (R14), R15 - MOVL -4(R14)(AX*1), BP + MOVL -4(R14)(CX*1), BP MOVL R15, (R10) - MOVL BP, -4(R10)(AX*1) - ADDQ AX, R14 - ADDQ AX, R10 + MOVL BP, -4(R10)(CX*1) + ADDQ CX, R14 + ADDQ CX, R10 JMP copy_5_end copy_5_move_8through16: MOVQ (R14), R15 - MOVQ -8(R14)(AX*1), BP + MOVQ -8(R14)(CX*1), BP MOVQ R15, (R10) - MOVQ BP, -8(R10)(AX*1) - ADDQ AX, R14 - ADDQ AX, R10 + MOVQ BP, -8(R10)(CX*1) + ADDQ CX, R14 + ADDQ CX, R10 copy_5_end: - ADDQ AX, R12 - SUBQ AX, R13 + ADDQ CX, R12 + SUBQ CX, AX // Copy match from the current buffer copy_match: - MOVQ R10, AX - SUBQ CX, AX + MOVQ R10, CX + SUBQ R13, CX // ml <= mo - CMPQ R13, CX + CMPQ AX, R13 JA copy_overlapping_match // Copy non-overlapping match - ADDQ R13, R12 - MOVQ R13, CX - SUBQ $0x10, CX + ADDQ AX, R12 + MOVQ AX, R13 + SUBQ $0x10, R13 JB copy_2_small copy_2_loop: - MOVUPS (AX), X0 + MOVUPS (CX), X0 MOVUPS X0, (R10) - ADDQ $0x10, AX + ADDQ $0x10, CX ADDQ $0x10, R10 - SUBQ $0x10, CX + SUBQ $0x10, R13 JAE copy_2_loop - LEAQ 16(AX)(CX*1), AX - LEAQ 16(R10)(CX*1), R10 - MOVUPS -16(AX), X0 + LEAQ 16(CX)(R13*1), CX + LEAQ 16(R10)(R13*1), R10 + MOVUPS -16(CX), X0 MOVUPS X0, -16(R10) JMP copy_2_end copy_2_small: - CMPQ R13, $0x03 + CMPQ AX, $0x03 JE copy_2_move_3 JB copy_2_move_1or2 - CMPQ R13, $0x08 + CMPQ AX, $0x08 JB copy_2_move_4through7 JMP copy_2_move_8through16 copy_2_move_1or2: - MOVB (AX), CL - MOVB -1(AX)(R13*1), R14 - MOVB CL, (R10) - MOVB R14, -1(R10)(R13*1) - ADDQ R13, AX - ADDQ R13, R10 + MOVB (CX), R13 + MOVB -1(CX)(AX*1), R14 + MOVB R13, (R10) + MOVB R14, -1(R10)(AX*1) + ADDQ AX, CX + ADDQ AX, R10 JMP copy_2_end copy_2_move_3: - MOVWQZX (AX), CX - MOVB 2(AX), R14 - MOVW CX, (R10) + MOVWQZX (CX), R13 + MOVB 2(CX), R14 + MOVW R13, (R10) MOVB R14, 2(R10) - ADDQ R13, AX - ADDQ R13, R10 + ADDQ AX, CX + ADDQ AX, R10 JMP copy_2_end copy_2_move_4through7: - MOVL (AX), CX - MOVL -4(AX)(R13*1), R14 - MOVL CX, (R10) - MOVL R14, -4(R10)(R13*1) - ADDQ R13, AX - ADDQ R13, R10 + MOVL (CX), R13 + MOVL -4(CX)(AX*1), R14 + MOVL R13, (R10) + MOVL R14, -4(R10)(AX*1) + ADDQ AX, CX + ADDQ AX, R10 JMP copy_2_end copy_2_move_8through16: - MOVQ (AX), CX - MOVQ -8(AX)(R13*1), R14 - MOVQ CX, (R10) - MOVQ R14, -8(R10)(R13*1) - ADDQ R13, AX - ADDQ R13, R10 + MOVQ (CX), R13 + MOVQ -8(CX)(AX*1), R14 + MOVQ R13, (R10) + MOVQ R14, -8(R10)(AX*1) + ADDQ AX, CX + ADDQ AX, R10 copy_2_end: JMP handle_loop // Copy overlapping match copy_overlapping_match: - ADDQ R13, R12 + ADDQ AX, R12 copy_slow_3: - MOVBQZX (AX), CX - MOVB CL, (R10) - INCQ AX + MOVBQZX (CX), R13 + MOVB R13, (R10) + INCQ CX INCQ R10 - DECQ R13 + DECQ AX JNZ copy_slow_3 handle_loop: - MOVQ ctx+16(FP), AX - DECQ 96(AX) + DECQ 32(SP) JNS sequenceDecs_decodeSync_safe_amd64_main_loop loop_finished: @@ -3468,6 +3497,13 @@ loop_finished: MOVB BL, 40(AX) MOVQ SI, 32(AX) + // s.seqSize += outPosition - ctx.outPosition + MOVQ ctx+16(FP), AX + MOVQ s+0(FP), CX + MOVQ R12, DX + SUBQ 136(AX), DX + ADDQ DX, 256(CX) + // Update the context MOVQ ctx+16(FP), AX MOVQ R12, 136(AX) @@ -3481,7 +3517,7 @@ loop_finished: // Return with match length error sequenceDecs_decodeSync_safe_amd64_error_match_len_ofs_mismatch: - MOVQ 16(SP), AX + MOVQ 48(SP), AX MOVQ ctx+16(FP), CX MOVQ AX, 216(CX) MOVQ $0x00000001, ret+24(FP) @@ -3490,7 +3526,7 @@ sequenceDecs_decodeSync_safe_amd64_error_match_len_ofs_mismatch: // Return with match too long error sequenceDecs_decodeSync_safe_amd64_error_match_len_too_big: MOVQ ctx+16(FP), AX - MOVQ 16(SP), CX + MOVQ 48(SP), CX MOVQ CX, 216(AX) MOVQ $0x00000002, ret+24(FP) RET @@ -3498,7 +3534,7 @@ sequenceDecs_decodeSync_safe_amd64_error_match_len_too_big: // Return with match offset too long error error_match_off_too_big: MOVQ ctx+16(FP), AX - MOVQ 8(SP), CX + MOVQ 40(SP), CX MOVQ CX, 224(AX) MOVQ R12, 136(AX) MOVQ $0x00000003, ret+24(FP) @@ -3507,7 +3543,7 @@ error_match_off_too_big: // Return with not enough literals error error_not_enough_literals: MOVQ ctx+16(FP), AX - MOVQ 24(SP), CX + MOVQ 56(SP), CX MOVQ CX, 208(AX) MOVQ $0x00000004, ret+24(FP) RET @@ -3520,9 +3556,9 @@ error_overread: // Return with not enough output space error error_not_enough_space: MOVQ ctx+16(FP), AX - MOVQ 24(SP), CX + MOVQ 56(SP), CX MOVQ CX, 208(AX) - MOVQ 16(SP), CX + MOVQ 48(SP), CX MOVQ CX, 216(AX) MOVQ R12, 136(AX) MOVQ $0x00000005, ret+24(FP) @@ -3530,7 +3566,7 @@ error_not_enough_space: // func sequenceDecs_decodeSync_safe_bmi2(s *sequenceDecs, br *bitReader, ctx *decodeSyncAsmContext) int // Requires: BMI, BMI2, CMOV, SSE -TEXT ·sequenceDecs_decodeSync_safe_bmi2(SB), $64-32 +TEXT ·sequenceDecs_decodeSync_safe_bmi2(SB), $96-32 MOVQ br+8(FP), BX MOVQ 24(BX), AX MOVBQZX 40(BX), DX @@ -3542,26 +3578,34 @@ TEXT ·sequenceDecs_decodeSync_safe_bmi2(SB), $64-32 MOVQ 72(CX), SI MOVQ 80(CX), DI MOVQ 88(CX), R8 - XORQ R9, R9 + MOVQ (CX), R9 MOVQ R9, 8(SP) + MOVQ 24(CX), R9 MOVQ R9, 16(SP) + MOVQ 48(CX), R9 MOVQ R9, 24(SP) + MOVQ 96(CX), R9 + MOVQ R9, 32(SP) + XORQ R9, R9 + MOVQ R9, 40(SP) + MOVQ R9, 48(SP) + MOVQ R9, 56(SP) MOVQ 112(CX), R9 MOVQ 128(CX), R10 - MOVQ R10, 32(SP) + MOVQ R10, 64(SP) MOVQ 144(CX), R10 MOVQ 136(CX), R11 MOVQ 200(CX), R12 - MOVQ R12, 56(SP) + MOVQ R12, 88(SP) MOVQ 176(CX), R12 - MOVQ R12, 48(SP) + MOVQ R12, 80(SP) MOVQ 184(CX), CX - MOVQ CX, 40(SP) - MOVQ 40(SP), CX - ADDQ CX, 48(SP) + MOVQ CX, 72(SP) + MOVQ 72(SP), CX + ADDQ CX, 80(SP) // Calculate pointer to s.out[cap(s.out)] (a past-end pointer) - ADDQ R9, 32(SP) + ADDQ R9, 64(SP) // outBase += outPosition ADDQ R11, R9 @@ -3609,7 +3653,7 @@ sequenceDecs_decodeSync_safe_bmi2_fill_end: MOVQ R8, CX SHRQ $0x20, CX ADDQ R14, CX - MOVQ CX, 8(SP) + MOVQ CX, 40(SP) // Update match length MOVQ $0x00000808, CX @@ -3622,7 +3666,7 @@ sequenceDecs_decodeSync_safe_bmi2_fill_end: MOVQ DI, CX SHRQ $0x20, CX ADDQ R14, CX - MOVQ CX, 16(SP) + MOVQ CX, 48(SP) // Fill bitreader to have enough for the remaining CMPQ BX, $0x08 @@ -3664,14 +3708,13 @@ sequenceDecs_decodeSync_safe_bmi2_fill_2_end: MOVQ SI, CX SHRQ $0x20, CX ADDQ R14, CX - MOVQ CX, 24(SP) + MOVQ CX, 56(SP) // Fill bitreader for state updates MOVQ R12, (SP) MOVQ $0x00000808, CX BEXTRQ CX, R8, R12 - MOVQ ctx+16(FP), CX - CMPQ 96(CX), $0x00 + CMPQ 32(SP), $0x00 JZ sequenceDecs_decodeSync_safe_bmi2_skip_update LEAQ (SI)(DI*1), R13 ADDQ R8, R13 @@ -3689,8 +3732,7 @@ sequenceDecs_decodeSync_safe_bmi2_fill_2_end: ADDQ CX, R8 // Load ctx.ofTable - MOVQ ctx+16(FP), CX - MOVQ 48(CX), CX + MOVQ 24(SP), CX MOVQ (CX)(R8*8), R8 // Update Match Length State @@ -3700,8 +3742,7 @@ sequenceDecs_decodeSync_safe_bmi2_fill_2_end: ADDQ CX, DI // Load ctx.mlTable - MOVQ ctx+16(FP), CX - MOVQ 24(CX), CX + MOVQ 16(SP), CX MOVQ (CX)(DI*8), DI // Update Literal Length State @@ -3710,14 +3751,13 @@ sequenceDecs_decodeSync_safe_bmi2_fill_2_end: ADDQ CX, SI // Load ctx.llTable - MOVQ ctx+16(FP), CX - MOVQ (CX), CX + MOVQ 8(SP), CX MOVQ (CX)(SI*8), SI sequenceDecs_decodeSync_safe_bmi2_skip_update: // Adjust offset MOVQ s+0(FP), CX - MOVQ 8(SP), R13 + MOVQ 40(SP), R13 CMPQ R12, $0x01 JBE sequenceDecs_decodeSync_safe_bmi2_adjust_offsetB_1_or_0 MOVUPS 144(CX), X0 @@ -3726,7 +3766,7 @@ sequenceDecs_decodeSync_safe_bmi2_skip_update: JMP sequenceDecs_decodeSync_safe_bmi2_after_adjust sequenceDecs_decodeSync_safe_bmi2_adjust_offsetB_1_or_0: - CMPQ 24(SP), $0x00000000 + CMPQ 56(SP), $0x00000000 JNE sequenceDecs_decodeSync_safe_bmi2_adjust_offset_maybezero INCQ R13 JMP sequenceDecs_decodeSync_safe_bmi2_adjust_offset_nonzero @@ -3761,14 +3801,11 @@ sequenceDecs_decodeSync_safe_bmi2_adjust_skip: MOVQ R14, R13 sequenceDecs_decodeSync_safe_bmi2_after_adjust: - MOVQ R13, 8(SP) + MOVQ R13, 40(SP) // Check values - MOVQ 16(SP), CX - MOVQ 24(SP), R12 - LEAQ (CX)(R12*1), R14 - MOVQ s+0(FP), R15 - ADDQ R14, 256(R15) + MOVQ 48(SP), CX + MOVQ 56(SP), R12 MOVQ ctx+16(FP), R14 SUBQ R12, 104(R14) JS error_not_enough_literals @@ -3780,20 +3817,16 @@ sequenceDecs_decodeSync_safe_bmi2_after_adjust: JNZ sequenceDecs_decodeSync_safe_bmi2_error_match_len_ofs_mismatch sequenceDecs_decodeSync_safe_bmi2_match_len_ofs_ok: - MOVQ 24(SP), CX - MOVQ 8(SP), R12 - MOVQ 16(SP), R13 - // Check if we have enough space in s.out - LEAQ (CX)(R13*1), R14 + LEAQ (R12)(CX*1), R14 ADDQ R9, R14 - CMPQ R14, 32(SP) + CMPQ R14, 64(SP) JA error_not_enough_space // Copy literals - TESTQ CX, CX + TESTQ R12, R12 JZ check_offset - MOVQ CX, R14 + MOVQ R12, R14 SUBQ $0x10, R14 JB copy_1_small @@ -3811,20 +3844,20 @@ copy_1_loop: JMP copy_1_end copy_1_small: - CMPQ CX, $0x03 + CMPQ R12, $0x03 JE copy_1_move_3 JB copy_1_move_1or2 - CMPQ CX, $0x08 + CMPQ R12, $0x08 JB copy_1_move_4through7 JMP copy_1_move_8through16 copy_1_move_1or2: MOVB (R10), R14 - MOVB -1(R10)(CX*1), R15 + MOVB -1(R10)(R12*1), R15 MOVB R14, (R9) - MOVB R15, -1(R9)(CX*1) - ADDQ CX, R10 - ADDQ CX, R9 + MOVB R15, -1(R9)(R12*1) + ADDQ R12, R10 + ADDQ R12, R9 JMP copy_1_end copy_1_move_3: @@ -3832,49 +3865,49 @@ copy_1_move_3: MOVB 2(R10), R15 MOVW R14, (R9) MOVB R15, 2(R9) - ADDQ CX, R10 - ADDQ CX, R9 + ADDQ R12, R10 + ADDQ R12, R9 JMP copy_1_end copy_1_move_4through7: MOVL (R10), R14 - MOVL -4(R10)(CX*1), R15 + MOVL -4(R10)(R12*1), R15 MOVL R14, (R9) - MOVL R15, -4(R9)(CX*1) - ADDQ CX, R10 - ADDQ CX, R9 + MOVL R15, -4(R9)(R12*1) + ADDQ R12, R10 + ADDQ R12, R9 JMP copy_1_end copy_1_move_8through16: MOVQ (R10), R14 - MOVQ -8(R10)(CX*1), R15 + MOVQ -8(R10)(R12*1), R15 MOVQ R14, (R9) - MOVQ R15, -8(R9)(CX*1) - ADDQ CX, R10 - ADDQ CX, R9 + MOVQ R15, -8(R9)(R12*1) + ADDQ R12, R10 + ADDQ R12, R9 copy_1_end: - ADDQ CX, R11 + ADDQ R12, R11 // Malformed input if seq.mo > t+len(hist) || seq.mo > s.windowSize) check_offset: - MOVQ R11, CX - ADDQ 40(SP), CX - CMPQ R12, CX + MOVQ R11, R12 + ADDQ 72(SP), R12 + CMPQ R13, R12 JG error_match_off_too_big - CMPQ R12, 56(SP) + CMPQ R13, 88(SP) JG error_match_off_too_big // Copy match from history - MOVQ R12, CX - SUBQ R11, CX + MOVQ R13, R12 + SUBQ R11, R12 JLS copy_match - MOVQ 48(SP), R14 - SUBQ CX, R14 - CMPQ R13, CX + MOVQ 80(SP), R14 + SUBQ R12, R14 + CMPQ CX, R12 JG copy_all_from_history - MOVQ R13, CX - SUBQ $0x10, CX + MOVQ CX, R12 + SUBQ $0x10, R12 JB copy_4_small copy_4_loop: @@ -3882,54 +3915,54 @@ copy_4_loop: MOVUPS X0, (R9) ADDQ $0x10, R14 ADDQ $0x10, R9 - SUBQ $0x10, CX + SUBQ $0x10, R12 JAE copy_4_loop - LEAQ 16(R14)(CX*1), R14 - LEAQ 16(R9)(CX*1), R9 + LEAQ 16(R14)(R12*1), R14 + LEAQ 16(R9)(R12*1), R9 MOVUPS -16(R14), X0 MOVUPS X0, -16(R9) JMP copy_4_end copy_4_small: - CMPQ R13, $0x03 + CMPQ CX, $0x03 JE copy_4_move_3 - CMPQ R13, $0x08 + CMPQ CX, $0x08 JB copy_4_move_4through7 JMP copy_4_move_8through16 copy_4_move_3: - MOVWQZX (R14), CX - MOVB 2(R14), R12 - MOVW CX, (R9) - MOVB R12, 2(R9) - ADDQ R13, R14 - ADDQ R13, R9 + MOVWQZX (R14), R12 + MOVB 2(R14), R13 + MOVW R12, (R9) + MOVB R13, 2(R9) + ADDQ CX, R14 + ADDQ CX, R9 JMP copy_4_end copy_4_move_4through7: - MOVL (R14), CX - MOVL -4(R14)(R13*1), R12 - MOVL CX, (R9) - MOVL R12, -4(R9)(R13*1) - ADDQ R13, R14 - ADDQ R13, R9 + MOVL (R14), R12 + MOVL -4(R14)(CX*1), R13 + MOVL R12, (R9) + MOVL R13, -4(R9)(CX*1) + ADDQ CX, R14 + ADDQ CX, R9 JMP copy_4_end copy_4_move_8through16: - MOVQ (R14), CX - MOVQ -8(R14)(R13*1), R12 - MOVQ CX, (R9) - MOVQ R12, -8(R9)(R13*1) - ADDQ R13, R14 - ADDQ R13, R9 + MOVQ (R14), R12 + MOVQ -8(R14)(CX*1), R13 + MOVQ R12, (R9) + MOVQ R13, -8(R9)(CX*1) + ADDQ CX, R14 + ADDQ CX, R9 copy_4_end: - ADDQ R13, R11 + ADDQ CX, R11 JMP handle_loop JMP loop_finished copy_all_from_history: - MOVQ CX, R15 + MOVQ R12, R15 SUBQ $0x10, R15 JB copy_5_small @@ -3947,20 +3980,20 @@ copy_5_loop: JMP copy_5_end copy_5_small: - CMPQ CX, $0x03 + CMPQ R12, $0x03 JE copy_5_move_3 JB copy_5_move_1or2 - CMPQ CX, $0x08 + CMPQ R12, $0x08 JB copy_5_move_4through7 JMP copy_5_move_8through16 copy_5_move_1or2: MOVB (R14), R15 - MOVB -1(R14)(CX*1), BP + MOVB -1(R14)(R12*1), BP MOVB R15, (R9) - MOVB BP, -1(R9)(CX*1) - ADDQ CX, R14 - ADDQ CX, R9 + MOVB BP, -1(R9)(R12*1) + ADDQ R12, R14 + ADDQ R12, R9 JMP copy_5_end copy_5_move_3: @@ -3968,120 +4001,119 @@ copy_5_move_3: MOVB 2(R14), BP MOVW R15, (R9) MOVB BP, 2(R9) - ADDQ CX, R14 - ADDQ CX, R9 + ADDQ R12, R14 + ADDQ R12, R9 JMP copy_5_end copy_5_move_4through7: MOVL (R14), R15 - MOVL -4(R14)(CX*1), BP + MOVL -4(R14)(R12*1), BP MOVL R15, (R9) - MOVL BP, -4(R9)(CX*1) - ADDQ CX, R14 - ADDQ CX, R9 + MOVL BP, -4(R9)(R12*1) + ADDQ R12, R14 + ADDQ R12, R9 JMP copy_5_end copy_5_move_8through16: MOVQ (R14), R15 - MOVQ -8(R14)(CX*1), BP + MOVQ -8(R14)(R12*1), BP MOVQ R15, (R9) - MOVQ BP, -8(R9)(CX*1) - ADDQ CX, R14 - ADDQ CX, R9 + MOVQ BP, -8(R9)(R12*1) + ADDQ R12, R14 + ADDQ R12, R9 copy_5_end: - ADDQ CX, R11 - SUBQ CX, R13 + ADDQ R12, R11 + SUBQ R12, CX // Copy match from the current buffer copy_match: - MOVQ R9, CX - SUBQ R12, CX + MOVQ R9, R12 + SUBQ R13, R12 // ml <= mo - CMPQ R13, R12 + CMPQ CX, R13 JA copy_overlapping_match // Copy non-overlapping match - ADDQ R13, R11 - MOVQ R13, R12 - SUBQ $0x10, R12 + ADDQ CX, R11 + MOVQ CX, R13 + SUBQ $0x10, R13 JB copy_2_small copy_2_loop: - MOVUPS (CX), X0 + MOVUPS (R12), X0 MOVUPS X0, (R9) - ADDQ $0x10, CX + ADDQ $0x10, R12 ADDQ $0x10, R9 - SUBQ $0x10, R12 + SUBQ $0x10, R13 JAE copy_2_loop - LEAQ 16(CX)(R12*1), CX - LEAQ 16(R9)(R12*1), R9 - MOVUPS -16(CX), X0 + LEAQ 16(R12)(R13*1), R12 + LEAQ 16(R9)(R13*1), R9 + MOVUPS -16(R12), X0 MOVUPS X0, -16(R9) JMP copy_2_end copy_2_small: - CMPQ R13, $0x03 + CMPQ CX, $0x03 JE copy_2_move_3 JB copy_2_move_1or2 - CMPQ R13, $0x08 + CMPQ CX, $0x08 JB copy_2_move_4through7 JMP copy_2_move_8through16 copy_2_move_1or2: - MOVB (CX), R12 - MOVB -1(CX)(R13*1), R14 - MOVB R12, (R9) - MOVB R14, -1(R9)(R13*1) - ADDQ R13, CX - ADDQ R13, R9 + MOVB (R12), R13 + MOVB -1(R12)(CX*1), R14 + MOVB R13, (R9) + MOVB R14, -1(R9)(CX*1) + ADDQ CX, R12 + ADDQ CX, R9 JMP copy_2_end copy_2_move_3: - MOVWQZX (CX), R12 - MOVB 2(CX), R14 - MOVW R12, (R9) + MOVWQZX (R12), R13 + MOVB 2(R12), R14 + MOVW R13, (R9) MOVB R14, 2(R9) - ADDQ R13, CX - ADDQ R13, R9 + ADDQ CX, R12 + ADDQ CX, R9 JMP copy_2_end copy_2_move_4through7: - MOVL (CX), R12 - MOVL -4(CX)(R13*1), R14 - MOVL R12, (R9) - MOVL R14, -4(R9)(R13*1) - ADDQ R13, CX - ADDQ R13, R9 + MOVL (R12), R13 + MOVL -4(R12)(CX*1), R14 + MOVL R13, (R9) + MOVL R14, -4(R9)(CX*1) + ADDQ CX, R12 + ADDQ CX, R9 JMP copy_2_end copy_2_move_8through16: - MOVQ (CX), R12 - MOVQ -8(CX)(R13*1), R14 - MOVQ R12, (R9) - MOVQ R14, -8(R9)(R13*1) - ADDQ R13, CX - ADDQ R13, R9 + MOVQ (R12), R13 + MOVQ -8(R12)(CX*1), R14 + MOVQ R13, (R9) + MOVQ R14, -8(R9)(CX*1) + ADDQ CX, R12 + ADDQ CX, R9 copy_2_end: JMP handle_loop // Copy overlapping match copy_overlapping_match: - ADDQ R13, R11 + ADDQ CX, R11 copy_slow_3: - MOVBQZX (CX), R12 - MOVB R12, (R9) - INCQ CX + MOVBQZX (R12), R13 + MOVB R13, (R9) + INCQ R12 INCQ R9 - DECQ R13 + DECQ CX JNZ copy_slow_3 handle_loop: - MOVQ ctx+16(FP), CX - DECQ 96(CX) + DECQ 32(SP) JNS sequenceDecs_decodeSync_safe_bmi2_main_loop loop_finished: @@ -4090,6 +4122,13 @@ loop_finished: MOVB DL, 40(CX) MOVQ BX, 32(CX) + // s.seqSize += outPosition - ctx.outPosition + MOVQ ctx+16(FP), AX + MOVQ s+0(FP), CX + MOVQ R11, DX + SUBQ 136(AX), DX + ADDQ DX, 256(CX) + // Update the context MOVQ ctx+16(FP), AX MOVQ R11, 136(AX) @@ -4103,7 +4142,7 @@ loop_finished: // Return with match length error sequenceDecs_decodeSync_safe_bmi2_error_match_len_ofs_mismatch: - MOVQ 16(SP), AX + MOVQ 48(SP), AX MOVQ ctx+16(FP), CX MOVQ AX, 216(CX) MOVQ $0x00000001, ret+24(FP) @@ -4112,7 +4151,7 @@ sequenceDecs_decodeSync_safe_bmi2_error_match_len_ofs_mismatch: // Return with match too long error sequenceDecs_decodeSync_safe_bmi2_error_match_len_too_big: MOVQ ctx+16(FP), AX - MOVQ 16(SP), CX + MOVQ 48(SP), CX MOVQ CX, 216(AX) MOVQ $0x00000002, ret+24(FP) RET @@ -4120,7 +4159,7 @@ sequenceDecs_decodeSync_safe_bmi2_error_match_len_too_big: // Return with match offset too long error error_match_off_too_big: MOVQ ctx+16(FP), AX - MOVQ 8(SP), CX + MOVQ 40(SP), CX MOVQ CX, 224(AX) MOVQ R11, 136(AX) MOVQ $0x00000003, ret+24(FP) @@ -4129,7 +4168,7 @@ error_match_off_too_big: // Return with not enough literals error error_not_enough_literals: MOVQ ctx+16(FP), AX - MOVQ 24(SP), CX + MOVQ 56(SP), CX MOVQ CX, 208(AX) MOVQ $0x00000004, ret+24(FP) RET @@ -4142,9 +4181,9 @@ error_overread: // Return with not enough output space error error_not_enough_space: MOVQ ctx+16(FP), AX - MOVQ 24(SP), CX + MOVQ 56(SP), CX MOVQ CX, 208(AX) - MOVQ 16(SP), CX + MOVQ 48(SP), CX MOVQ CX, 216(AX) MOVQ R11, 136(AX) MOVQ $0x00000005, ret+24(FP) diff --git a/vendor/github.com/klauspost/compress/zstd/seqdec_arm64.s b/vendor/github.com/klauspost/compress/zstd/seqdec_arm64.s index fa48be19e4e8..56391bfc1839 100644 --- a/vendor/github.com/klauspost/compress/zstd/seqdec_arm64.s +++ b/vendor/github.com/klauspost/compress/zstd/seqdec_arm64.s @@ -5,7 +5,7 @@ // func sequenceDecs_decode_amd64(s *sequenceDecs, br *bitReader, ctx *decodeAsmContext) int // Requires: CMOV -TEXT ·sequenceDecs_decode_arm64(SB), $8-32 +TEXT ·sequenceDecs_decode_arm64(SB), $40-32 MOVD br+8(FP), R1 MOVD 24(R1), R2 MOVBU 40(R1), R3 @@ -17,6 +17,14 @@ TEXT ·sequenceDecs_decode_arm64(SB), $8-32 MOVD 72(R0), R6 MOVD 80(R0), R7 MOVD 88(R0), R8 + MOVD (R0), R1 + MOVD R1, 16(RSP) + MOVD 24(R0), R1 + MOVD R1, 24(RSP) + MOVD 48(R0), R1 + MOVD R1, 32(RSP) + MOVD 96(R0), R1 + MOVD R1, 40(RSP) MOVD 104(R0), R9 MOVD s+0(FP), R0 MOVD 144(R0), R10 @@ -29,8 +37,7 @@ sequenceDecs_decode_amd64_main_loop: // Fill bitreader to have enough for the offset and match length. CMP $0x08, R5 BLT sequenceDecs_decode_amd64_fill_byte_by_byte - MOVD R3, R0 - LSR $0x03, R0, R0 + LSR $0x03, R3, R0 SUB R0, R13, R13 MOVD (R13), R2 SUB R0, R5, R5 @@ -57,51 +64,32 @@ sequenceDecs_decode_amd64_fill_check_overread: sequenceDecs_decode_amd64_fill_end: // Update offset MOVD R8, R0 - MOVD R3, R1 - MOVD R2, R14 - LSL R1, R14, R14 + LSL R3, R2, R14 + LSR $0x01, R14, R14 UBFX $8, R0, $8, R1 LSR $0x20, R0, R0 - TST R1, R1 - BEQ sequenceDecs_decode_amd64_of_update_zero ADD R1, R3, R3 - CMP $0x40, R3 - BHI sequenceDecs_decode_amd64_of_update_zero - CMP $0x40, R1 - BHS sequenceDecs_decode_amd64_of_update_zero - NEG R1, R1 + EOR $0x3f, R1, R1 LSR R1, R14, R14 ADD R14, R0, R0 - -sequenceDecs_decode_amd64_of_update_zero: MOVD R0, 16(R9) // Update match length MOVD R7, R0 - MOVD R3, R1 - MOVD R2, R14 - LSL R1, R14, R14 + LSL R3, R2, R14 + LSR $0x01, R14, R14 UBFX $8, R0, $8, R1 LSR $0x20, R0, R0 - TST R1, R1 - BEQ sequenceDecs_decode_amd64_ml_update_zero ADD R1, R3, R3 - CMP $0x40, R3 - BHI sequenceDecs_decode_amd64_ml_update_zero - CMP $0x40, R1 - BHS sequenceDecs_decode_amd64_ml_update_zero - NEG R1, R1 + EOR $0x3f, R1, R1 LSR R1, R14, R14 ADD R14, R0, R0 - -sequenceDecs_decode_amd64_ml_update_zero: MOVD R0, 8(R9) // Fill bitreader to have enough for the remaining CMP $0x08, R5 BLT sequenceDecs_decode_amd64_fill_2_byte_by_byte - MOVD R3, R0 - LSR $0x03, R0, R0 + LSR $0x03, R3, R0 SUB R0, R13, R13 MOVD (R13), R2 SUB R0, R5, R5 @@ -128,91 +116,63 @@ sequenceDecs_decode_amd64_fill_2_check_overread: sequenceDecs_decode_amd64_fill_2_end: // Update literal length MOVD R6, R0 - MOVD R3, R1 - MOVD R2, R14 - LSL R1, R14, R14 + LSL R3, R2, R14 + LSR $0x01, R14, R14 UBFX $8, R0, $8, R1 LSR $0x20, R0, R0 - TST R1, R1 - BEQ sequenceDecs_decode_amd64_ll_update_zero ADD R1, R3, R3 - CMP $0x40, R3 - BHI sequenceDecs_decode_amd64_ll_update_zero - CMP $0x40, R1 - BHS sequenceDecs_decode_amd64_ll_update_zero - NEG R1, R1 + EOR $0x3f, R1, R1 LSR R1, R14, R14 ADD R14, R0, R0 - -sequenceDecs_decode_amd64_ll_update_zero: MOVD R0, (R9) // Fill bitreader for state updates MOVD R13, 8(RSP) UBFX $8, R8, $8, R0 - MOVD ctx+16(FP), R1 - MOVD 96(R1), R16 + MOVD 40(RSP), R16 CMP $0x00, R16 BEQ sequenceDecs_decode_amd64_skip_update // Update Literal Length State MOVBU R6, R13 LSRW $0x10, R6, R6 - ADD R13, R3, R1 - MOVD R2, R14 - MOVD R1, R3 - NEG R1, R16 - ROR R16, R14, R14 - MOVD $0x00000001, R4 - MOVBU R13, R1 - LSLW R1, R4, R4 - SUBW $1, R4, R4 - AND R4, R14, R14 + LSL R3, R2, R14 + LSR $0x01, R14, R14 + ADD R13, R3, R3 + EOR $0x3f, R13, R13 + LSR R13, R14, R14 ADD R14, R6, R6 // Load ctx.llTable - MOVD ctx+16(FP), R1 - MOVD (R1), R1 + MOVD 16(RSP), R1 MOVD (R1)(R6<<3), R6 // Update Match Length State MOVBU R7, R13 LSRW $0x10, R7, R7 - ADD R13, R3, R1 - MOVD R2, R14 - MOVD R1, R3 - NEG R1, R16 - ROR R16, R14, R14 - MOVD $0x00000001, R4 - MOVBU R13, R1 - LSLW R1, R4, R4 - SUBW $1, R4, R4 - AND R4, R14, R14 + LSL R3, R2, R14 + LSR $0x01, R14, R14 + ADD R13, R3, R3 + EOR $0x3f, R13, R13 + LSR R13, R14, R14 ADD R14, R7, R7 // Load ctx.mlTable - MOVD ctx+16(FP), R1 - MOVD 24(R1), R1 + MOVD 24(RSP), R1 MOVD (R1)(R7<<3), R7 // Update Offset State MOVBU R8, R13 LSRW $0x10, R8, R8 - ADD R13, R3, R1 - MOVD R2, R14 - MOVD R1, R3 - NEG R1, R16 - ROR R16, R14, R14 - MOVD $0x00000001, R4 - MOVBU R13, R1 - LSLW R1, R4, R4 - SUBW $1, R4, R4 - AND R4, R14, R14 + LSL R3, R2, R14 + LSR $0x01, R14, R14 + ADD R13, R3, R3 + EOR $0x3f, R13, R13 + LSR R13, R14, R14 ADD R14, R8, R8 // Load ctx.ofTable - MOVD ctx+16(FP), R1 - MOVD 48(R1), R1 + MOVD 32(RSP), R1 MOVD (R1)(R8<<3), R8 sequenceDecs_decode_amd64_skip_update: @@ -298,10 +258,9 @@ sequenceDecs_decode_amd64_after_adjust: sequenceDecs_decode_amd64_match_len_ofs_ok: ADD $0x18, R9, R9 - MOVD ctx+16(FP), R0 - MOVD 96(R0), R16 + MOVD 40(RSP), R16 SUBS $1, R16, R16 - MOVD R16, 96(R0) + MOVD R16, 40(RSP) BPL sequenceDecs_decode_amd64_main_loop MOVD s+0(FP), R0 MOVD R10, 144(R0) @@ -313,42 +272,60 @@ sequenceDecs_decode_amd64_match_len_ofs_ok: MOVD R5, 32(R0) // Return success + MOVD 40(RSP), R0 + MOVD ctx+16(FP), R1 + MOVD R0, 96(R1) MOVD $0x00000000, R16 MOVD R16, ret+24(FP) RET // Return with match length error sequenceDecs_decode_amd64_error_match_len_ofs_mismatch: + MOVD 40(RSP), R0 + MOVD ctx+16(FP), R1 + MOVD R0, 96(R1) MOVD $0x00000001, R16 MOVD R16, ret+24(FP) RET // Return with match too long error sequenceDecs_decode_amd64_error_match_len_too_big: + MOVD 40(RSP), R0 + MOVD ctx+16(FP), R1 + MOVD R0, 96(R1) MOVD $0x00000002, R16 MOVD R16, ret+24(FP) RET // Return with match offset too long error + MOVD 40(RSP), R0 + MOVD ctx+16(FP), R1 + MOVD R0, 96(R1) MOVD $0x00000003, R16 MOVD R16, ret+24(FP) RET // Return with not enough literals error error_not_enough_literals: + MOVD 40(RSP), R0 + MOVD ctx+16(FP), R1 + MOVD R0, 96(R1) MOVD $0x00000004, R16 MOVD R16, ret+24(FP) RET // Return with overread error error_overread: + MOVD 40(RSP), R0 + MOVD ctx+16(FP), R1 + MOVD R0, 96(R1) MOVD $0x00000006, R16 MOVD R16, ret+24(FP) RET // func sequenceDecs_decode_56_amd64(s *sequenceDecs, br *bitReader, ctx *decodeAsmContext) int // Requires: CMOV -TEXT ·sequenceDecs_decode_56_arm64(SB), $8-32 +TEXT ·sequenceDecs_decode_56_arm64(SB), $40-32 MOVD br+8(FP), R1 MOVD 24(R1), R2 MOVBU 40(R1), R3 @@ -360,6 +337,14 @@ TEXT ·sequenceDecs_decode_56_arm64(SB), $8-32 MOVD 72(R0), R6 MOVD 80(R0), R7 MOVD 88(R0), R8 + MOVD (R0), R1 + MOVD R1, 16(RSP) + MOVD 24(R0), R1 + MOVD R1, 24(RSP) + MOVD 48(R0), R1 + MOVD R1, 32(RSP) + MOVD 96(R0), R1 + MOVD R1, 40(RSP) MOVD 104(R0), R9 MOVD s+0(FP), R0 MOVD 144(R0), R10 @@ -372,8 +357,7 @@ sequenceDecs_decode_56_amd64_main_loop: // Fill bitreader to have enough for the offset and match length. CMP $0x08, R5 BLT sequenceDecs_decode_56_amd64_fill_byte_by_byte - MOVD R3, R0 - LSR $0x03, R0, R0 + LSR $0x03, R3, R0 SUB R0, R13, R13 MOVD (R13), R2 SUB R0, R5, R5 @@ -400,133 +384,87 @@ sequenceDecs_decode_56_amd64_fill_check_overread: sequenceDecs_decode_56_amd64_fill_end: // Update offset MOVD R8, R0 - MOVD R3, R1 - MOVD R2, R14 - LSL R1, R14, R14 + LSL R3, R2, R14 + LSR $0x01, R14, R14 UBFX $8, R0, $8, R1 LSR $0x20, R0, R0 - TST R1, R1 - BEQ sequenceDecs_decode_56_amd64_of_update_zero ADD R1, R3, R3 - CMP $0x40, R3 - BHI sequenceDecs_decode_56_amd64_of_update_zero - CMP $0x40, R1 - BHS sequenceDecs_decode_56_amd64_of_update_zero - NEG R1, R1 + EOR $0x3f, R1, R1 LSR R1, R14, R14 ADD R14, R0, R0 - -sequenceDecs_decode_56_amd64_of_update_zero: MOVD R0, 16(R9) // Update match length MOVD R7, R0 - MOVD R3, R1 - MOVD R2, R14 - LSL R1, R14, R14 + LSL R3, R2, R14 + LSR $0x01, R14, R14 UBFX $8, R0, $8, R1 LSR $0x20, R0, R0 - TST R1, R1 - BEQ sequenceDecs_decode_56_amd64_ml_update_zero ADD R1, R3, R3 - CMP $0x40, R3 - BHI sequenceDecs_decode_56_amd64_ml_update_zero - CMP $0x40, R1 - BHS sequenceDecs_decode_56_amd64_ml_update_zero - NEG R1, R1 + EOR $0x3f, R1, R1 LSR R1, R14, R14 ADD R14, R0, R0 - -sequenceDecs_decode_56_amd64_ml_update_zero: MOVD R0, 8(R9) // Update literal length MOVD R6, R0 - MOVD R3, R1 - MOVD R2, R14 - LSL R1, R14, R14 + LSL R3, R2, R14 + LSR $0x01, R14, R14 UBFX $8, R0, $8, R1 LSR $0x20, R0, R0 - TST R1, R1 - BEQ sequenceDecs_decode_56_amd64_ll_update_zero ADD R1, R3, R3 - CMP $0x40, R3 - BHI sequenceDecs_decode_56_amd64_ll_update_zero - CMP $0x40, R1 - BHS sequenceDecs_decode_56_amd64_ll_update_zero - NEG R1, R1 + EOR $0x3f, R1, R1 LSR R1, R14, R14 ADD R14, R0, R0 - -sequenceDecs_decode_56_amd64_ll_update_zero: MOVD R0, (R9) // Fill bitreader for state updates MOVD R13, 8(RSP) UBFX $8, R8, $8, R0 - MOVD ctx+16(FP), R1 - MOVD 96(R1), R16 + MOVD 40(RSP), R16 CMP $0x00, R16 BEQ sequenceDecs_decode_56_amd64_skip_update // Update Literal Length State MOVBU R6, R13 LSRW $0x10, R6, R6 - ADD R13, R3, R1 - MOVD R2, R14 - MOVD R1, R3 - NEG R1, R16 - ROR R16, R14, R14 - MOVD $0x00000001, R4 - MOVBU R13, R1 - LSLW R1, R4, R4 - SUBW $1, R4, R4 - AND R4, R14, R14 + LSL R3, R2, R14 + LSR $0x01, R14, R14 + ADD R13, R3, R3 + EOR $0x3f, R13, R13 + LSR R13, R14, R14 ADD R14, R6, R6 // Load ctx.llTable - MOVD ctx+16(FP), R1 - MOVD (R1), R1 + MOVD 16(RSP), R1 MOVD (R1)(R6<<3), R6 // Update Match Length State MOVBU R7, R13 LSRW $0x10, R7, R7 - ADD R13, R3, R1 - MOVD R2, R14 - MOVD R1, R3 - NEG R1, R16 - ROR R16, R14, R14 - MOVD $0x00000001, R4 - MOVBU R13, R1 - LSLW R1, R4, R4 - SUBW $1, R4, R4 - AND R4, R14, R14 + LSL R3, R2, R14 + LSR $0x01, R14, R14 + ADD R13, R3, R3 + EOR $0x3f, R13, R13 + LSR R13, R14, R14 ADD R14, R7, R7 // Load ctx.mlTable - MOVD ctx+16(FP), R1 - MOVD 24(R1), R1 + MOVD 24(RSP), R1 MOVD (R1)(R7<<3), R7 // Update Offset State MOVBU R8, R13 LSRW $0x10, R8, R8 - ADD R13, R3, R1 - MOVD R2, R14 - MOVD R1, R3 - NEG R1, R16 - ROR R16, R14, R14 - MOVD $0x00000001, R4 - MOVBU R13, R1 - LSLW R1, R4, R4 - SUBW $1, R4, R4 - AND R4, R14, R14 + LSL R3, R2, R14 + LSR $0x01, R14, R14 + ADD R13, R3, R3 + EOR $0x3f, R13, R13 + LSR R13, R14, R14 ADD R14, R8, R8 // Load ctx.ofTable - MOVD ctx+16(FP), R1 - MOVD 48(R1), R1 + MOVD 32(RSP), R1 MOVD (R1)(R8<<3), R8 sequenceDecs_decode_56_amd64_skip_update: @@ -612,10 +550,9 @@ sequenceDecs_decode_56_amd64_after_adjust: sequenceDecs_decode_56_amd64_match_len_ofs_ok: ADD $0x18, R9, R9 - MOVD ctx+16(FP), R0 - MOVD 96(R0), R16 + MOVD 40(RSP), R16 SUBS $1, R16, R16 - MOVD R16, 96(R0) + MOVD R16, 40(RSP) BPL sequenceDecs_decode_56_amd64_main_loop MOVD s+0(FP), R0 MOVD R10, 144(R0) @@ -627,35 +564,53 @@ sequenceDecs_decode_56_amd64_match_len_ofs_ok: MOVD R5, 32(R0) // Return success + MOVD 40(RSP), R0 + MOVD ctx+16(FP), R1 + MOVD R0, 96(R1) MOVD $0x00000000, R16 MOVD R16, ret+24(FP) RET // Return with match length error sequenceDecs_decode_56_amd64_error_match_len_ofs_mismatch: + MOVD 40(RSP), R0 + MOVD ctx+16(FP), R1 + MOVD R0, 96(R1) MOVD $0x00000001, R16 MOVD R16, ret+24(FP) RET // Return with match too long error sequenceDecs_decode_56_amd64_error_match_len_too_big: + MOVD 40(RSP), R0 + MOVD ctx+16(FP), R1 + MOVD R0, 96(R1) MOVD $0x00000002, R16 MOVD R16, ret+24(FP) RET // Return with match offset too long error + MOVD 40(RSP), R0 + MOVD ctx+16(FP), R1 + MOVD R0, 96(R1) MOVD $0x00000003, R16 MOVD R16, ret+24(FP) RET // Return with not enough literals error error_not_enough_literals: + MOVD 40(RSP), R0 + MOVD ctx+16(FP), R1 + MOVD R0, 96(R1) MOVD $0x00000004, R16 MOVD R16, ret+24(FP) RET // Return with overread error error_overread: + MOVD 40(RSP), R0 + MOVD ctx+16(FP), R1 + MOVD R0, 96(R1) MOVD $0x00000006, R16 MOVD R16, ret+24(FP) RET @@ -695,9 +650,11 @@ main_loop: MOVD 8(R0), R12 // Copy literals - TST R10, R10 - BEQ check_offset - MOVD $0, R13 + FMOVQ (R5), F0 + FMOVQ F0, (R3) + CMP $0x10, R10 + BLS copy_1_end + MOVD $0x00000010, R13 copy_1: ADD R13, R5, R15 @@ -707,12 +664,13 @@ copy_1: ADD $0x10, R13, R13 CMP R10, R13 BLO copy_1 - ADD R10, R5, R5 - ADD R10, R3, R3 - ADD R10, R6, R6 + +copy_1_end: + ADD R10, R5, R5 + ADD R10, R3, R3 + ADD R10, R6, R6 // Malformed input if seq.mo > t+len(hist) || seq.mo > s.windowSize) -check_offset: ADD R9, R6, R10 CMP R10, R11 BGT error_match_off_too_big @@ -878,9 +836,19 @@ copy_match: BHI copy_overlapping_match // Copy non-overlapping match - ADD R12, R6, R6 - MOVD R3, R11 - ADD R12, R3, R3 + ADD R12, R6, R6 + FMOVQ (R10), F0 + FMOVQ F0, (R3) + CMP $0x10, R12 + BHI copy_2_long + ADD R12, R3, R3 + JMP handle_loop + +copy_2_long: + ADD $16, R3, R11 + ADD R12, R3, R3 + ADD $0x10, R10, R10 + SUB $0x10, R12, R12 copy_2: FMOVQ (R10), F0 @@ -1346,7 +1314,7 @@ empty_seqs: // func sequenceDecs_decodeSync_amd64(s *sequenceDecs, br *bitReader, ctx *decodeSyncAsmContext) int // Requires: CMOV, SSE -TEXT ·sequenceDecs_decodeSync_arm64(SB), $64-32 +TEXT ·sequenceDecs_decodeSync_arm64(SB), $96-32 MOVD br+8(FP), R1 MOVD 24(R1), R2 MOVBU 40(R1), R3 @@ -1358,30 +1326,38 @@ TEXT ·sequenceDecs_decodeSync_arm64(SB), $64-32 MOVD 72(R0), R6 MOVD 80(R0), R7 MOVD 88(R0), R8 - MOVD $0, R1 + MOVD (R0), R1 MOVD R1, 16(RSP) + MOVD 24(R0), R1 MOVD R1, 24(RSP) + MOVD 48(R0), R1 MOVD R1, 32(RSP) + MOVD 96(R0), R1 + MOVD R1, 40(RSP) + MOVD $0, R1 + MOVD R1, 48(RSP) + MOVD R1, 56(RSP) + MOVD R1, 64(RSP) MOVD 112(R0), R9 MOVD 128(R0), R1 - MOVD R1, 40(RSP) + MOVD R1, 72(RSP) MOVD 144(R0), R10 MOVD 136(R0), R11 MOVD 200(R0), R1 - MOVD R1, 64(RSP) + MOVD R1, 96(RSP) MOVD 176(R0), R1 - MOVD R1, 56(RSP) + MOVD R1, 88(RSP) MOVD 184(R0), R0 - MOVD R0, 48(RSP) - MOVD 48(RSP), R0 - MOVD 56(RSP), R16 + MOVD R0, 80(RSP) + MOVD 80(RSP), R0 + MOVD 88(RSP), R16 ADD R0, R16, R16 - MOVD R16, 56(RSP) + MOVD R16, 88(RSP) // Calculate pointer to s.out[cap(s.out)] (a past-end pointer) - MOVD 40(RSP), R16 + MOVD 72(RSP), R16 ADD R9, R16, R16 - MOVD R16, 40(RSP) + MOVD R16, 72(RSP) // outBase += outPosition ADD R11, R9, R9 @@ -1392,8 +1368,7 @@ sequenceDecs_decodeSync_amd64_main_loop: // Fill bitreader to have enough for the offset and match length. CMP $0x08, R5 BLT sequenceDecs_decodeSync_amd64_fill_byte_by_byte - MOVD R3, R0 - LSR $0x03, R0, R0 + LSR $0x03, R3, R0 SUB R0, R12, R12 MOVD (R12), R2 SUB R0, R5, R5 @@ -1420,51 +1395,32 @@ sequenceDecs_decodeSync_amd64_fill_check_overread: sequenceDecs_decodeSync_amd64_fill_end: // Update offset MOVD R8, R0 - MOVD R3, R1 - MOVD R2, R13 - LSL R1, R13, R13 + LSL R3, R2, R13 + LSR $0x01, R13, R13 UBFX $8, R0, $8, R1 LSR $0x20, R0, R0 - TST R1, R1 - BEQ sequenceDecs_decodeSync_amd64_of_update_zero ADD R1, R3, R3 - CMP $0x40, R3 - BHI sequenceDecs_decodeSync_amd64_of_update_zero - CMP $0x40, R1 - BHS sequenceDecs_decodeSync_amd64_of_update_zero - NEG R1, R1 + EOR $0x3f, R1, R1 LSR R1, R13, R13 ADD R13, R0, R0 - -sequenceDecs_decodeSync_amd64_of_update_zero: - MOVD R0, 16(RSP) + MOVD R0, 48(RSP) // Update match length MOVD R7, R0 - MOVD R3, R1 - MOVD R2, R13 - LSL R1, R13, R13 + LSL R3, R2, R13 + LSR $0x01, R13, R13 UBFX $8, R0, $8, R1 LSR $0x20, R0, R0 - TST R1, R1 - BEQ sequenceDecs_decodeSync_amd64_ml_update_zero ADD R1, R3, R3 - CMP $0x40, R3 - BHI sequenceDecs_decodeSync_amd64_ml_update_zero - CMP $0x40, R1 - BHS sequenceDecs_decodeSync_amd64_ml_update_zero - NEG R1, R1 + EOR $0x3f, R1, R1 LSR R1, R13, R13 ADD R13, R0, R0 - -sequenceDecs_decodeSync_amd64_ml_update_zero: - MOVD R0, 24(RSP) + MOVD R0, 56(RSP) // Fill bitreader to have enough for the remaining CMP $0x08, R5 BLT sequenceDecs_decodeSync_amd64_fill_2_byte_by_byte - MOVD R3, R0 - LSR $0x03, R0, R0 + LSR $0x03, R3, R0 SUB R0, R12, R12 MOVD (R12), R2 SUB R0, R5, R5 @@ -1491,97 +1447,69 @@ sequenceDecs_decodeSync_amd64_fill_2_check_overread: sequenceDecs_decodeSync_amd64_fill_2_end: // Update literal length MOVD R6, R0 - MOVD R3, R1 - MOVD R2, R13 - LSL R1, R13, R13 + LSL R3, R2, R13 + LSR $0x01, R13, R13 UBFX $8, R0, $8, R1 LSR $0x20, R0, R0 - TST R1, R1 - BEQ sequenceDecs_decodeSync_amd64_ll_update_zero ADD R1, R3, R3 - CMP $0x40, R3 - BHI sequenceDecs_decodeSync_amd64_ll_update_zero - CMP $0x40, R1 - BHS sequenceDecs_decodeSync_amd64_ll_update_zero - NEG R1, R1 + EOR $0x3f, R1, R1 LSR R1, R13, R13 ADD R13, R0, R0 - -sequenceDecs_decodeSync_amd64_ll_update_zero: - MOVD R0, 32(RSP) + MOVD R0, 64(RSP) // Fill bitreader for state updates MOVD R12, 8(RSP) UBFX $8, R8, $8, R0 - MOVD ctx+16(FP), R1 - MOVD 96(R1), R16 + MOVD 40(RSP), R16 CMP $0x00, R16 BEQ sequenceDecs_decodeSync_amd64_skip_update // Update Literal Length State MOVBU R6, R12 LSRW $0x10, R6, R6 - ADD R12, R3, R1 - MOVD R2, R13 - MOVD R1, R3 - NEG R1, R16 - ROR R16, R13, R13 - MOVD $0x00000001, R14 - MOVBU R12, R1 - LSLW R1, R14, R14 - SUBW $1, R14, R14 - AND R14, R13, R13 + LSL R3, R2, R13 + LSR $0x01, R13, R13 + ADD R12, R3, R3 + EOR $0x3f, R12, R12 + LSR R12, R13, R13 ADD R13, R6, R6 // Load ctx.llTable - MOVD ctx+16(FP), R1 - MOVD (R1), R1 + MOVD 16(RSP), R1 MOVD (R1)(R6<<3), R6 // Update Match Length State MOVBU R7, R12 LSRW $0x10, R7, R7 - ADD R12, R3, R1 - MOVD R2, R13 - MOVD R1, R3 - NEG R1, R16 - ROR R16, R13, R13 - MOVD $0x00000001, R14 - MOVBU R12, R1 - LSLW R1, R14, R14 - SUBW $1, R14, R14 - AND R14, R13, R13 + LSL R3, R2, R13 + LSR $0x01, R13, R13 + ADD R12, R3, R3 + EOR $0x3f, R12, R12 + LSR R12, R13, R13 ADD R13, R7, R7 // Load ctx.mlTable - MOVD ctx+16(FP), R1 - MOVD 24(R1), R1 + MOVD 24(RSP), R1 MOVD (R1)(R7<<3), R7 // Update Offset State MOVBU R8, R12 LSRW $0x10, R8, R8 - ADD R12, R3, R1 - MOVD R2, R13 - MOVD R1, R3 - NEG R1, R16 - ROR R16, R13, R13 - MOVD $0x00000001, R14 - MOVBU R12, R1 - LSLW R1, R14, R14 - SUBW $1, R14, R14 - AND R14, R13, R13 + LSL R3, R2, R13 + LSR $0x01, R13, R13 + ADD R12, R3, R3 + EOR $0x3f, R12, R12 + LSR R12, R13, R13 ADD R13, R8, R8 // Load ctx.ofTable - MOVD ctx+16(FP), R1 - MOVD 48(R1), R1 + MOVD 32(RSP), R1 MOVD (R1)(R8<<3), R8 sequenceDecs_decodeSync_amd64_skip_update: // Adjust offset MOVD s+0(FP), R1 - MOVD 16(RSP), R12 + MOVD 48(RSP), R12 CMP $0x01, R0 BLS sequenceDecs_decodeSync_amd64_adjust_offsetB_1_or_0 FMOVQ 144(R1), F0 @@ -1590,7 +1518,7 @@ sequenceDecs_decodeSync_amd64_skip_update: JMP sequenceDecs_decodeSync_amd64_after_adjust sequenceDecs_decodeSync_amd64_adjust_offsetB_1_or_0: - MOVD 32(RSP), R16 + MOVD 64(RSP), R16 CMP $0x00000000, R16 BNE sequenceDecs_decodeSync_amd64_adjust_offset_maybezero ADD $1, R12, R12 @@ -1628,16 +1556,11 @@ sequenceDecs_decodeSync_amd64_adjust_skip: MOVD R13, R12 sequenceDecs_decodeSync_amd64_after_adjust: - MOVD R12, 16(RSP) + MOVD R12, 48(RSP) // Check values - MOVD 24(RSP), R0 - MOVD 32(RSP), R1 - ADD R1, R0, R13 - MOVD s+0(FP), R14 - MOVD 256(R14), R16 - ADD R13, R16, R16 - MOVD R16, 256(R14) + MOVD 56(RSP), R0 + MOVD 64(RSP), R1 MOVD ctx+16(FP), R13 MOVD 104(R13), R16 SUBS R1, R16, R16 @@ -1651,22 +1574,20 @@ sequenceDecs_decodeSync_amd64_after_adjust: BNE sequenceDecs_decodeSync_amd64_error_match_len_ofs_mismatch sequenceDecs_decodeSync_amd64_match_len_ofs_ok: - MOVD 32(RSP), R0 - MOVD 16(RSP), R1 - MOVD 24(RSP), R12 - // Check if we have enough space in s.out - ADD R12, R0, R13 + ADD R0, R1, R13 ADD $16, R13, R13 ADD R9, R13, R13 - MOVD 40(RSP), R16 + MOVD 72(RSP), R16 CMP R16, R13 BHI error_not_enough_space // Copy literals - TST R0, R0 - BEQ check_offset - MOVD $0, R13 + FMOVQ (R10), F0 + FMOVQ F0, (R9) + CMP $0x10, R1 + BLS copy_1_end + MOVD $0x00000010, R13 copy_1: ADD R13, R10, R15 @@ -1674,33 +1595,34 @@ copy_1: ADD R13, R9, R15 FMOVQ F0, (R15) ADD $0x10, R13, R13 - CMP R0, R13 + CMP R1, R13 BLO copy_1 - ADD R0, R10, R10 - ADD R0, R9, R9 - ADD R0, R11, R11 + +copy_1_end: + ADD R1, R10, R10 + ADD R1, R9, R9 + ADD R1, R11, R11 // Malformed input if seq.mo > t+len(hist) || seq.mo > s.windowSize) -check_offset: - MOVD R11, R0 - MOVD 48(RSP), R16 - ADD R16, R0, R0 - CMP R0, R1 + MOVD R11, R1 + MOVD 80(RSP), R16 + ADD R16, R1, R1 + CMP R1, R12 BGT error_match_off_too_big - MOVD 64(RSP), R16 - CMP R16, R1 + MOVD 96(RSP), R16 + CMP R16, R12 BGT error_match_off_too_big // Copy match from history - MOVD R1, R0 - SUBS R11, R0, R0 + MOVD R12, R1 + SUBS R11, R1, R1 BLS copy_match - MOVD 56(RSP), R13 - SUB R0, R13, R13 - CMP R0, R12 + MOVD 88(RSP), R13 + SUB R1, R13, R13 + CMP R1, R0 BGT copy_all_from_history - MOVD R12, R0 - SUBS $0x10, R0, R0 + MOVD R0, R1 + SUBS $0x10, R1, R1 BLO copy_4_small copy_4_loop: @@ -1708,61 +1630,61 @@ copy_4_loop: FMOVQ F0, (R9) ADD $0x10, R13, R13 ADD $0x10, R9, R9 - SUBS $0x10, R0, R0 + SUBS $0x10, R1, R1 BHS copy_4_loop - ADD R0, R13, R13 + ADD R1, R13, R13 ADD $16, R13, R13 - ADD R0, R9, R9 + ADD R1, R9, R9 ADD $16, R9, R9 FMOVQ -16(R13), F0 FMOVQ F0, -16(R9) JMP copy_4_end copy_4_small: - CMP $0x03, R12 + CMP $0x03, R0 BEQ copy_4_move_3 - CMP $0x08, R12 + CMP $0x08, R0 BLO copy_4_move_4through7 JMP copy_4_move_8through16 copy_4_move_3: - MOVHU (R13), R0 + MOVHU (R13), R1 MOVBU 2(R13), R16 - BFI $0, R16, $8, R1 - MOVH R0, (R9) - MOVB R1, 2(R9) - ADD R12, R13, R13 - ADD R12, R9, R9 + BFI $0, R16, $8, R12 + MOVH R1, (R9) + MOVB R12, 2(R9) + ADD R0, R13, R13 + ADD R0, R9, R9 JMP copy_4_end copy_4_move_4through7: - MOVWU (R13), R0 - ADD R12, R13, R15 - MOVWU -4(R15), R1 - MOVW R0, (R9) - ADD R12, R9, R15 - MOVW R1, -4(R15) - ADD R12, R13, R13 - ADD R12, R9, R9 + MOVWU (R13), R1 + ADD R0, R13, R15 + MOVWU -4(R15), R12 + MOVW R1, (R9) + ADD R0, R9, R15 + MOVW R12, -4(R15) + ADD R0, R13, R13 + ADD R0, R9, R9 JMP copy_4_end copy_4_move_8through16: - MOVD (R13), R0 - ADD R12, R13, R15 - MOVD -8(R15), R1 - MOVD R0, (R9) - ADD R12, R9, R15 - MOVD R1, -8(R15) - ADD R12, R13, R13 - ADD R12, R9, R9 + MOVD (R13), R1 + ADD R0, R13, R15 + MOVD -8(R15), R12 + MOVD R1, (R9) + ADD R0, R9, R15 + MOVD R12, -8(R15) + ADD R0, R13, R13 + ADD R0, R9, R9 copy_4_end: - ADD R12, R11, R11 + ADD R0, R11, R11 JMP handle_loop JMP loop_finished copy_all_from_history: - MOVD R0, R14 + MOVD R1, R14 SUBS $0x10, R14, R14 BLO copy_5_small @@ -1782,24 +1704,24 @@ copy_5_loop: JMP copy_5_end copy_5_small: - CMP $0x03, R0 + CMP $0x03, R1 BEQ copy_5_move_3 BLO copy_5_move_1or2 - CMP $0x08, R0 + CMP $0x08, R1 BLO copy_5_move_4through7 JMP copy_5_move_8through16 copy_5_move_1or2: MOVBU (R13), R16 BFI $0, R16, $8, R14 - ADD R0, R13, R15 + ADD R1, R13, R15 MOVBU -1(R15), R16 BFI $0, R16, $8, R4 MOVB R14, (R9) - ADD R0, R9, R15 + ADD R1, R9, R15 MOVB R4, -1(R15) - ADD R0, R13, R13 - ADD R0, R9, R9 + ADD R1, R13, R13 + ADD R1, R9, R9 JMP copy_5_end copy_5_move_3: @@ -1808,75 +1730,84 @@ copy_5_move_3: BFI $0, R16, $8, R4 MOVH R14, (R9) MOVB R4, 2(R9) - ADD R0, R13, R13 - ADD R0, R9, R9 + ADD R1, R13, R13 + ADD R1, R9, R9 JMP copy_5_end copy_5_move_4through7: MOVWU (R13), R14 - ADD R0, R13, R15 + ADD R1, R13, R15 MOVWU -4(R15), R4 MOVW R14, (R9) - ADD R0, R9, R15 + ADD R1, R9, R15 MOVW R4, -4(R15) - ADD R0, R13, R13 - ADD R0, R9, R9 + ADD R1, R13, R13 + ADD R1, R9, R9 JMP copy_5_end copy_5_move_8through16: MOVD (R13), R14 - ADD R0, R13, R15 + ADD R1, R13, R15 MOVD -8(R15), R4 MOVD R14, (R9) - ADD R0, R9, R15 + ADD R1, R9, R15 MOVD R4, -8(R15) - ADD R0, R13, R13 - ADD R0, R9, R9 + ADD R1, R13, R13 + ADD R1, R9, R9 copy_5_end: - ADD R0, R11, R11 - SUB R0, R12, R12 + ADD R1, R11, R11 + SUB R1, R0, R0 // Copy match from the current buffer copy_match: - MOVD R9, R0 - SUB R1, R0, R0 + MOVD R9, R1 + SUB R12, R1, R1 // ml <= mo - CMP R1, R12 + CMP R12, R0 BHI copy_overlapping_match // Copy non-overlapping match - ADD R12, R11, R11 - MOVD R9, R1 - ADD R12, R9, R9 + ADD R0, R11, R11 + FMOVQ (R1), F0 + FMOVQ F0, (R9) + CMP $0x10, R0 + BHI copy_2_long + ADD R0, R9, R9 + JMP handle_loop + +copy_2_long: + ADD $16, R9, R12 + ADD R0, R9, R9 + ADD $0x10, R1, R1 + SUB $0x10, R0, R0 copy_2: - FMOVQ (R0), F0 - FMOVQ F0, (R1) - ADD $0x10, R0, R0 + FMOVQ (R1), F0 + FMOVQ F0, (R12) ADD $0x10, R1, R1 - SUBS $0x10, R12, R12 + ADD $0x10, R12, R12 + SUBS $0x10, R0, R0 BHI copy_2 JMP handle_loop // Copy overlapping match copy_overlapping_match: - ADD R12, R11, R11 + ADD R0, R11, R11 copy_slow_3: - MOVBU (R0), R1 - MOVB R1, (R9) - ADD $1, R0, R0 + MOVBU (R1), R12 + MOVB R12, (R9) + ADD $1, R1, R1 ADD $1, R9, R9 - SUBS $1, R12, R12 + SUBS $1, R0, R0 BNE copy_slow_3 handle_loop: - MOVD ctx+16(FP), R0 - MOVD 96(R0), R16 + MOVD 40(RSP), R16 SUBS $1, R16, R16 - MOVD R16, 96(R0) + MOVD R16, 40(RSP) BPL sequenceDecs_decodeSync_amd64_main_loop loop_finished: @@ -1885,6 +1816,16 @@ loop_finished: MOVB R3, 40(R0) MOVD R5, 32(R0) + // s.seqSize += outPosition - ctx.outPosition + MOVD ctx+16(FP), R0 + MOVD s+0(FP), R1 + MOVD R11, R2 + MOVD 136(R0), R16 + SUB R16, R2, R2 + MOVD 256(R1), R16 + ADD R2, R16, R16 + MOVD R16, 256(R1) + // Update the context MOVD ctx+16(FP), R0 MOVD R11, 136(R0) @@ -1899,7 +1840,7 @@ loop_finished: // Return with match length error sequenceDecs_decodeSync_amd64_error_match_len_ofs_mismatch: - MOVD 24(RSP), R0 + MOVD 56(RSP), R0 MOVD ctx+16(FP), R1 MOVD R0, 216(R1) MOVD $0x00000001, R16 @@ -1909,7 +1850,7 @@ sequenceDecs_decodeSync_amd64_error_match_len_ofs_mismatch: // Return with match too long error sequenceDecs_decodeSync_amd64_error_match_len_too_big: MOVD ctx+16(FP), R0 - MOVD 24(RSP), R1 + MOVD 56(RSP), R1 MOVD R1, 216(R0) MOVD $0x00000002, R16 MOVD R16, ret+24(FP) @@ -1918,7 +1859,7 @@ sequenceDecs_decodeSync_amd64_error_match_len_too_big: // Return with match offset too long error error_match_off_too_big: MOVD ctx+16(FP), R0 - MOVD 16(RSP), R1 + MOVD 48(RSP), R1 MOVD R1, 224(R0) MOVD R11, 136(R0) MOVD $0x00000003, R16 @@ -1928,7 +1869,7 @@ error_match_off_too_big: // Return with not enough literals error error_not_enough_literals: MOVD ctx+16(FP), R0 - MOVD 32(RSP), R1 + MOVD 64(RSP), R1 MOVD R1, 208(R0) MOVD $0x00000004, R16 MOVD R16, ret+24(FP) @@ -1943,9 +1884,9 @@ error_overread: // Return with not enough output space error error_not_enough_space: MOVD ctx+16(FP), R0 - MOVD 32(RSP), R1 + MOVD 64(RSP), R1 MOVD R1, 208(R0) - MOVD 24(RSP), R1 + MOVD 56(RSP), R1 MOVD R1, 216(R0) MOVD R11, 136(R0) MOVD $0x00000005, R16 @@ -1956,7 +1897,7 @@ error_not_enough_space: // func sequenceDecs_decodeSync_safe_amd64(s *sequenceDecs, br *bitReader, ctx *decodeSyncAsmContext) int // Requires: CMOV, SSE -TEXT ·sequenceDecs_decodeSync_safe_arm64(SB), $64-32 +TEXT ·sequenceDecs_decodeSync_safe_arm64(SB), $96-32 MOVD br+8(FP), R1 MOVD 24(R1), R2 MOVBU 40(R1), R3 @@ -1968,30 +1909,38 @@ TEXT ·sequenceDecs_decodeSync_safe_arm64(SB), $64-32 MOVD 72(R0), R6 MOVD 80(R0), R7 MOVD 88(R0), R8 - MOVD $0, R1 + MOVD (R0), R1 MOVD R1, 16(RSP) + MOVD 24(R0), R1 MOVD R1, 24(RSP) + MOVD 48(R0), R1 MOVD R1, 32(RSP) + MOVD 96(R0), R1 + MOVD R1, 40(RSP) + MOVD $0, R1 + MOVD R1, 48(RSP) + MOVD R1, 56(RSP) + MOVD R1, 64(RSP) MOVD 112(R0), R9 MOVD 128(R0), R1 - MOVD R1, 40(RSP) + MOVD R1, 72(RSP) MOVD 144(R0), R10 MOVD 136(R0), R11 MOVD 200(R0), R1 - MOVD R1, 64(RSP) + MOVD R1, 96(RSP) MOVD 176(R0), R1 - MOVD R1, 56(RSP) + MOVD R1, 88(RSP) MOVD 184(R0), R0 - MOVD R0, 48(RSP) - MOVD 48(RSP), R0 - MOVD 56(RSP), R16 + MOVD R0, 80(RSP) + MOVD 80(RSP), R0 + MOVD 88(RSP), R16 ADD R0, R16, R16 - MOVD R16, 56(RSP) + MOVD R16, 88(RSP) // Calculate pointer to s.out[cap(s.out)] (a past-end pointer) - MOVD 40(RSP), R16 + MOVD 72(RSP), R16 ADD R9, R16, R16 - MOVD R16, 40(RSP) + MOVD R16, 72(RSP) // outBase += outPosition ADD R11, R9, R9 @@ -2002,8 +1951,7 @@ sequenceDecs_decodeSync_safe_amd64_main_loop: // Fill bitreader to have enough for the offset and match length. CMP $0x08, R5 BLT sequenceDecs_decodeSync_safe_amd64_fill_byte_by_byte - MOVD R3, R0 - LSR $0x03, R0, R0 + LSR $0x03, R3, R0 SUB R0, R12, R12 MOVD (R12), R2 SUB R0, R5, R5 @@ -2030,51 +1978,32 @@ sequenceDecs_decodeSync_safe_amd64_fill_check_overread: sequenceDecs_decodeSync_safe_amd64_fill_end: // Update offset MOVD R8, R0 - MOVD R3, R1 - MOVD R2, R13 - LSL R1, R13, R13 + LSL R3, R2, R13 + LSR $0x01, R13, R13 UBFX $8, R0, $8, R1 LSR $0x20, R0, R0 - TST R1, R1 - BEQ sequenceDecs_decodeSync_safe_amd64_of_update_zero ADD R1, R3, R3 - CMP $0x40, R3 - BHI sequenceDecs_decodeSync_safe_amd64_of_update_zero - CMP $0x40, R1 - BHS sequenceDecs_decodeSync_safe_amd64_of_update_zero - NEG R1, R1 + EOR $0x3f, R1, R1 LSR R1, R13, R13 ADD R13, R0, R0 - -sequenceDecs_decodeSync_safe_amd64_of_update_zero: - MOVD R0, 16(RSP) + MOVD R0, 48(RSP) // Update match length MOVD R7, R0 - MOVD R3, R1 - MOVD R2, R13 - LSL R1, R13, R13 + LSL R3, R2, R13 + LSR $0x01, R13, R13 UBFX $8, R0, $8, R1 LSR $0x20, R0, R0 - TST R1, R1 - BEQ sequenceDecs_decodeSync_safe_amd64_ml_update_zero ADD R1, R3, R3 - CMP $0x40, R3 - BHI sequenceDecs_decodeSync_safe_amd64_ml_update_zero - CMP $0x40, R1 - BHS sequenceDecs_decodeSync_safe_amd64_ml_update_zero - NEG R1, R1 + EOR $0x3f, R1, R1 LSR R1, R13, R13 ADD R13, R0, R0 - -sequenceDecs_decodeSync_safe_amd64_ml_update_zero: - MOVD R0, 24(RSP) + MOVD R0, 56(RSP) // Fill bitreader to have enough for the remaining CMP $0x08, R5 BLT sequenceDecs_decodeSync_safe_amd64_fill_2_byte_by_byte - MOVD R3, R0 - LSR $0x03, R0, R0 + LSR $0x03, R3, R0 SUB R0, R12, R12 MOVD (R12), R2 SUB R0, R5, R5 @@ -2101,97 +2030,69 @@ sequenceDecs_decodeSync_safe_amd64_fill_2_check_overread: sequenceDecs_decodeSync_safe_amd64_fill_2_end: // Update literal length MOVD R6, R0 - MOVD R3, R1 - MOVD R2, R13 - LSL R1, R13, R13 + LSL R3, R2, R13 + LSR $0x01, R13, R13 UBFX $8, R0, $8, R1 LSR $0x20, R0, R0 - TST R1, R1 - BEQ sequenceDecs_decodeSync_safe_amd64_ll_update_zero ADD R1, R3, R3 - CMP $0x40, R3 - BHI sequenceDecs_decodeSync_safe_amd64_ll_update_zero - CMP $0x40, R1 - BHS sequenceDecs_decodeSync_safe_amd64_ll_update_zero - NEG R1, R1 + EOR $0x3f, R1, R1 LSR R1, R13, R13 ADD R13, R0, R0 - -sequenceDecs_decodeSync_safe_amd64_ll_update_zero: - MOVD R0, 32(RSP) + MOVD R0, 64(RSP) // Fill bitreader for state updates MOVD R12, 8(RSP) UBFX $8, R8, $8, R0 - MOVD ctx+16(FP), R1 - MOVD 96(R1), R16 + MOVD 40(RSP), R16 CMP $0x00, R16 BEQ sequenceDecs_decodeSync_safe_amd64_skip_update // Update Literal Length State MOVBU R6, R12 LSRW $0x10, R6, R6 - ADD R12, R3, R1 - MOVD R2, R13 - MOVD R1, R3 - NEG R1, R16 - ROR R16, R13, R13 - MOVD $0x00000001, R14 - MOVBU R12, R1 - LSLW R1, R14, R14 - SUBW $1, R14, R14 - AND R14, R13, R13 + LSL R3, R2, R13 + LSR $0x01, R13, R13 + ADD R12, R3, R3 + EOR $0x3f, R12, R12 + LSR R12, R13, R13 ADD R13, R6, R6 // Load ctx.llTable - MOVD ctx+16(FP), R1 - MOVD (R1), R1 + MOVD 16(RSP), R1 MOVD (R1)(R6<<3), R6 // Update Match Length State MOVBU R7, R12 LSRW $0x10, R7, R7 - ADD R12, R3, R1 - MOVD R2, R13 - MOVD R1, R3 - NEG R1, R16 - ROR R16, R13, R13 - MOVD $0x00000001, R14 - MOVBU R12, R1 - LSLW R1, R14, R14 - SUBW $1, R14, R14 - AND R14, R13, R13 + LSL R3, R2, R13 + LSR $0x01, R13, R13 + ADD R12, R3, R3 + EOR $0x3f, R12, R12 + LSR R12, R13, R13 ADD R13, R7, R7 // Load ctx.mlTable - MOVD ctx+16(FP), R1 - MOVD 24(R1), R1 + MOVD 24(RSP), R1 MOVD (R1)(R7<<3), R7 // Update Offset State MOVBU R8, R12 LSRW $0x10, R8, R8 - ADD R12, R3, R1 - MOVD R2, R13 - MOVD R1, R3 - NEG R1, R16 - ROR R16, R13, R13 - MOVD $0x00000001, R14 - MOVBU R12, R1 - LSLW R1, R14, R14 - SUBW $1, R14, R14 - AND R14, R13, R13 + LSL R3, R2, R13 + LSR $0x01, R13, R13 + ADD R12, R3, R3 + EOR $0x3f, R12, R12 + LSR R12, R13, R13 ADD R13, R8, R8 // Load ctx.ofTable - MOVD ctx+16(FP), R1 - MOVD 48(R1), R1 + MOVD 32(RSP), R1 MOVD (R1)(R8<<3), R8 sequenceDecs_decodeSync_safe_amd64_skip_update: // Adjust offset MOVD s+0(FP), R1 - MOVD 16(RSP), R12 + MOVD 48(RSP), R12 CMP $0x01, R0 BLS sequenceDecs_decodeSync_safe_amd64_adjust_offsetB_1_or_0 FMOVQ 144(R1), F0 @@ -2200,7 +2101,7 @@ sequenceDecs_decodeSync_safe_amd64_skip_update: JMP sequenceDecs_decodeSync_safe_amd64_after_adjust sequenceDecs_decodeSync_safe_amd64_adjust_offsetB_1_or_0: - MOVD 32(RSP), R16 + MOVD 64(RSP), R16 CMP $0x00000000, R16 BNE sequenceDecs_decodeSync_safe_amd64_adjust_offset_maybezero ADD $1, R12, R12 @@ -2238,16 +2139,11 @@ sequenceDecs_decodeSync_safe_amd64_adjust_skip: MOVD R13, R12 sequenceDecs_decodeSync_safe_amd64_after_adjust: - MOVD R12, 16(RSP) + MOVD R12, 48(RSP) // Check values - MOVD 24(RSP), R0 - MOVD 32(RSP), R1 - ADD R1, R0, R13 - MOVD s+0(FP), R14 - MOVD 256(R14), R16 - ADD R13, R16, R16 - MOVD R16, 256(R14) + MOVD 56(RSP), R0 + MOVD 64(RSP), R1 MOVD ctx+16(FP), R13 MOVD 104(R13), R16 SUBS R1, R16, R16 @@ -2261,21 +2157,17 @@ sequenceDecs_decodeSync_safe_amd64_after_adjust: BNE sequenceDecs_decodeSync_safe_amd64_error_match_len_ofs_mismatch sequenceDecs_decodeSync_safe_amd64_match_len_ofs_ok: - MOVD 32(RSP), R0 - MOVD 16(RSP), R1 - MOVD 24(RSP), R12 - // Check if we have enough space in s.out - ADD R12, R0, R13 + ADD R0, R1, R13 ADD R9, R13, R13 - MOVD 40(RSP), R16 + MOVD 72(RSP), R16 CMP R16, R13 BHI error_not_enough_space // Copy literals - TST R0, R0 + TST R1, R1 BEQ check_offset - MOVD R0, R13 + MOVD R1, R13 SUBS $0x10, R13, R13 BLO copy_1_small @@ -2295,24 +2187,24 @@ copy_1_loop: JMP copy_1_end copy_1_small: - CMP $0x03, R0 + CMP $0x03, R1 BEQ copy_1_move_3 BLO copy_1_move_1or2 - CMP $0x08, R0 + CMP $0x08, R1 BLO copy_1_move_4through7 JMP copy_1_move_8through16 copy_1_move_1or2: MOVBU (R10), R16 BFI $0, R16, $8, R13 - ADD R0, R10, R15 + ADD R1, R10, R15 MOVBU -1(R15), R16 BFI $0, R16, $8, R14 MOVB R13, (R9) - ADD R0, R9, R15 + ADD R1, R9, R15 MOVB R14, -1(R15) - ADD R0, R10, R10 - ADD R0, R9, R9 + ADD R1, R10, R10 + ADD R1, R9, R9 JMP copy_1_end copy_1_move_3: @@ -2321,55 +2213,55 @@ copy_1_move_3: BFI $0, R16, $8, R14 MOVH R13, (R9) MOVB R14, 2(R9) - ADD R0, R10, R10 - ADD R0, R9, R9 + ADD R1, R10, R10 + ADD R1, R9, R9 JMP copy_1_end copy_1_move_4through7: MOVWU (R10), R13 - ADD R0, R10, R15 + ADD R1, R10, R15 MOVWU -4(R15), R14 MOVW R13, (R9) - ADD R0, R9, R15 + ADD R1, R9, R15 MOVW R14, -4(R15) - ADD R0, R10, R10 - ADD R0, R9, R9 + ADD R1, R10, R10 + ADD R1, R9, R9 JMP copy_1_end copy_1_move_8through16: MOVD (R10), R13 - ADD R0, R10, R15 + ADD R1, R10, R15 MOVD -8(R15), R14 MOVD R13, (R9) - ADD R0, R9, R15 + ADD R1, R9, R15 MOVD R14, -8(R15) - ADD R0, R10, R10 - ADD R0, R9, R9 + ADD R1, R10, R10 + ADD R1, R9, R9 copy_1_end: - ADD R0, R11, R11 + ADD R1, R11, R11 // Malformed input if seq.mo > t+len(hist) || seq.mo > s.windowSize) check_offset: - MOVD R11, R0 - MOVD 48(RSP), R16 - ADD R16, R0, R0 - CMP R0, R1 + MOVD R11, R1 + MOVD 80(RSP), R16 + ADD R16, R1, R1 + CMP R1, R12 BGT error_match_off_too_big - MOVD 64(RSP), R16 - CMP R16, R1 + MOVD 96(RSP), R16 + CMP R16, R12 BGT error_match_off_too_big // Copy match from history - MOVD R1, R0 - SUBS R11, R0, R0 + MOVD R12, R1 + SUBS R11, R1, R1 BLS copy_match - MOVD 56(RSP), R13 - SUB R0, R13, R13 - CMP R0, R12 + MOVD 88(RSP), R13 + SUB R1, R13, R13 + CMP R1, R0 BGT copy_all_from_history - MOVD R12, R0 - SUBS $0x10, R0, R0 + MOVD R0, R1 + SUBS $0x10, R1, R1 BLO copy_4_small copy_4_loop: @@ -2377,61 +2269,61 @@ copy_4_loop: FMOVQ F0, (R9) ADD $0x10, R13, R13 ADD $0x10, R9, R9 - SUBS $0x10, R0, R0 + SUBS $0x10, R1, R1 BHS copy_4_loop - ADD R0, R13, R13 + ADD R1, R13, R13 ADD $16, R13, R13 - ADD R0, R9, R9 + ADD R1, R9, R9 ADD $16, R9, R9 FMOVQ -16(R13), F0 FMOVQ F0, -16(R9) JMP copy_4_end copy_4_small: - CMP $0x03, R12 + CMP $0x03, R0 BEQ copy_4_move_3 - CMP $0x08, R12 + CMP $0x08, R0 BLO copy_4_move_4through7 JMP copy_4_move_8through16 copy_4_move_3: - MOVHU (R13), R0 + MOVHU (R13), R1 MOVBU 2(R13), R16 - BFI $0, R16, $8, R1 - MOVH R0, (R9) - MOVB R1, 2(R9) - ADD R12, R13, R13 - ADD R12, R9, R9 + BFI $0, R16, $8, R12 + MOVH R1, (R9) + MOVB R12, 2(R9) + ADD R0, R13, R13 + ADD R0, R9, R9 JMP copy_4_end copy_4_move_4through7: - MOVWU (R13), R0 - ADD R12, R13, R15 - MOVWU -4(R15), R1 - MOVW R0, (R9) - ADD R12, R9, R15 - MOVW R1, -4(R15) - ADD R12, R13, R13 - ADD R12, R9, R9 + MOVWU (R13), R1 + ADD R0, R13, R15 + MOVWU -4(R15), R12 + MOVW R1, (R9) + ADD R0, R9, R15 + MOVW R12, -4(R15) + ADD R0, R13, R13 + ADD R0, R9, R9 JMP copy_4_end copy_4_move_8through16: - MOVD (R13), R0 - ADD R12, R13, R15 - MOVD -8(R15), R1 - MOVD R0, (R9) - ADD R12, R9, R15 - MOVD R1, -8(R15) - ADD R12, R13, R13 - ADD R12, R9, R9 + MOVD (R13), R1 + ADD R0, R13, R15 + MOVD -8(R15), R12 + MOVD R1, (R9) + ADD R0, R9, R15 + MOVD R12, -8(R15) + ADD R0, R13, R13 + ADD R0, R9, R9 copy_4_end: - ADD R12, R11, R11 + ADD R0, R11, R11 JMP handle_loop JMP loop_finished copy_all_from_history: - MOVD R0, R14 + MOVD R1, R14 SUBS $0x10, R14, R14 BLO copy_5_small @@ -2451,24 +2343,24 @@ copy_5_loop: JMP copy_5_end copy_5_small: - CMP $0x03, R0 + CMP $0x03, R1 BEQ copy_5_move_3 BLO copy_5_move_1or2 - CMP $0x08, R0 + CMP $0x08, R1 BLO copy_5_move_4through7 JMP copy_5_move_8through16 copy_5_move_1or2: MOVBU (R13), R16 BFI $0, R16, $8, R14 - ADD R0, R13, R15 + ADD R1, R13, R15 MOVBU -1(R15), R16 BFI $0, R16, $8, R4 MOVB R14, (R9) - ADD R0, R9, R15 + ADD R1, R9, R15 MOVB R4, -1(R15) - ADD R0, R13, R13 - ADD R0, R9, R9 + ADD R1, R13, R13 + ADD R1, R9, R9 JMP copy_5_end copy_5_move_3: @@ -2477,137 +2369,136 @@ copy_5_move_3: BFI $0, R16, $8, R4 MOVH R14, (R9) MOVB R4, 2(R9) - ADD R0, R13, R13 - ADD R0, R9, R9 + ADD R1, R13, R13 + ADD R1, R9, R9 JMP copy_5_end copy_5_move_4through7: MOVWU (R13), R14 - ADD R0, R13, R15 + ADD R1, R13, R15 MOVWU -4(R15), R4 MOVW R14, (R9) - ADD R0, R9, R15 + ADD R1, R9, R15 MOVW R4, -4(R15) - ADD R0, R13, R13 - ADD R0, R9, R9 + ADD R1, R13, R13 + ADD R1, R9, R9 JMP copy_5_end copy_5_move_8through16: MOVD (R13), R14 - ADD R0, R13, R15 + ADD R1, R13, R15 MOVD -8(R15), R4 MOVD R14, (R9) - ADD R0, R9, R15 + ADD R1, R9, R15 MOVD R4, -8(R15) - ADD R0, R13, R13 - ADD R0, R9, R9 + ADD R1, R13, R13 + ADD R1, R9, R9 copy_5_end: - ADD R0, R11, R11 - SUB R0, R12, R12 + ADD R1, R11, R11 + SUB R1, R0, R0 // Copy match from the current buffer copy_match: - MOVD R9, R0 - SUB R1, R0, R0 + MOVD R9, R1 + SUB R12, R1, R1 // ml <= mo - CMP R1, R12 + CMP R12, R0 BHI copy_overlapping_match // Copy non-overlapping match - ADD R12, R11, R11 - MOVD R12, R1 - SUBS $0x10, R1, R1 + ADD R0, R11, R11 + MOVD R0, R12 + SUBS $0x10, R12, R12 BLO copy_2_small copy_2_loop: - FMOVQ (R0), F0 + FMOVQ (R1), F0 FMOVQ F0, (R9) - ADD $0x10, R0, R0 + ADD $0x10, R1, R1 ADD $0x10, R9, R9 - SUBS $0x10, R1, R1 + SUBS $0x10, R12, R12 BHS copy_2_loop - ADD R1, R0, R0 - ADD $16, R0, R0 - ADD R1, R9, R9 + ADD R12, R1, R1 + ADD $16, R1, R1 + ADD R12, R9, R9 ADD $16, R9, R9 - FMOVQ -16(R0), F0 + FMOVQ -16(R1), F0 FMOVQ F0, -16(R9) JMP copy_2_end copy_2_small: - CMP $0x03, R12 + CMP $0x03, R0 BEQ copy_2_move_3 BLO copy_2_move_1or2 - CMP $0x08, R12 + CMP $0x08, R0 BLO copy_2_move_4through7 JMP copy_2_move_8through16 copy_2_move_1or2: - MOVBU (R0), R16 - BFI $0, R16, $8, R1 - ADD R12, R0, R15 + MOVBU (R1), R16 + BFI $0, R16, $8, R12 + ADD R0, R1, R15 MOVBU -1(R15), R16 BFI $0, R16, $8, R13 - MOVB R1, (R9) - ADD R12, R9, R15 + MOVB R12, (R9) + ADD R0, R9, R15 MOVB R13, -1(R15) - ADD R12, R0, R0 - ADD R12, R9, R9 + ADD R0, R1, R1 + ADD R0, R9, R9 JMP copy_2_end copy_2_move_3: - MOVHU (R0), R1 - MOVBU 2(R0), R16 + MOVHU (R1), R12 + MOVBU 2(R1), R16 BFI $0, R16, $8, R13 - MOVH R1, (R9) + MOVH R12, (R9) MOVB R13, 2(R9) - ADD R12, R0, R0 - ADD R12, R9, R9 + ADD R0, R1, R1 + ADD R0, R9, R9 JMP copy_2_end copy_2_move_4through7: - MOVWU (R0), R1 - ADD R12, R0, R15 + MOVWU (R1), R12 + ADD R0, R1, R15 MOVWU -4(R15), R13 - MOVW R1, (R9) - ADD R12, R9, R15 + MOVW R12, (R9) + ADD R0, R9, R15 MOVW R13, -4(R15) - ADD R12, R0, R0 - ADD R12, R9, R9 + ADD R0, R1, R1 + ADD R0, R9, R9 JMP copy_2_end copy_2_move_8through16: - MOVD (R0), R1 - ADD R12, R0, R15 + MOVD (R1), R12 + ADD R0, R1, R15 MOVD -8(R15), R13 - MOVD R1, (R9) - ADD R12, R9, R15 + MOVD R12, (R9) + ADD R0, R9, R15 MOVD R13, -8(R15) - ADD R12, R0, R0 - ADD R12, R9, R9 + ADD R0, R1, R1 + ADD R0, R9, R9 copy_2_end: JMP handle_loop // Copy overlapping match copy_overlapping_match: - ADD R12, R11, R11 + ADD R0, R11, R11 copy_slow_3: - MOVBU (R0), R1 - MOVB R1, (R9) - ADD $1, R0, R0 + MOVBU (R1), R12 + MOVB R12, (R9) + ADD $1, R1, R1 ADD $1, R9, R9 - SUBS $1, R12, R12 + SUBS $1, R0, R0 BNE copy_slow_3 handle_loop: - MOVD ctx+16(FP), R0 - MOVD 96(R0), R16 + MOVD 40(RSP), R16 SUBS $1, R16, R16 - MOVD R16, 96(R0) + MOVD R16, 40(RSP) BPL sequenceDecs_decodeSync_safe_amd64_main_loop loop_finished: @@ -2616,6 +2507,16 @@ loop_finished: MOVB R3, 40(R0) MOVD R5, 32(R0) + // s.seqSize += outPosition - ctx.outPosition + MOVD ctx+16(FP), R0 + MOVD s+0(FP), R1 + MOVD R11, R2 + MOVD 136(R0), R16 + SUB R16, R2, R2 + MOVD 256(R1), R16 + ADD R2, R16, R16 + MOVD R16, 256(R1) + // Update the context MOVD ctx+16(FP), R0 MOVD R11, 136(R0) @@ -2630,7 +2531,7 @@ loop_finished: // Return with match length error sequenceDecs_decodeSync_safe_amd64_error_match_len_ofs_mismatch: - MOVD 24(RSP), R0 + MOVD 56(RSP), R0 MOVD ctx+16(FP), R1 MOVD R0, 216(R1) MOVD $0x00000001, R16 @@ -2640,7 +2541,7 @@ sequenceDecs_decodeSync_safe_amd64_error_match_len_ofs_mismatch: // Return with match too long error sequenceDecs_decodeSync_safe_amd64_error_match_len_too_big: MOVD ctx+16(FP), R0 - MOVD 24(RSP), R1 + MOVD 56(RSP), R1 MOVD R1, 216(R0) MOVD $0x00000002, R16 MOVD R16, ret+24(FP) @@ -2649,7 +2550,7 @@ sequenceDecs_decodeSync_safe_amd64_error_match_len_too_big: // Return with match offset too long error error_match_off_too_big: MOVD ctx+16(FP), R0 - MOVD 16(RSP), R1 + MOVD 48(RSP), R1 MOVD R1, 224(R0) MOVD R11, 136(R0) MOVD $0x00000003, R16 @@ -2659,7 +2560,7 @@ error_match_off_too_big: // Return with not enough literals error error_not_enough_literals: MOVD ctx+16(FP), R0 - MOVD 32(RSP), R1 + MOVD 64(RSP), R1 MOVD R1, 208(R0) MOVD $0x00000004, R16 MOVD R16, ret+24(FP) @@ -2674,9 +2575,9 @@ error_overread: // Return with not enough output space error error_not_enough_space: MOVD ctx+16(FP), R0 - MOVD 32(RSP), R1 + MOVD 64(RSP), R1 MOVD R1, 208(R0) - MOVD 24(RSP), R1 + MOVD 56(RSP), R1 MOVD R1, 216(R0) MOVD R11, 136(R0) MOVD $0x00000005, R16 diff --git a/vendor/github.com/klauspost/compress/zstd/seqdec_asm.go b/vendor/github.com/klauspost/compress/zstd/seqdec_asm.go index 42ce5de71636..94559c459a8f 100644 --- a/vendor/github.com/klauspost/compress/zstd/seqdec_asm.go +++ b/vendor/github.com/klauspost/compress/zstd/seqdec_asm.go @@ -83,9 +83,10 @@ const errorOverread = 6 // When the output and literal buffers have compressedBlockOverAlloc (16) // bytes of slack past their logical use, the assembly may use extended // memory copies that read and write in 16-byte blocks, overrunning the end -// of a literal run or match by up to 15 bytes. Otherwise it must use the -// bounds-exact ("safe") copies. This mirrors the analogous, always-dynamic -// selection in executeSimple below. +// of a literal run or match by up to 15 bytes; an empty literal run still +// copies one block, reading and writing the full 16 bytes of slack past +// its position. Otherwise it must use the bounds-exact ("safe") copies. +// This mirrors the analogous, always-dynamic selection in executeSimple below. // // The unsafe copies were disabled in #644 (2022) as a mitigation for a // crash, but that crash's root cause — an unguarded bitReader overread that diff --git a/vendor/modules.txt b/vendor/modules.txt index d1e1f791be17..77d89fad8ef0 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -145,7 +145,7 @@ github.com/grpc-ecosystem/grpc-gateway/v2/utilities # github.com/inconshreveable/mousetrap v1.1.0 ## explicit; go 1.18 github.com/inconshreveable/mousetrap -# github.com/klauspost/compress v1.20.0 +# github.com/klauspost/compress v1.20.1 ## explicit; go 1.25 github.com/klauspost/compress github.com/klauspost/compress/fse