diff --git a/vendor.mod b/vendor.mod index b1bfb7897ab7..f334409c135e 100644 --- a/vendor.mod +++ b/vendor.mod @@ -87,7 +87,7 @@ require ( github.com/gorilla/mux v1.8.1 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.30.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect - github.com/klauspost/compress v1.19.2 // indirect + github.com/klauspost/compress v1.20.1 // indirect github.com/moby/docker-image-spec v1.3.1 // indirect github.com/moby/sys/user v0.4.1 // indirect github.com/moby/sys/userns v0.2.1 // indirect diff --git a/vendor.sum b/vendor.sum index 36d253337784..d1581457c7cb 100644 --- a/vendor.sum +++ b/vendor.sum @@ -78,8 +78,8 @@ github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.19.2 h1:hMRETovs/pu/dVWN7zIT1PGG8t509MwT6bO7XSi26R8= -github.com/klauspost/compress v1.19.2/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= +github.com/klauspost/compress v1.20.1 h1:T7kKElXUMXrUJ2E9QhQhxFtcK5rPyLdsGZvdbLMPdiQ= +github.com/klauspost/compress v1.20.1/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/mattn/go-runewidth v0.0.29 h1:3oGF3R/S2N9DQ3ptftzVIvg2eicmojCzlwBEmqEPDfQ= diff --git a/vendor/github.com/klauspost/compress/README.md b/vendor/github.com/klauspost/compress/README.md index 0e9f170d0120..bf31b905f81f 100644 --- a/vendor/github.com/klauspost/compress/README.md +++ b/vendor/github.com/klauspost/compress/README.md @@ -6,7 +6,9 @@ This package provides various compression algorithms. * [S2](https://github.com/klauspost/compress/tree/master/s2#s2-compression) is a high performance replacement for Snappy. * Optimized [deflate](https://godoc.org/github.com/klauspost/compress/flate) packages which can be used as a dropin replacement for [gzip](https://godoc.org/github.com/klauspost/compress/gzip), [zip](https://godoc.org/github.com/klauspost/compress/zip) and [zlib](https://godoc.org/github.com/klauspost/compress/zlib). * [snappy](https://github.com/klauspost/compress/tree/master/snappy) is a drop-in replacement for `github.com/golang/snappy` offering better compression and concurrent streams. +* [lzw](https://github.com/klauspost/compress/tree/master/lzw) is a drop-in replacement for `compress/lzw` with 1.4-4x faster decompression and 1.1-2.7x faster compression, depending on the data. * [huff0](https://github.com/klauspost/compress/tree/master/huff0) and [FSE](https://github.com/klauspost/compress/tree/master/fse) implementations for raw entropy encoding. +* [Xpress](https://github.com/klauspost/compress/tree/master/xpress) decompression of the Microsoft XPRESS (MS-XCA) plain LZ77 and LZ77+Huffman formats (the LZ77+Huffman variant is the one used in WIM images and Windows Compact OS / WOF data). * [gzhttp](https://github.com/klauspost/compress/tree/master/gzhttp) Provides client and server wrappers for handling gzipped/zstd HTTP requests efficiently. * [pgzip](https://github.com/klauspost/pgzip) is a separate package that provides a very fast parallel gzip implementation. diff --git a/vendor/github.com/klauspost/compress/huff0/bitreader.go b/vendor/github.com/klauspost/compress/huff0/bitreader.go index bfc7a523dee7..faaa418a7599 100644 --- a/vendor/github.com/klauspost/compress/huff0/bitreader.go +++ b/vendor/github.com/klauspost/compress/huff0/bitreader.go @@ -9,6 +9,7 @@ import ( "errors" "fmt" "io" + "math/bits" "github.com/klauspost/compress/internal/le" ) @@ -210,6 +211,68 @@ func (b *bitReaderShifted) remaining() uint { return b.off*8 + uint(64-b.bitsRead) } +// canUseAsm reports whether the reader has a full 8-byte window ahead of +// its read pointer, which the Decompress4X asm loops need to take over. +func (b *bitReaderShifted) canUseAsm() bool { + return b.off >= 8 +} + +// prepareForAsm establishes the invariant the Decompress4X asm loops rely +// on: value == load64(in[off:off+8]) << bitsRead with bitsRead <= 7, so +// that a full group of symbols can never shift their sentinel bit out of +// the container. init leaves bitsRead == 8 when the final byte of the +// stream is exactly 0x01; that whole byte is consumed, so the same position +// is the window one byte lower with nothing consumed. Requires canUseAsm. +func (b *bitReaderShifted) prepareForAsm() { + if b.bitsRead >= 8 { + b.off-- + b.value = le.Load64(b.in, b.off) + b.bitsRead -= 8 + } +} + +// restoreFromAsm converts the state left behind by the Decompress4X asm +// loops back to the invariant the Go code relies on: value holds the 8 +// bytes at in[off:off+8] shifted left by bitsRead, with the low bitsRead +// bits zero. +// +// The asm keeps a sentinel bit in value just below the unread bits, so its +// trailing zero count is the number of consumed bits. The sentinel is ORed +// over the lowest bit of the window, which the loop never consumes before +// re-reading memory, so the window is re-read here too rather than taken +// from value. The asm reports off as the signed distance from the start of +// the stream, and it can be negative: a reload always reads a whole 8-byte +// window, so a stream that is nearly drained ends with up to 7 bytes of the +// previous stream (or the jump table) below its start inside the window. +// Those bytes sit below the stream's own bits and count as consumed. +// Anything further below, or more bits consumed than the stream holds, is +// corruption. +func (b *bitReaderShifted) restoreFromAsm() error { + off := int(b.off) + consumed := uint(bits.TrailingZeros64(b.value)) + if off < 0 { + if off < -7 { + return errors.New("corruption detected: stream underrun") + } + consumed += uint(-off) * 8 + if consumed > 64 { + return errors.New("corruption detected: stream underrun") + } + off = 0 + } + if off+8 > len(b.in) { + return errors.New("corruption detected: stream overrun") + } + b.off = uint(off) + b.bitsRead = uint8(consumed) + if consumed >= 64 { + b.value = 0 + } else { + b.value = le.Load64(b.in, b.off) << consumed + } + return nil +} + // close the bitstream and returns an error if out-of-buffer reads occurred. func (b *bitReaderShifted) close() error { // Release reference. diff --git a/vendor/github.com/klauspost/compress/huff0/decompress_amd64.go b/vendor/github.com/klauspost/compress/huff0/decompress_amd64.go index 7035d656d991..e651f30ecbd8 100644 --- a/vendor/github.com/klauspost/compress/huff0/decompress_amd64.go +++ b/vendor/github.com/klauspost/compress/huff0/decompress_amd64.go @@ -8,36 +8,99 @@ import ( ) // decompress4x_main_loop_amd64 is an x86 assembler implementation -// of Decompress4X when tablelog > 8. +// of Decompress4X when tablelog > 8, decoding fastSymbols symbols per +// stream between bit container reloads. // //go:noescape func decompress4x_main_loop_amd64(ctx *decompress4xContext) // decompress4x_8b_main_loop_amd64 is an x86 assembler implementation -// of Decompress4X when tablelog <= 8 which decodes 4 entries -// per loop. +// of Decompress4X when tablelog <= 8, decoding fast8bSymbols symbols +// per stream between bit container reloads. // //go:noescape func decompress4x_8b_main_loop_amd64(ctx *decompress4xContext) +// decompress4x_4b_main_loop_amd64 is an x86 assembler implementation +// of Decompress4X when tablelog <= 4, decoding fast4bSymbols symbols +// per stream between bit container reloads. +// +//go:noescape +func decompress4x_4b_main_loop_amd64(ctx *decompress4xContext) + +// decompress4x_4b_main_loop_bmi2 is the BMI2 twin of decompress4x_4b_main_loop_amd64. +// +//go:noescape +func decompress4x_4b_main_loop_bmi2(ctx *decompress4xContext) + +// decompress4x_main_loop_bmi2 is the BMI2 twin of decompress4x_main_loop_amd64. +// +//go:noescape +func decompress4x_main_loop_bmi2(ctx *decompress4xContext) + +// decompress4x_8b_main_loop_bmi2 is the BMI2 twin of decompress4x_8b_main_loop_amd64. +// +//go:noescape +func decompress4x_8b_main_loop_bmi2(ctx *decompress4xContext) + // decompress1x_main_loop_amd64 is an x86 assembler implementation -// of Decompress1X when tablelog > 8. +// of Decompress1X when tablelog > 8, decoding fastSymbols symbols +// between bit container reloads. // //go:noescape func decompress1x_main_loop_amd64(ctx *decompress1xContext) -// decompress1x_main_loop_bmi2 is an x86 with BMI2 assembler implementation -// of Decompress1X when tablelog > 8. +// decompress1x_8b_main_loop_amd64 is an x86 assembler implementation +// of Decompress1X when tablelog <= 8, decoding fast8bSymbols symbols +// between bit container reloads. +// +//go:noescape +func decompress1x_8b_main_loop_amd64(ctx *decompress1xContext) + +// decompress1x_4b_main_loop_amd64 is an x86 assembler implementation +// of Decompress1X when tablelog <= 4, decoding fast4bSymbols symbols +// between bit container reloads. +// +//go:noescape +func decompress1x_4b_main_loop_amd64(ctx *decompress1xContext) + +// decompress1x_main_loop_bmi2 is the BMI2 twin of decompress1x_main_loop_amd64. // //go:noescape func decompress1x_main_loop_bmi2(ctx *decompress1xContext) +// decompress1x_8b_main_loop_bmi2 is the BMI2 twin of decompress1x_8b_main_loop_amd64. +// +//go:noescape +func decompress1x_8b_main_loop_bmi2(ctx *decompress1xContext) + +// decompress1x_4b_main_loop_bmi2 is the BMI2 twin of decompress1x_4b_main_loop_amd64. +// +//go:noescape +func decompress1x_4b_main_loop_bmi2(ctx *decompress1xContext) + func decompress4x_main_loop_asm(ctx *decompress4xContext) { - decompress4x_main_loop_amd64(ctx) + if cpuinfo.HasBMI2() { + decompress4x_main_loop_bmi2(ctx) + } else { + decompress4x_main_loop_amd64(ctx) + } } func decompress4x_8b_main_loop_asm(ctx *decompress4xContext) { - decompress4x_8b_main_loop_amd64(ctx) + if cpuinfo.HasBMI2() { + decompress4x_8b_main_loop_bmi2(ctx) + } else { + decompress4x_8b_main_loop_amd64(ctx) + } +} + +func decompress4x_4b_main_loop_asm(ctx *decompress4xContext) { + if cpuinfo.HasBMI2() { + decompress4x_4b_main_loop_bmi2(ctx) + } else { + decompress4x_4b_main_loop_amd64(ctx) + } } func decompress1x_main_loop_asm(ctx *decompress1xContext) { @@ -47,3 +110,19 @@ func decompress1x_main_loop_asm(ctx *decompress1xContext) { decompress1x_main_loop_amd64(ctx) } } + +func decompress1x_8b_main_loop_asm(ctx *decompress1xContext) { + if cpuinfo.HasBMI2() { + decompress1x_8b_main_loop_bmi2(ctx) + } else { + decompress1x_8b_main_loop_amd64(ctx) + } +} + +func decompress1x_4b_main_loop_asm(ctx *decompress1xContext) { + if cpuinfo.HasBMI2() { + decompress1x_4b_main_loop_bmi2(ctx) + } else { + decompress1x_4b_main_loop_amd64(ctx) + } +} diff --git a/vendor/github.com/klauspost/compress/huff0/decompress_amd64.s b/vendor/github.com/klauspost/compress/huff0/decompress_amd64.s index c5d4a710a093..f072c18f04ed 100644 --- a/vendor/github.com/klauspost/compress/huff0/decompress_amd64.s +++ b/vendor/github.com/klauspost/compress/huff0/decompress_amd64.s @@ -3,828 +3,3384 @@ //go:build !appengine && !noasm && gc // func decompress4x_main_loop_amd64(ctx *decompress4xContext) +// Requires: BMI, CMOV TEXT ·decompress4x_main_loop_amd64(SB), $0-8 - // Preload values - MOVQ ctx+0(FP), AX - MOVBQZX 8(AX), DI - MOVQ 16(AX), BX - MOVQ 48(AX), SI - MOVQ 24(AX), R8 - MOVQ 32(AX), R9 - MOVQ (AX), R10 - - // Main loop -main_loop: - XORL DX, DX - CMPQ BX, SI - SETGE DL - - // br0.fillFast32() - MOVQ 32(R10), R11 - MOVBQZX 40(R10), R12 - CMPQ R12, $0x20 - JBE skip_fill0 - MOVQ 24(R10), AX - SUBQ $0x20, R12 - SUBQ $0x04, AX - MOVQ (R10), R13 - - // b.value |= uint64(low) << (b.bitsRead & 63) - MOVL (AX)(R13*1), R13 - MOVQ R12, CX - SHLQ CL, R13 - MOVQ AX, 24(R10) - ORQ R13, R11 - - // exhausted += (br0.off < 4) - CMPQ AX, $0x04 - ADCB $+0, DL - -skip_fill0: - // val0 := br0.peekTopBits(peekBits) - MOVQ R11, R13 - MOVQ DI, CX - SHRQ CL, R13 - - // v0 := table[val0&mask] - MOVW (R9)(R13*2), CX - - // br0.advance(uint8(v0.entry) - MOVB CH, AL - SHLQ CL, R11 - ADDB CL, R12 - - // val1 := br0.peekTopBits(peekBits) - MOVQ DI, CX - MOVQ R11, R13 - SHRQ CL, R13 - - // v1 := table[val1&mask] - MOVW (R9)(R13*2), CX - - // br0.advance(uint8(v1.entry)) - MOVB CH, AH - SHLQ CL, R11 - ADDB CL, R12 - - // these two writes get coalesced - // out[id * dstEvery + 0] = uint8(v0.entry >> 8) - // out[id * dstEvery + 1] = uint8(v1.entry >> 8) - MOVW AX, (BX) - - // update the bitreader structure - MOVQ R11, 32(R10) - MOVB R12, 40(R10) - - // br1.fillFast32() - MOVQ 80(R10), R11 - MOVBQZX 88(R10), R12 - CMPQ R12, $0x20 - JBE skip_fill1 - MOVQ 72(R10), AX - SUBQ $0x20, R12 - SUBQ $0x04, AX - MOVQ 48(R10), R13 - - // b.value |= uint64(low) << (b.bitsRead & 63) - MOVL (AX)(R13*1), R13 - MOVQ R12, CX - SHLQ CL, R13 - MOVQ AX, 72(R10) - ORQ R13, R11 - - // exhausted += (br1.off < 4) - CMPQ AX, $0x04 - ADCB $+0, DL - -skip_fill1: - // val0 := br1.peekTopBits(peekBits) - MOVQ R11, R13 - MOVQ DI, CX - SHRQ CL, R13 - - // v0 := table[val0&mask] - MOVW (R9)(R13*2), CX - - // br1.advance(uint8(v0.entry) - MOVB CH, AL - SHLQ CL, R11 - ADDB CL, R12 - - // val1 := br1.peekTopBits(peekBits) - MOVQ DI, CX - MOVQ R11, R13 - SHRQ CL, R13 - - // v1 := table[val1&mask] - MOVW (R9)(R13*2), CX - - // br1.advance(uint8(v1.entry)) - MOVB CH, AH - SHLQ CL, R11 - ADDB CL, R12 - - // these two writes get coalesced - // out[id * dstEvery + 0] = uint8(v0.entry >> 8) - // out[id * dstEvery + 1] = uint8(v1.entry >> 8) - MOVW AX, (BX)(R8*1) - - // update the bitreader structure - MOVQ R11, 80(R10) - MOVB R12, 88(R10) - - // br2.fillFast32() - MOVQ 128(R10), R11 - MOVBQZX 136(R10), R12 - CMPQ R12, $0x20 - JBE skip_fill2 - MOVQ 120(R10), AX - SUBQ $0x20, R12 - SUBQ $0x04, AX - MOVQ 96(R10), R13 - - // b.value |= uint64(low) << (b.bitsRead & 63) - MOVL (AX)(R13*1), R13 - MOVQ R12, CX - SHLQ CL, R13 - MOVQ AX, 120(R10) - ORQ R13, R11 - - // exhausted += (br2.off < 4) - CMPQ AX, $0x04 - ADCB $+0, DL - -skip_fill2: - // val0 := br2.peekTopBits(peekBits) - MOVQ R11, R13 - MOVQ DI, CX - SHRQ CL, R13 - - // v0 := table[val0&mask] - MOVW (R9)(R13*2), CX - - // br2.advance(uint8(v0.entry) - MOVB CH, AL - SHLQ CL, R11 - ADDB CL, R12 - - // val1 := br2.peekTopBits(peekBits) - MOVQ DI, CX - MOVQ R11, R13 - SHRQ CL, R13 - - // v1 := table[val1&mask] - MOVW (R9)(R13*2), CX - - // br2.advance(uint8(v1.entry)) - MOVB CH, AH - SHLQ CL, R11 - ADDB CL, R12 - - // these two writes get coalesced - // out[id * dstEvery + 0] = uint8(v0.entry >> 8) - // out[id * dstEvery + 1] = uint8(v1.entry >> 8) - MOVW AX, (BX)(R8*2) - - // update the bitreader structure - MOVQ R11, 128(R10) - MOVB R12, 136(R10) - - // br3.fillFast32() - MOVQ 176(R10), R11 - MOVBQZX 184(R10), R12 - CMPQ R12, $0x20 - JBE skip_fill3 - MOVQ 168(R10), AX - SUBQ $0x20, R12 - SUBQ $0x04, AX - MOVQ 144(R10), R13 - - // b.value |= uint64(low) << (b.bitsRead & 63) - MOVL (AX)(R13*1), R13 - MOVQ R12, CX - SHLQ CL, R13 - MOVQ AX, 168(R10) - ORQ R13, R11 - - // exhausted += (br3.off < 4) - CMPQ AX, $0x04 - ADCB $+0, DL - -skip_fill3: - // val0 := br3.peekTopBits(peekBits) - MOVQ R11, R13 - MOVQ DI, CX - SHRQ CL, R13 - - // v0 := table[val0&mask] - MOVW (R9)(R13*2), CX - - // br3.advance(uint8(v0.entry) - MOVB CH, AL - SHLQ CL, R11 - ADDB CL, R12 + MOVQ ctx+0(FP), AX - // val1 := br3.peekTopBits(peekBits) - MOVQ DI, CX - MOVQ R11, R13 - SHRQ CL, R13 - - // v1 := table[val1&mask] - MOVW (R9)(R13*2), CX - - // br3.advance(uint8(v1.entry)) - MOVB CH, AH - SHLQ CL, R11 - ADDB CL, R12 - - // these two writes get coalesced - // out[id * dstEvery + 0] = uint8(v0.entry >> 8) - // out[id * dstEvery + 1] = uint8(v1.entry >> 8) - LEAQ (R8)(R8*2), CX - MOVW AX, (BX)(CX*1) - - // update the bitreader structure - MOVQ R11, 176(R10) - MOVB R12, 184(R10) - ADDQ $0x02, BX - TESTB DL, DL - JZ main_loop - MOVQ ctx+0(FP), AX - SUBQ 16(AX), BX - SHLQ $0x02, BX - MOVQ BX, 40(AX) + // Preload values + MOVBQZX 8(AX), DX + MOVQ 32(AX), BX + MOVQ 16(AX), SI + MOVQ 24(AX), CX + LEAQ (SI)(CX*1), R8 + LEAQ (SI)(CX*2), R10 + LEAQ (CX)(CX*2), R12 + ADDQ SI, R12 + + // Convert each bit reader to sentinel form: bits = value | 1<> 8) - // out[id * dstEvery + 1] = uint8(v1.entry >> 8) - // out[id * dstEvery + 3] = uint8(v2.entry >> 8) - // out[id * dstEvery + 4] = uint8(v3.entry >> 8) - MOVL AX, (BX) - - // update the bitreader structure - MOVQ R11, 32(R10) - MOVB R12, 40(R10) - - // br1.fillFast32() - MOVQ 80(R10), R11 - MOVBQZX 88(R10), R12 - CMPQ R12, $0x20 - JBE skip_fill1 - MOVQ 72(R10), R13 - SUBQ $0x20, R12 - SUBQ $0x04, R13 - MOVQ 48(R10), R14 - - // b.value |= uint64(low) << (b.bitsRead & 63) - MOVL (R13)(R14*1), R14 - MOVQ R12, CX - SHLQ CL, R14 - MOVQ R13, 72(R10) - ORQ R14, R11 + TZCNTQ R13, R14 + MOVQ R14, CX + ANDQ $0x07, CX + SHRQ $0x03, R14 + MOVQ 88(AX), R13 + SUBQ R14, R13 + MOVQ R13, 88(AX) + MOVQ (R13), R13 + ORQ $0x01, R13 + SHLQ CL, R13 + ADDQ $0x0e, SI + ADDQ $0x0e, R8 + ADDQ $0x0e, R10 + ADDQ $0x0e, R12 + CMPQ SI, 96(AX) + JB inner_loop + JMP outer_loop + +done: + // Hand the state back: off = ip - in (negative when the window reached below the stream start), + // value = the sentinel-form container. bitReaderShifted.restoreFromAsm normalizes both. + MOVQ (AX), CX + MOVQ 64(AX), DX + SUBQ (CX), DX + MOVQ DX, 24(CX) + MOVQ DI, 32(CX) + MOVQ 72(AX), DX + SUBQ 48(CX), DX + MOVQ DX, 72(CX) + MOVQ R9, 80(CX) + MOVQ 80(AX), DX + SUBQ 96(CX), DX + MOVQ DX, 120(CX) + MOVQ R11, 128(CX) + MOVQ 88(AX), DX + SUBQ 144(CX), DX + MOVQ DX, 168(CX) + MOVQ R13, 176(CX) + SUBQ 16(AX), SI + SHLQ $0x02, SI + MOVQ SI, 40(AX) + RET - // exhausted += (br1.off < 4) - CMPQ R13, $0x04 - ADCB $+0, DL +// func decompress4x_main_loop_bmi2(ctx *decompress4xContext) +// Requires: BMI, BMI2, CMOV +TEXT ·decompress4x_main_loop_bmi2(SB), $0-8 + MOVQ ctx+0(FP), AX -skip_fill1: - // val0 := br1.peekTopBits(peekBits) - MOVQ R11, R13 - MOVQ DI, CX - SHRQ CL, R13 + // Preload values + MOVBQZX 8(AX), CX + MOVQ 32(AX), DX + MOVQ 16(AX), BX + MOVQ 24(AX), SI + LEAQ (BX)(SI*1), DI + LEAQ (BX)(SI*2), R9 + LEAQ (SI)(SI*2), R11 + ADDQ BX, R11 + + // Convert each bit reader to sentinel form: bits = value | 1<> 8) - // out[id * dstEvery + 1] = uint8(v1.entry >> 8) - // out[id * dstEvery + 3] = uint8(v2.entry >> 8) - // out[id * dstEvery + 4] = uint8(v3.entry >> 8) - MOVL AX, (BX)(R8*1) - - // update the bitreader structure - MOVQ R11, 80(R10) - MOVB R12, 88(R10) - - // br2.fillFast32() - MOVQ 128(R10), R11 - MOVBQZX 136(R10), R12 - CMPQ R12, $0x20 - JBE skip_fill2 - MOVQ 120(R10), R13 - SUBQ $0x20, R12 - SUBQ $0x04, R13 - MOVQ 96(R10), R14 - - // b.value |= uint64(low) << (b.bitsRead & 63) - MOVL (R13)(R14*1), R14 - MOVQ R12, CX + ANDQ $0x38, R13 + MOVQ -8(R8), R14 + SHRQ $0x01, R14 + ANDQ $0x07, R11 + MOVQ R13, CX + XORQ $0x3f, CX + SHRQ CL, R14 + MOVQ R11, CX SHLQ CL, R14 - MOVQ R13, 120(R10) - ORQ R14, R11 - - // exhausted += (br2.off < 4) - CMPQ R13, $0x04 - ADCB $+0, DL - -skip_fill2: - // val0 := br2.peekTopBits(peekBits) - MOVQ R11, R13 - MOVQ DI, CX - SHRQ CL, R13 - - // v0 := table[val0&mask] - MOVW (R9)(R13*2), CX - - // br2.advance(uint8(v0.entry) - MOVB CH, AL - SHLQ CL, R11 - ADDB CL, R12 - - // val1 := br2.peekTopBits(peekBits) - MOVQ R11, R13 - MOVQ DI, CX - SHRQ CL, R13 - - // v1 := table[val0&mask] - MOVW (R9)(R13*2), CX + ORQ R14, R10 + SHRQ $0x03, R13 + SUBQ R13, R8 + ADDQ $0x05, SI + CMPQ SI, R12 + JB inner_loop + JMP outer_loop + +done: + // Hand the state back in the Go bit reader's own form: off = ip - in, value, bitsRead. + MOVQ (AX), CX + SUBQ (CX), R8 + MOVQ R8, 24(CX) + MOVQ R10, 32(CX) + MOVB R11, 40(CX) + SUBQ 16(AX), SI + MOVQ SI, 40(AX) + RET - // br2.advance(uint8(v1.entry) - MOVB CH, AH - SHLQ CL, R11 - ADDB CL, R12 - BSWAPL AX +// func decompress1x_8b_main_loop_amd64(ctx *decompress1xContext) +// Requires: CMOV +TEXT ·decompress1x_8b_main_loop_amd64(SB), $0-8 + MOVQ ctx+0(FP), AX - // val2 := br2.peekTopBits(peekBits) - MOVQ R11, R13 + // Preload values + MOVBQZX 8(AX), DX + MOVQ 32(AX), BX + MOVQ 16(AX), SI + MOVQ 24(AX), DI + ADDQ SI, DI + MOVQ 56(AX), R8 + MOVQ 48(AX), R9 + MOVQ (AX), CX + MOVQ 32(CX), R10 + MOVBQZX 40(CX), R11 + +outer_loop: + // Iterations allowed by the output: (limit - op) / 8 MOVQ DI, CX - SHRQ CL, R13 - - // v2 := table[val0&mask] - MOVW (R9)(R13*2), CX - - // br2.advance(uint8(v2.entry) - MOVB CH, AH - SHLQ CL, R11 - ADDB CL, R12 + SUBQ SI, CX + JLE done + SHRQ $0x03, CX + + // Iterations allowed by the input: a refill reads the 8 bytes below the window and + // moves it down by at most 7, so every read stays inside the stream while ip stays + // at least 8 above ilowest. + MOVQ R8, R12 + SUBQ R9, R12 + SHRQ $0x03, R12 + CMPQ R12, CX + CMOVQCS R12, CX + TESTQ CX, CX + JZ done + IMUL3Q $0x07, CX, R12 + ADDQ SI, R12 + +inner_loop: + // symbol 0 + MOVQ DX, CX + MOVQ R10, R13 + SHRQ CL, R13 + MOVWQZX (BX)(R13*2), CX + SHLQ CL, R10 + ADDQ CX, R11 + SHRQ $0x08, CX + MOVB CL, (SI) + + // symbol 1 + MOVQ DX, CX + MOVQ R10, R13 + SHRQ CL, R13 + MOVWQZX (BX)(R13*2), CX + SHLQ CL, R10 + ADDQ CX, R11 + SHRQ $0x08, CX + MOVB CL, 1(SI) + + // symbol 2 + MOVQ DX, CX + MOVQ R10, R13 + SHRQ CL, R13 + MOVWQZX (BX)(R13*2), CX + SHLQ CL, R10 + ADDQ CX, R11 + SHRQ $0x08, CX + MOVB CL, 2(SI) + + // symbol 3 + MOVQ DX, CX + MOVQ R10, R13 + SHRQ CL, R13 + MOVWQZX (BX)(R13*2), CX + SHLQ CL, R10 + ADDQ CX, R11 + SHRQ $0x08, CX + MOVB CL, 3(SI) + + // symbol 4 + MOVQ DX, CX + MOVQ R10, R13 + SHRQ CL, R13 + MOVWQZX (BX)(R13*2), CX + SHLQ CL, R10 + ADDQ CX, R11 + SHRQ $0x08, CX + MOVB CL, 4(SI) + + // symbol 5 + MOVQ DX, CX + MOVQ R10, R13 + SHRQ CL, R13 + MOVWQZX (BX)(R13*2), CX + SHLQ CL, R10 + ADDQ CX, R11 + SHRQ $0x08, CX + MOVB CL, 5(SI) + + // symbol 6 + MOVQ DX, CX + MOVQ R10, R13 + SHRQ CL, R13 + MOVWQZX (BX)(R13*2), CX + SHLQ CL, R10 + ADDQ CX, R11 + SHRQ $0x08, CX + MOVB CL, 6(SI) - // val3 := br2.peekTopBits(peekBits) + // Refill the whole bytes consumed from below the window MOVQ R11, R13 - MOVQ DI, CX - SHRQ CL, R13 - - // v3 := table[val0&mask] - MOVW (R9)(R13*2), CX - - // br2.advance(uint8(v3.entry) - MOVB CH, AL - SHLQ CL, R11 - ADDB CL, R12 - BSWAPL AX - - // these four writes get coalesced - // out[id * dstEvery + 0] = uint8(v0.entry >> 8) - // out[id * dstEvery + 1] = uint8(v1.entry >> 8) - // out[id * dstEvery + 3] = uint8(v2.entry >> 8) - // out[id * dstEvery + 4] = uint8(v3.entry >> 8) - MOVL AX, (BX)(R8*2) - - // update the bitreader structure - MOVQ R11, 128(R10) - MOVB R12, 136(R10) - - // br3.fillFast32() - MOVQ 176(R10), R11 - MOVBQZX 184(R10), R12 - CMPQ R12, $0x20 - JBE skip_fill3 - MOVQ 168(R10), R13 - SUBQ $0x20, R12 - SUBQ $0x04, R13 - MOVQ 144(R10), R14 - - // b.value |= uint64(low) << (b.bitsRead & 63) - MOVL (R13)(R14*1), R14 - MOVQ R12, CX + ANDQ $0x38, R13 + MOVQ -8(R8), R14 + SHRQ $0x01, R14 + ANDQ $0x07, R11 + MOVQ R13, CX + XORQ $0x3f, CX + SHRQ CL, R14 + MOVQ R11, CX SHLQ CL, R14 - MOVQ R13, 168(R10) - ORQ R14, R11 - - // exhausted += (br3.off < 4) - CMPQ R13, $0x04 - ADCB $+0, DL - -skip_fill3: - // val0 := br3.peekTopBits(peekBits) - MOVQ R11, R13 - MOVQ DI, CX - SHRQ CL, R13 - - // v0 := table[val0&mask] - MOVW (R9)(R13*2), CX - - // br3.advance(uint8(v0.entry) - MOVB CH, AL - SHLQ CL, R11 - ADDB CL, R12 - - // val1 := br3.peekTopBits(peekBits) - MOVQ R11, R13 - MOVQ DI, CX - SHRQ CL, R13 - - // v1 := table[val0&mask] - MOVW (R9)(R13*2), CX - - // br3.advance(uint8(v1.entry) - MOVB CH, AH - SHLQ CL, R11 - ADDB CL, R12 - BSWAPL AX - - // val2 := br3.peekTopBits(peekBits) - MOVQ R11, R13 - MOVQ DI, CX - SHRQ CL, R13 - - // v2 := table[val0&mask] - MOVW (R9)(R13*2), CX + ORQ R14, R10 + SHRQ $0x03, R13 + SUBQ R13, R8 + ADDQ $0x07, SI + CMPQ SI, R12 + JB inner_loop + JMP outer_loop + +done: + // Hand the state back in the Go bit reader's own form: off = ip - in, value, bitsRead. + MOVQ (AX), CX + SUBQ (CX), R8 + MOVQ R8, 24(CX) + MOVQ R10, 32(CX) + MOVB R11, 40(CX) + SUBQ 16(AX), SI + MOVQ SI, 40(AX) + RET - // br3.advance(uint8(v2.entry) - MOVB CH, AH - SHLQ CL, R11 - ADDB CL, R12 +// func decompress1x_4b_main_loop_amd64(ctx *decompress1xContext) +// Requires: CMOV +TEXT ·decompress1x_4b_main_loop_amd64(SB), $0-8 + MOVQ ctx+0(FP), AX - // val3 := br3.peekTopBits(peekBits) - MOVQ R11, R13 + // Preload values + MOVBQZX 8(AX), DX + MOVQ 32(AX), BX + MOVQ 16(AX), SI + MOVQ 24(AX), DI + ADDQ SI, DI + MOVQ 56(AX), R8 + MOVQ 48(AX), R9 + MOVQ (AX), CX + MOVQ 32(CX), R10 + MOVBQZX 40(CX), R11 + +outer_loop: + // Iterations allowed by the output: (limit - op) / 16 MOVQ DI, CX - SHRQ CL, R13 - - // v3 := table[val0&mask] - MOVW (R9)(R13*2), CX - - // br3.advance(uint8(v3.entry) - MOVB CH, AL - SHLQ CL, R11 - ADDB CL, R12 - BSWAPL AX - - // these four writes get coalesced - // out[id * dstEvery + 0] = uint8(v0.entry >> 8) - // out[id * dstEvery + 1] = uint8(v1.entry >> 8) - // out[id * dstEvery + 3] = uint8(v2.entry >> 8) - // out[id * dstEvery + 4] = uint8(v3.entry >> 8) - LEAQ (R8)(R8*2), CX - MOVL AX, (BX)(CX*1) - - // update the bitreader structure - MOVQ R11, 176(R10) - MOVB R12, 184(R10) - ADDQ $0x04, BX - TESTB DL, DL - JZ main_loop - MOVQ ctx+0(FP), AX - SUBQ 16(AX), BX - SHLQ $0x02, BX - MOVQ BX, 40(AX) - RET - -// func decompress1x_main_loop_amd64(ctx *decompress1xContext) -TEXT ·decompress1x_main_loop_amd64(SB), $0-8 - MOVQ ctx+0(FP), CX - MOVQ 16(CX), DX - MOVQ 24(CX), BX - CMPQ BX, $0x04 - JB error_max_decoded_size_exceeded - LEAQ (DX)(BX*1), BX - MOVQ (CX), SI - MOVQ (SI), R8 - MOVQ 24(SI), R9 - MOVQ 32(SI), R10 - MOVBQZX 40(SI), R11 - MOVQ 32(CX), SI - MOVBQZX 8(CX), DI - JMP loop_condition - -main_loop: - // Check if we have room for 4 bytes in the output buffer - LEAQ 4(DX), CX - CMPQ CX, BX - JGE error_max_decoded_size_exceeded - - // Decode 4 values - CMPQ R11, $0x20 - JL bitReader_fillFast_1_end - SUBQ $0x20, R11 - SUBQ $0x04, R9 - MOVL (R8)(R9*1), R12 - MOVQ R11, CX - SHLQ CL, R12 - ORQ R12, R10 - -bitReader_fillFast_1_end: - MOVQ DI, CX - MOVQ R10, R12 - SHRQ CL, R12 - MOVW (SI)(R12*2), CX - MOVB CH, AL - MOVBQZX CL, CX + SUBQ SI, CX + JLE done + SHRQ $0x04, CX + + // Iterations allowed by the input: a refill reads the 8 bytes below the window and + // moves it down by at most 7, so every read stays inside the stream while ip stays + // at least 8 above ilowest. + MOVQ R8, R12 + SUBQ R9, R12 + SHRQ $0x03, R12 + CMPQ R12, CX + CMOVQCS R12, CX + TESTQ CX, CX + JZ done + IMUL3Q $0x0e, CX, R12 + ADDQ SI, R12 + +inner_loop: + // symbol 0 + MOVQ DX, CX + MOVQ R10, R13 + SHRQ CL, R13 + MOVWQZX (BX)(R13*2), CX + SHLQ CL, R10 ADDQ CX, R11 + SHRQ $0x08, CX + MOVB CL, (SI) + + // symbol 1 + MOVQ DX, CX + MOVQ R10, R13 + SHRQ CL, R13 + MOVWQZX (BX)(R13*2), CX SHLQ CL, R10 - MOVQ DI, CX - MOVQ R10, R12 - SHRQ CL, R12 - MOVW (SI)(R12*2), CX - MOVB CH, AH - MOVBQZX CL, CX ADDQ CX, R11 + SHRQ $0x08, CX + MOVB CL, 1(SI) + + // symbol 2 + MOVQ DX, CX + MOVQ R10, R13 + SHRQ CL, R13 + MOVWQZX (BX)(R13*2), CX SHLQ CL, R10 - BSWAPL AX - CMPQ R11, $0x20 - JL bitReader_fillFast_2_end - SUBQ $0x20, R11 - SUBQ $0x04, R9 - MOVL (R8)(R9*1), R12 - MOVQ R11, CX - SHLQ CL, R12 - ORQ R12, R10 - -bitReader_fillFast_2_end: - MOVQ DI, CX - MOVQ R10, R12 - SHRQ CL, R12 - MOVW (SI)(R12*2), CX - MOVB CH, AH - MOVBQZX CL, CX ADDQ CX, R11 + SHRQ $0x08, CX + MOVB CL, 2(SI) + + // symbol 3 + MOVQ DX, CX + MOVQ R10, R13 + SHRQ CL, R13 + MOVWQZX (BX)(R13*2), CX SHLQ CL, R10 - MOVQ DI, CX - MOVQ R10, R12 - SHRQ CL, R12 - MOVW (SI)(R12*2), CX - MOVB CH, AL - MOVBQZX CL, CX ADDQ CX, R11 + SHRQ $0x08, CX + MOVB CL, 3(SI) + + // symbol 4 + MOVQ DX, CX + MOVQ R10, R13 + SHRQ CL, R13 + MOVWQZX (BX)(R13*2), CX + SHLQ CL, R10 + ADDQ CX, R11 + SHRQ $0x08, CX + MOVB CL, 4(SI) + + // symbol 5 + MOVQ DX, CX + MOVQ R10, R13 + SHRQ CL, R13 + MOVWQZX (BX)(R13*2), CX + SHLQ CL, R10 + ADDQ CX, R11 + SHRQ $0x08, CX + MOVB CL, 5(SI) + + // symbol 6 + MOVQ DX, CX + MOVQ R10, R13 + SHRQ CL, R13 + MOVWQZX (BX)(R13*2), CX + SHLQ CL, R10 + ADDQ CX, R11 + SHRQ $0x08, CX + MOVB CL, 6(SI) + + // symbol 7 + MOVQ DX, CX + MOVQ R10, R13 + SHRQ CL, R13 + MOVWQZX (BX)(R13*2), CX + SHLQ CL, R10 + ADDQ CX, R11 + SHRQ $0x08, CX + MOVB CL, 7(SI) + + // symbol 8 + MOVQ DX, CX + MOVQ R10, R13 + SHRQ CL, R13 + MOVWQZX (BX)(R13*2), CX + SHLQ CL, R10 + ADDQ CX, R11 + SHRQ $0x08, CX + MOVB CL, 8(SI) + + // symbol 9 + MOVQ DX, CX + MOVQ R10, R13 + SHRQ CL, R13 + MOVWQZX (BX)(R13*2), CX SHLQ CL, R10 - BSWAPL AX - - // Store the decoded values - MOVL AX, (DX) - ADDQ $0x04, DX - -loop_condition: - CMPQ R9, $0x08 - JGE main_loop - - // Update ctx structure - MOVQ ctx+0(FP), AX - SUBQ 16(AX), DX - MOVQ DX, 40(AX) - MOVQ (AX), AX - MOVQ R9, 24(AX) - MOVQ R10, 32(AX) - MOVB R11, 40(AX) - RET - - // Report error -error_max_decoded_size_exceeded: - MOVQ ctx+0(FP), AX - MOVQ $-1, CX - MOVQ CX, 40(AX) - RET - -// func decompress1x_main_loop_bmi2(ctx *decompress1xContext) -// Requires: BMI2 -TEXT ·decompress1x_main_loop_bmi2(SB), $0-8 - MOVQ ctx+0(FP), CX - MOVQ 16(CX), DX - MOVQ 24(CX), BX - CMPQ BX, $0x04 - JB error_max_decoded_size_exceeded - LEAQ (DX)(BX*1), BX - MOVQ (CX), SI - MOVQ (SI), R8 - MOVQ 24(SI), R9 - MOVQ 32(SI), R10 - MOVBQZX 40(SI), R11 - MOVQ 32(CX), SI - MOVBQZX 8(CX), DI - JMP loop_condition - -main_loop: - // Check if we have room for 4 bytes in the output buffer - LEAQ 4(DX), CX - CMPQ CX, BX - JGE error_max_decoded_size_exceeded - - // Decode 4 values - CMPQ R11, $0x20 - JL bitReader_fillFast_1_end - SUBQ $0x20, R11 - SUBQ $0x04, R9 - MOVL (R8)(R9*1), CX - SHLXQ R11, CX, CX - ORQ CX, R10 - -bitReader_fillFast_1_end: - SHRXQ DI, R10, CX - MOVW (SI)(CX*2), CX - MOVB CH, AL - MOVBQZX CL, CX ADDQ CX, R11 - SHLXQ CX, R10, R10 - SHRXQ DI, R10, CX - MOVW (SI)(CX*2), CX - MOVB CH, AH - MOVBQZX CL, CX + SHRQ $0x08, CX + MOVB CL, 9(SI) + + // symbol 10 + MOVQ DX, CX + MOVQ R10, R13 + SHRQ CL, R13 + MOVWQZX (BX)(R13*2), CX + SHLQ CL, R10 ADDQ CX, R11 - SHLXQ CX, R10, R10 - BSWAPL AX - CMPQ R11, $0x20 - JL bitReader_fillFast_2_end - SUBQ $0x20, R11 - SUBQ $0x04, R9 - MOVL (R8)(R9*1), CX - SHLXQ R11, CX, CX - ORQ CX, R10 - -bitReader_fillFast_2_end: - SHRXQ DI, R10, CX - MOVW (SI)(CX*2), CX - MOVB CH, AH - MOVBQZX CL, CX + SHRQ $0x08, CX + MOVB CL, 10(SI) + + // symbol 11 + MOVQ DX, CX + MOVQ R10, R13 + SHRQ CL, R13 + MOVWQZX (BX)(R13*2), CX + SHLQ CL, R10 ADDQ CX, R11 - SHLXQ CX, R10, R10 - SHRXQ DI, R10, CX - MOVW (SI)(CX*2), CX - MOVB CH, AL - MOVBQZX CL, CX + SHRQ $0x08, CX + MOVB CL, 11(SI) + + // symbol 12 + MOVQ DX, CX + MOVQ R10, R13 + SHRQ CL, R13 + MOVWQZX (BX)(R13*2), CX + SHLQ CL, R10 ADDQ CX, R11 - SHLXQ CX, R10, R10 - BSWAPL AX + SHRQ $0x08, CX + MOVB CL, 12(SI) + + // symbol 13 + MOVQ DX, CX + MOVQ R10, R13 + SHRQ CL, R13 + MOVWQZX (BX)(R13*2), CX + SHLQ CL, R10 + ADDQ CX, R11 + SHRQ $0x08, CX + MOVB CL, 13(SI) - // Store the decoded values - MOVL AX, (DX) - ADDQ $0x04, DX + // Refill the whole bytes consumed from below the window + MOVQ R11, R13 + ANDQ $0x38, R13 + MOVQ -8(R8), R14 + SHRQ $0x01, R14 + ANDQ $0x07, R11 + MOVQ R13, CX + XORQ $0x3f, CX + SHRQ CL, R14 + MOVQ R11, CX + SHLQ CL, R14 + ORQ R14, R10 + SHRQ $0x03, R13 + SUBQ R13, R8 + ADDQ $0x0e, SI + CMPQ SI, R12 + JB inner_loop + JMP outer_loop + +done: + // Hand the state back in the Go bit reader's own form: off = ip - in, value, bitsRead. + MOVQ (AX), CX + SUBQ (CX), R8 + MOVQ R8, 24(CX) + MOVQ R10, 32(CX) + MOVB R11, 40(CX) + SUBQ 16(AX), SI + MOVQ SI, 40(AX) + RET + +// func decompress1x_main_loop_bmi2(ctx *decompress1xContext) +// Requires: BMI2, CMOV +TEXT ·decompress1x_main_loop_bmi2(SB), $0-8 + MOVQ ctx+0(FP), AX -loop_condition: - CMPQ R9, $0x08 - JGE main_loop + // Preload values + MOVBQZX 8(AX), CX + MOVQ 32(AX), DX + MOVQ 16(AX), BX + MOVQ 24(AX), SI + ADDQ BX, SI + MOVQ 56(AX), DI + MOVQ 48(AX), R8 + MOVQ (AX), R10 + MOVQ 32(R10), R9 + MOVBQZX 40(R10), R10 + +outer_loop: + // Iterations allowed by the output: (limit - op) / 8 + MOVQ SI, R11 + SUBQ BX, R11 + JLE done + SHRQ $0x03, R11 + + // Iterations allowed by the input: a refill reads the 8 bytes below the window and + // moves it down by at most 7, so every read stays inside the stream while ip stays + // at least 8 above ilowest. + MOVQ DI, R12 + SUBQ R8, R12 + SHRQ $0x03, R12 + CMPQ R12, R11 + CMOVQCS R12, R11 + TESTQ R11, R11 + JZ done + IMUL3Q $0x05, R11, R11 + ADDQ BX, R11 + +inner_loop: + // symbol 0 + SHRXQ CX, R9, R12 + MOVWQZX (DX)(R12*2), R12 + SHLXQ R12, R9, R9 + ADDQ R12, R10 + SHRQ $0x08, R12 + MOVB R12, (BX) + + // symbol 1 + SHRXQ CX, R9, R12 + MOVWQZX (DX)(R12*2), R12 + SHLXQ R12, R9, R9 + ADDQ R12, R10 + SHRQ $0x08, R12 + MOVB R12, 1(BX) + + // symbol 2 + SHRXQ CX, R9, R12 + MOVWQZX (DX)(R12*2), R12 + SHLXQ R12, R9, R9 + ADDQ R12, R10 + SHRQ $0x08, R12 + MOVB R12, 2(BX) + + // symbol 3 + SHRXQ CX, R9, R12 + MOVWQZX (DX)(R12*2), R12 + SHLXQ R12, R9, R9 + ADDQ R12, R10 + SHRQ $0x08, R12 + MOVB R12, 3(BX) + + // symbol 4 + SHRXQ CX, R9, R12 + MOVWQZX (DX)(R12*2), R12 + SHLXQ R12, R9, R9 + ADDQ R12, R10 + SHRQ $0x08, R12 + MOVB R12, 4(BX) + + // Refill the whole bytes consumed from below the window + MOVQ R10, R12 + ANDQ $0x38, R12 + MOVQ -8(DI), R13 + SHRQ $0x01, R13 + ANDQ $0x07, R10 + MOVQ R12, R14 + XORQ $0x3f, R14 + SHRXQ R14, R13, R13 + SHLXQ R10, R13, R13 + ORQ R13, R9 + SHRQ $0x03, R12 + SUBQ R12, DI + ADDQ $0x05, BX + CMPQ BX, R11 + JB inner_loop + JMP outer_loop + +done: + // Hand the state back in the Go bit reader's own form: off = ip - in, value, bitsRead. + MOVQ (AX), CX + SUBQ (CX), DI + MOVQ DI, 24(CX) + MOVQ R9, 32(CX) + MOVB R10, 40(CX) + SUBQ 16(AX), BX + MOVQ BX, 40(AX) + RET - // Update ctx structure +// func decompress1x_8b_main_loop_bmi2(ctx *decompress1xContext) +// Requires: BMI2, CMOV +TEXT ·decompress1x_8b_main_loop_bmi2(SB), $0-8 MOVQ ctx+0(FP), AX - SUBQ 16(AX), DX - MOVQ DX, 40(AX) - MOVQ (AX), AX - MOVQ R9, 24(AX) - MOVQ R10, 32(AX) - MOVB R11, 40(AX) + + // Preload values + MOVBQZX 8(AX), CX + MOVQ 32(AX), DX + MOVQ 16(AX), BX + MOVQ 24(AX), SI + ADDQ BX, SI + MOVQ 56(AX), DI + MOVQ 48(AX), R8 + MOVQ (AX), R10 + MOVQ 32(R10), R9 + MOVBQZX 40(R10), R10 + +outer_loop: + // Iterations allowed by the output: (limit - op) / 8 + MOVQ SI, R11 + SUBQ BX, R11 + JLE done + SHRQ $0x03, R11 + + // Iterations allowed by the input: a refill reads the 8 bytes below the window and + // moves it down by at most 7, so every read stays inside the stream while ip stays + // at least 8 above ilowest. + MOVQ DI, R12 + SUBQ R8, R12 + SHRQ $0x03, R12 + CMPQ R12, R11 + CMOVQCS R12, R11 + TESTQ R11, R11 + JZ done + IMUL3Q $0x07, R11, R11 + ADDQ BX, R11 + +inner_loop: + // symbol 0 + SHRXQ CX, R9, R12 + MOVWQZX (DX)(R12*2), R12 + SHLXQ R12, R9, R9 + ADDQ R12, R10 + SHRQ $0x08, R12 + MOVB R12, (BX) + + // symbol 1 + SHRXQ CX, R9, R12 + MOVWQZX (DX)(R12*2), R12 + SHLXQ R12, R9, R9 + ADDQ R12, R10 + SHRQ $0x08, R12 + MOVB R12, 1(BX) + + // symbol 2 + SHRXQ CX, R9, R12 + MOVWQZX (DX)(R12*2), R12 + SHLXQ R12, R9, R9 + ADDQ R12, R10 + SHRQ $0x08, R12 + MOVB R12, 2(BX) + + // symbol 3 + SHRXQ CX, R9, R12 + MOVWQZX (DX)(R12*2), R12 + SHLXQ R12, R9, R9 + ADDQ R12, R10 + SHRQ $0x08, R12 + MOVB R12, 3(BX) + + // symbol 4 + SHRXQ CX, R9, R12 + MOVWQZX (DX)(R12*2), R12 + SHLXQ R12, R9, R9 + ADDQ R12, R10 + SHRQ $0x08, R12 + MOVB R12, 4(BX) + + // symbol 5 + SHRXQ CX, R9, R12 + MOVWQZX (DX)(R12*2), R12 + SHLXQ R12, R9, R9 + ADDQ R12, R10 + SHRQ $0x08, R12 + MOVB R12, 5(BX) + + // symbol 6 + SHRXQ CX, R9, R12 + MOVWQZX (DX)(R12*2), R12 + SHLXQ R12, R9, R9 + ADDQ R12, R10 + SHRQ $0x08, R12 + MOVB R12, 6(BX) + + // Refill the whole bytes consumed from below the window + MOVQ R10, R12 + ANDQ $0x38, R12 + MOVQ -8(DI), R13 + SHRQ $0x01, R13 + ANDQ $0x07, R10 + MOVQ R12, R14 + XORQ $0x3f, R14 + SHRXQ R14, R13, R13 + SHLXQ R10, R13, R13 + ORQ R13, R9 + SHRQ $0x03, R12 + SUBQ R12, DI + ADDQ $0x07, BX + CMPQ BX, R11 + JB inner_loop + JMP outer_loop + +done: + // Hand the state back in the Go bit reader's own form: off = ip - in, value, bitsRead. + MOVQ (AX), CX + SUBQ (CX), DI + MOVQ DI, 24(CX) + MOVQ R9, 32(CX) + MOVB R10, 40(CX) + SUBQ 16(AX), BX + MOVQ BX, 40(AX) RET - // Report error -error_max_decoded_size_exceeded: +// func decompress1x_4b_main_loop_bmi2(ctx *decompress1xContext) +// Requires: BMI2, CMOV +TEXT ·decompress1x_4b_main_loop_bmi2(SB), $0-8 MOVQ ctx+0(FP), AX - MOVQ $-1, CX - MOVQ CX, 40(AX) + + // Preload values + MOVBQZX 8(AX), CX + MOVQ 32(AX), DX + MOVQ 16(AX), BX + MOVQ 24(AX), SI + ADDQ BX, SI + MOVQ 56(AX), DI + MOVQ 48(AX), R8 + MOVQ (AX), R10 + MOVQ 32(R10), R9 + MOVBQZX 40(R10), R10 + +outer_loop: + // Iterations allowed by the output: (limit - op) / 16 + MOVQ SI, R11 + SUBQ BX, R11 + JLE done + SHRQ $0x04, R11 + + // Iterations allowed by the input: a refill reads the 8 bytes below the window and + // moves it down by at most 7, so every read stays inside the stream while ip stays + // at least 8 above ilowest. + MOVQ DI, R12 + SUBQ R8, R12 + SHRQ $0x03, R12 + CMPQ R12, R11 + CMOVQCS R12, R11 + TESTQ R11, R11 + JZ done + IMUL3Q $0x0e, R11, R11 + ADDQ BX, R11 + +inner_loop: + // symbol 0 + SHRXQ CX, R9, R12 + MOVWQZX (DX)(R12*2), R12 + SHLXQ R12, R9, R9 + ADDQ R12, R10 + SHRQ $0x08, R12 + MOVB R12, (BX) + + // symbol 1 + SHRXQ CX, R9, R12 + MOVWQZX (DX)(R12*2), R12 + SHLXQ R12, R9, R9 + ADDQ R12, R10 + SHRQ $0x08, R12 + MOVB R12, 1(BX) + + // symbol 2 + SHRXQ CX, R9, R12 + MOVWQZX (DX)(R12*2), R12 + SHLXQ R12, R9, R9 + ADDQ R12, R10 + SHRQ $0x08, R12 + MOVB R12, 2(BX) + + // symbol 3 + SHRXQ CX, R9, R12 + MOVWQZX (DX)(R12*2), R12 + SHLXQ R12, R9, R9 + ADDQ R12, R10 + SHRQ $0x08, R12 + MOVB R12, 3(BX) + + // symbol 4 + SHRXQ CX, R9, R12 + MOVWQZX (DX)(R12*2), R12 + SHLXQ R12, R9, R9 + ADDQ R12, R10 + SHRQ $0x08, R12 + MOVB R12, 4(BX) + + // symbol 5 + SHRXQ CX, R9, R12 + MOVWQZX (DX)(R12*2), R12 + SHLXQ R12, R9, R9 + ADDQ R12, R10 + SHRQ $0x08, R12 + MOVB R12, 5(BX) + + // symbol 6 + SHRXQ CX, R9, R12 + MOVWQZX (DX)(R12*2), R12 + SHLXQ R12, R9, R9 + ADDQ R12, R10 + SHRQ $0x08, R12 + MOVB R12, 6(BX) + + // symbol 7 + SHRXQ CX, R9, R12 + MOVWQZX (DX)(R12*2), R12 + SHLXQ R12, R9, R9 + ADDQ R12, R10 + SHRQ $0x08, R12 + MOVB R12, 7(BX) + + // symbol 8 + SHRXQ CX, R9, R12 + MOVWQZX (DX)(R12*2), R12 + SHLXQ R12, R9, R9 + ADDQ R12, R10 + SHRQ $0x08, R12 + MOVB R12, 8(BX) + + // symbol 9 + SHRXQ CX, R9, R12 + MOVWQZX (DX)(R12*2), R12 + SHLXQ R12, R9, R9 + ADDQ R12, R10 + SHRQ $0x08, R12 + MOVB R12, 9(BX) + + // symbol 10 + SHRXQ CX, R9, R12 + MOVWQZX (DX)(R12*2), R12 + SHLXQ R12, R9, R9 + ADDQ R12, R10 + SHRQ $0x08, R12 + MOVB R12, 10(BX) + + // symbol 11 + SHRXQ CX, R9, R12 + MOVWQZX (DX)(R12*2), R12 + SHLXQ R12, R9, R9 + ADDQ R12, R10 + SHRQ $0x08, R12 + MOVB R12, 11(BX) + + // symbol 12 + SHRXQ CX, R9, R12 + MOVWQZX (DX)(R12*2), R12 + SHLXQ R12, R9, R9 + ADDQ R12, R10 + SHRQ $0x08, R12 + MOVB R12, 12(BX) + + // symbol 13 + SHRXQ CX, R9, R12 + MOVWQZX (DX)(R12*2), R12 + SHLXQ R12, R9, R9 + ADDQ R12, R10 + SHRQ $0x08, R12 + MOVB R12, 13(BX) + + // Refill the whole bytes consumed from below the window + MOVQ R10, R12 + ANDQ $0x38, R12 + MOVQ -8(DI), R13 + SHRQ $0x01, R13 + ANDQ $0x07, R10 + MOVQ R12, R14 + XORQ $0x3f, R14 + SHRXQ R14, R13, R13 + SHLXQ R10, R13, R13 + ORQ R13, R9 + SHRQ $0x03, R12 + SUBQ R12, DI + ADDQ $0x0e, BX + CMPQ BX, R11 + JB inner_loop + JMP outer_loop + +done: + // Hand the state back in the Go bit reader's own form: off = ip - in, value, bitsRead. + MOVQ (AX), CX + SUBQ (CX), DI + MOVQ DI, 24(CX) + MOVQ R9, 32(CX) + MOVB R10, 40(CX) + SUBQ 16(AX), BX + MOVQ BX, 40(AX) RET diff --git a/vendor/github.com/klauspost/compress/huff0/decompress_arm64.go b/vendor/github.com/klauspost/compress/huff0/decompress_arm64.go index 8ba3c810e60a..d1988ad54dc4 100644 --- a/vendor/github.com/klauspost/compress/huff0/decompress_arm64.go +++ b/vendor/github.com/klauspost/compress/huff0/decompress_arm64.go @@ -6,24 +6,47 @@ package huff0 // decompress4x_main_loop_arm64 is an arm64 assembler implementation -// of Decompress4X when tablelog > 8. +// of Decompress4X when tablelog > 8, decoding fastSymbols symbols per +// stream between bit container reloads. // //go:noescape func decompress4x_main_loop_arm64(ctx *decompress4xContext) // decompress4x_8b_main_loop_arm64 is an arm64 assembler implementation -// of Decompress4X when tablelog <= 8 which decodes 4 entries -// per loop. +// of Decompress4X when tablelog <= 8, decoding fast8bSymbols symbols +// per stream between bit container reloads. // //go:noescape func decompress4x_8b_main_loop_arm64(ctx *decompress4xContext) +// decompress4x_4b_main_loop_arm64 is an arm64 assembler implementation +// of Decompress4X when tablelog <= 4, decoding fast4bSymbols symbols +// per stream between bit container reloads. +// +//go:noescape +func decompress4x_4b_main_loop_arm64(ctx *decompress4xContext) + // decompress1x_main_loop_arm64 is an arm64 assembler implementation -// of Decompress1X when tablelog > 8. +// of Decompress1X when tablelog > 8, decoding fastSymbols symbols +// between bit container reloads. // //go:noescape func decompress1x_main_loop_arm64(ctx *decompress1xContext) +// decompress1x_8b_main_loop_arm64 is an arm64 assembler implementation +// of Decompress1X when tablelog <= 8, decoding fast8bSymbols symbols +// between bit container reloads. +// +//go:noescape +func decompress1x_8b_main_loop_arm64(ctx *decompress1xContext) + +// decompress1x_4b_main_loop_arm64 is an arm64 assembler implementation +// of Decompress1X when tablelog <= 4, decoding fast4bSymbols symbols +// between bit container reloads. +// +//go:noescape +func decompress1x_4b_main_loop_arm64(ctx *decompress1xContext) + func decompress4x_main_loop_asm(ctx *decompress4xContext) { decompress4x_main_loop_arm64(ctx) } @@ -32,6 +55,18 @@ func decompress4x_8b_main_loop_asm(ctx *decompress4xContext) { decompress4x_8b_main_loop_arm64(ctx) } +func decompress4x_4b_main_loop_asm(ctx *decompress4xContext) { + decompress4x_4b_main_loop_arm64(ctx) +} + func decompress1x_main_loop_asm(ctx *decompress1xContext) { decompress1x_main_loop_arm64(ctx) } + +func decompress1x_8b_main_loop_asm(ctx *decompress1xContext) { + decompress1x_8b_main_loop_arm64(ctx) +} + +func decompress1x_4b_main_loop_asm(ctx *decompress1xContext) { + decompress1x_4b_main_loop_arm64(ctx) +} diff --git a/vendor/github.com/klauspost/compress/huff0/decompress_arm64.s b/vendor/github.com/klauspost/compress/huff0/decompress_arm64.s index e16012721a1c..971b8eb8bdc0 100644 --- a/vendor/github.com/klauspost/compress/huff0/decompress_arm64.s +++ b/vendor/github.com/klauspost/compress/huff0/decompress_arm64.s @@ -1,851 +1,1657 @@ // Code generated by command: go run gen.go -out ../decompress.s -arch amd64,arm64 -pkg=huff0. DO NOT EDIT. // EXPERIMENTAL arm64 output lowered from an amd64 avo program. -//go:build arm64 && !appengine && !noasm && gc +//go:build arm64 && (!appengine && !noasm && gc) // func decompress4x_main_loop_amd64(ctx *decompress4xContext) +// Requires: BMI, CMOV TEXT ·decompress4x_main_loop_arm64(SB), $0-8 + MOVD ctx+0(FP), R0 + // Preload values - MOVD ctx+0(FP), R0 - MOVBU 8(R0), R6 - MOVD 16(R0), R3 - MOVD 48(R0), R5 - MOVD 24(R0), R7 - MOVD 32(R0), R8 - MOVD (R0), R9 - - // Main loop -main_loop: - MOVD $0, R2 - CMP R5, R3 - CSET GE, R16 - BFI $0, R16, $8, R2 - - // br0.fillFast32() - MOVD 32(R9), R10 - MOVBU 40(R9), R11 - CMP $0x20, R11 - BLS skip_fill0 - MOVD 24(R9), R0 - SUB $0x20, R11, R11 - SUB $0x04, R0, R0 - MOVD (R9), R12 - - // b.value |= uint64(low) << (b.bitsRead & 63) - ADD R12, R0, R15 - MOVWU (R15), R12 - MOVD R11, R1 + MOVBU 8(R0), R2 + MOVD 32(R0), R3 + MOVD 16(R0), R5 + MOVD 24(R0), R1 + ADD R1, R5, R7 + ADD R1<<1, R5, R9 + ADD R1<<1, R1, R11 + ADD R5, R11, R11 + + // Convert each bit reader to sentinel form: bits = value | 1<> 8) - // out[id * dstEvery + 1] = uint8(v1.entry >> 8) - MOVH R0, (R3) - - // update the bitreader structure - MOVD R10, 32(R9) - MOVB R11, 40(R9) - - // br1.fillFast32() - MOVD 80(R9), R10 - MOVBU 88(R9), R11 - CMP $0x20, R11 - BLS skip_fill1 - MOVD 72(R9), R0 - SUB $0x20, R11, R11 - SUB $0x04, R0, R0 - MOVD 48(R9), R12 - - // b.value |= uint64(low) << (b.bitsRead & 63) - ADD R12, R0, R15 - MOVWU (R15), R12 - MOVD R11, R1 + LSR $0x08, R1, R1 + MOVB R1, (R11) + + // stream 0, symbol 1 + LSR R2, R6, R13 + MOVHU (R3)(R13<<1), R1 + LSL R1, R6, R6 + LSR $0x08, R1, R1 + MOVB R1, 1(R5) + + // stream 1, symbol 1 + LSR R2, R8, R13 + MOVHU (R3)(R13<<1), R1 + LSL R1, R8, R8 + LSR $0x08, R1, R1 + MOVB R1, 1(R7) + + // stream 2, symbol 1 + LSR R2, R10, R13 + MOVHU (R3)(R13<<1), R1 + LSL R1, R10, R10 + LSR $0x08, R1, R1 + MOVB R1, 1(R9) + + // stream 3, symbol 1 + LSR R2, R12, R13 + MOVHU (R3)(R13<<1), R1 LSL R1, R12, R12 - MOVD R0, 72(R9) - ORR R12, R10, R10 - - // exhausted += (br1.off < 4) - CMP $0x04, R0 - CSINC HS, R2, R2, R16 - BFI $0, R16, $8, R2 - -skip_fill1: - // val0 := br1.peekTopBits(peekBits) - MOVD R10, R12 - MOVD R6, R1 - LSR R1, R12, R12 - - // v0 := table[val0&mask] - ADD R12<<1, R8, R15 - MOVHU (R15), R1 - - // br1.advance(uint8(v0.entry) - UBFX $8, R1, $8, R16 - BFI $0, R16, $8, R0 - LSL R1, R10, R10 - ADD R1, R11, R15 - BFI $0, R15, $8, R11 - - // val1 := br1.peekTopBits(peekBits) - MOVD R6, R1 - MOVD R10, R12 - LSR R1, R12, R12 - - // v1 := table[val1&mask] - ADD R12<<1, R8, R15 - MOVHU (R15), R1 - - // br1.advance(uint8(v1.entry)) - UBFX $8, R1, $8, R16 - BFI $8, R16, $8, R0 - LSL R1, R10, R10 - ADD R1, R11, R15 - BFI $0, R15, $8, R11 - - // these two writes get coalesced - // out[id * dstEvery + 0] = uint8(v0.entry >> 8) - // out[id * dstEvery + 1] = uint8(v1.entry >> 8) - ADD R7, R3, R15 - MOVH R0, (R15) - - // update the bitreader structure - MOVD R10, 80(R9) - MOVB R11, 88(R9) - - // br2.fillFast32() - MOVD 128(R9), R10 - MOVBU 136(R9), R11 - CMP $0x20, R11 - BLS skip_fill2 - MOVD 120(R9), R0 - SUB $0x20, R11, R11 - SUB $0x04, R0, R0 - MOVD 96(R9), R12 - - // b.value |= uint64(low) << (b.bitsRead & 63) - ADD R12, R0, R15 - MOVWU (R15), R12 - MOVD R11, R1 + LSR $0x08, R1, R1 + MOVB R1, 1(R11) + + // stream 0, symbol 2 + LSR R2, R6, R13 + MOVHU (R3)(R13<<1), R1 + LSL R1, R6, R6 + LSR $0x08, R1, R1 + MOVB R1, 2(R5) + + // stream 1, symbol 2 + LSR R2, R8, R13 + MOVHU (R3)(R13<<1), R1 + LSL R1, R8, R8 + LSR $0x08, R1, R1 + MOVB R1, 2(R7) + + // stream 2, symbol 2 + LSR R2, R10, R13 + MOVHU (R3)(R13<<1), R1 + LSL R1, R10, R10 + LSR $0x08, R1, R1 + MOVB R1, 2(R9) + + // stream 3, symbol 2 + LSR R2, R12, R13 + MOVHU (R3)(R13<<1), R1 LSL R1, R12, R12 - MOVD R0, 120(R9) - ORR R12, R10, R10 - - // exhausted += (br2.off < 4) - CMP $0x04, R0 - CSINC HS, R2, R2, R16 - BFI $0, R16, $8, R2 - -skip_fill2: - // val0 := br2.peekTopBits(peekBits) - MOVD R10, R12 - MOVD R6, R1 - LSR R1, R12, R12 - - // v0 := table[val0&mask] - ADD R12<<1, R8, R15 - MOVHU (R15), R1 - - // br2.advance(uint8(v0.entry) - UBFX $8, R1, $8, R16 - BFI $0, R16, $8, R0 - LSL R1, R10, R10 - ADD R1, R11, R15 - BFI $0, R15, $8, R11 - - // val1 := br2.peekTopBits(peekBits) - MOVD R6, R1 - MOVD R10, R12 - LSR R1, R12, R12 - - // v1 := table[val1&mask] - ADD R12<<1, R8, R15 - MOVHU (R15), R1 - - // br2.advance(uint8(v1.entry)) - UBFX $8, R1, $8, R16 - BFI $8, R16, $8, R0 - LSL R1, R10, R10 - ADD R1, R11, R15 - BFI $0, R15, $8, R11 - - // these two writes get coalesced - // out[id * dstEvery + 0] = uint8(v0.entry >> 8) - // out[id * dstEvery + 1] = uint8(v1.entry >> 8) - ADD R7<<1, R3, R15 - MOVH R0, (R15) - - // update the bitreader structure - MOVD R10, 128(R9) - MOVB R11, 136(R9) - - // br3.fillFast32() - MOVD 176(R9), R10 - MOVBU 184(R9), R11 - CMP $0x20, R11 - BLS skip_fill3 - MOVD 168(R9), R0 - SUB $0x20, R11, R11 - SUB $0x04, R0, R0 - MOVD 144(R9), R12 - - // b.value |= uint64(low) << (b.bitsRead & 63) - ADD R12, R0, R15 - MOVWU (R15), R12 - MOVD R11, R1 + LSR $0x08, R1, R1 + MOVB R1, 2(R11) + + // stream 0, symbol 3 + LSR R2, R6, R13 + MOVHU (R3)(R13<<1), R1 + LSL R1, R6, R6 + LSR $0x08, R1, R1 + MOVB R1, 3(R5) + + // stream 1, symbol 3 + LSR R2, R8, R13 + MOVHU (R3)(R13<<1), R1 + LSL R1, R8, R8 + LSR $0x08, R1, R1 + MOVB R1, 3(R7) + + // stream 2, symbol 3 + LSR R2, R10, R13 + MOVHU (R3)(R13<<1), R1 + LSL R1, R10, R10 + LSR $0x08, R1, R1 + MOVB R1, 3(R9) + + // stream 3, symbol 3 + LSR R2, R12, R13 + MOVHU (R3)(R13<<1), R1 LSL R1, R12, R12 - MOVD R0, 168(R9) - ORR R12, R10, R10 - - // exhausted += (br3.off < 4) - CMP $0x04, R0 - CSINC HS, R2, R2, R16 - BFI $0, R16, $8, R2 - -skip_fill3: - // val0 := br3.peekTopBits(peekBits) - MOVD R10, R12 - MOVD R6, R1 - LSR R1, R12, R12 - - // v0 := table[val0&mask] - ADD R12<<1, R8, R15 - MOVHU (R15), R1 - - // br3.advance(uint8(v0.entry) - UBFX $8, R1, $8, R16 - BFI $0, R16, $8, R0 - LSL R1, R10, R10 - ADD R1, R11, R15 - BFI $0, R15, $8, R11 - - // val1 := br3.peekTopBits(peekBits) - MOVD R6, R1 - MOVD R10, R12 - LSR R1, R12, R12 - - // v1 := table[val1&mask] - ADD R12<<1, R8, R15 - MOVHU (R15), R1 - - // br3.advance(uint8(v1.entry)) - UBFX $8, R1, $8, R16 - BFI $8, R16, $8, R0 + LSR $0x08, R1, R1 + MOVB R1, 3(R11) + + // stream 0, symbol 4 + LSR R2, R6, R13 + MOVHU (R3)(R13<<1), R1 + LSL R1, R6, R6 + LSR $0x08, R1, R1 + MOVB R1, 4(R5) + + // stream 1, symbol 4 + LSR R2, R8, R13 + MOVHU (R3)(R13<<1), R1 + LSL R1, R8, R8 + LSR $0x08, R1, R1 + MOVB R1, 4(R7) + + // stream 2, symbol 4 + LSR R2, R10, R13 + MOVHU (R3)(R13<<1), R1 + LSL R1, R10, R10 + LSR $0x08, R1, R1 + MOVB R1, 4(R9) + + // stream 3, symbol 4 + LSR R2, R12, R13 + MOVHU (R3)(R13<<1), R1 + LSL R1, R12, R12 + LSR $0x08, R1, R1 + MOVB R1, 4(R11) + + // Reload the four bit containers + RBIT R6, R13 + CLZ R13, R13 + MOVD R13, R1 + AND $0x07, R1, R1 + LSR $0x03, R13, R13 + MOVD 64(R0), R6 + SUB R13, R6, R6 + MOVD R6, 64(R0) + MOVD (R6), R6 + ORR $0x01, R6, R6 + LSL R1, R6, R6 + RBIT R8, R13 + CLZ R13, R13 + MOVD R13, R1 + AND $0x07, R1, R1 + LSR $0x03, R13, R13 + MOVD 72(R0), R8 + SUB R13, R8, R8 + MOVD R8, 72(R0) + MOVD (R8), R8 + ORR $0x01, R8, R8 + LSL R1, R8, R8 + RBIT R10, R13 + CLZ R13, R13 + MOVD R13, R1 + AND $0x07, R1, R1 + LSR $0x03, R13, R13 + MOVD 80(R0), R10 + SUB R13, R10, R10 + MOVD R10, 80(R0) + MOVD (R10), R10 + ORR $0x01, R10, R10 LSL R1, R10, R10 - ADD R1, R11, R15 - BFI $0, R15, $8, R11 - - // these two writes get coalesced - // out[id * dstEvery + 0] = uint8(v0.entry >> 8) - // out[id * dstEvery + 1] = uint8(v1.entry >> 8) - ADD R7<<1, R7, R1 - ADD R1, R3, R15 - MOVH R0, (R15) - - // update the bitreader structure - MOVD R10, 176(R9) - MOVB R11, 184(R9) - ADD $0x02, R3, R3 - AND $0xff, R2, R15 - AND $0xff, R2, R16 - TST R16, R15 - BEQ main_loop - MOVD ctx+0(FP), R0 + RBIT R12, R13 + CLZ R13, R13 + MOVD R13, R1 + AND $0x07, R1, R1 + LSR $0x03, R13, R13 + MOVD 88(R0), R12 + SUB R13, R12, R12 + MOVD R12, 88(R0) + MOVD (R12), R12 + ORR $0x01, R12, R12 + LSL R1, R12, R12 + ADD $0x05, R5, R5 + ADD $0x05, R7, R7 + ADD $0x05, R9, R9 + ADD $0x05, R11, R11 + MOVD 96(R0), R16 + CMP R16, R5 + BLO inner_loop + JMP outer_loop + +done: + // Hand the state back: off = ip - in (negative when the window reached below the stream start), + // value = the sentinel-form container. bitReaderShifted.restoreFromAsm normalizes both. + MOVD (R0), R1 + MOVD 64(R0), R2 + MOVD (R1), R16 + SUB R16, R2, R2 + MOVD R2, 24(R1) + MOVD R6, 32(R1) + MOVD 72(R0), R2 + MOVD 48(R1), R16 + SUB R16, R2, R2 + MOVD R2, 72(R1) + MOVD R8, 80(R1) + MOVD 80(R0), R2 + MOVD 96(R1), R16 + SUB R16, R2, R2 + MOVD R2, 120(R1) + MOVD R10, 128(R1) + MOVD 88(R0), R2 + MOVD 144(R1), R16 + SUB R16, R2, R2 + MOVD R2, 168(R1) + MOVD R12, 176(R1) MOVD 16(R0), R16 - SUB R16, R3, R3 - LSL $0x02, R3, R3 - MOVD R3, 40(R0) + SUB R16, R5, R5 + LSL $0x02, R5, R5 + MOVD R5, 40(R0) RET // func decompress4x_8b_main_loop_amd64(ctx *decompress4xContext) +// Requires: BMI, CMOV TEXT ·decompress4x_8b_main_loop_arm64(SB), $0-8 - // Preload values - MOVD ctx+0(FP), R1 - MOVBU 8(R1), R6 - MOVD 16(R1), R3 - MOVD 48(R1), R5 - MOVD 24(R1), R7 - MOVD 32(R1), R8 - MOVD (R1), R9 - - // Main loop -main_loop: - MOVD $0, R2 - CMP R5, R3 - CSET GE, R16 - BFI $0, R16, $8, R2 - - // br0.fillFast32() - MOVD 32(R9), R10 - MOVBU 40(R9), R11 - CMP $0x20, R11 - BLS skip_fill0 - MOVD 24(R9), R12 - SUB $0x20, R11, R11 - SUB $0x04, R12, R12 - MOVD (R9), R13 - - // b.value |= uint64(low) << (b.bitsRead & 63) - ADD R13, R12, R15 - MOVWU (R15), R13 - MOVD R11, R1 - LSL R1, R13, R13 - MOVD R12, 24(R9) - ORR R13, R10, R10 - - // exhausted += (br0.off < 4) - CMP $0x04, R12 - CSINC HS, R2, R2, R16 - BFI $0, R16, $8, R2 - -skip_fill0: - // val0 := br0.peekTopBits(peekBits) - MOVD R10, R12 - MOVD R6, R1 - LSR R1, R12, R12 - - // v0 := table[val0&mask] - ADD R12<<1, R8, R15 - MOVHU (R15), R1 - - // br0.advance(uint8(v0.entry) - UBFX $8, R1, $8, R16 - BFI $0, R16, $8, R0 - LSL R1, R10, R10 - ADD R1, R11, R15 - BFI $0, R15, $8, R11 - - // val1 := br0.peekTopBits(peekBits) - MOVD R10, R12 - MOVD R6, R1 - LSR R1, R12, R12 - - // v1 := table[val0&mask] - ADD R12<<1, R8, R15 - MOVHU (R15), R1 - - // br0.advance(uint8(v1.entry) - UBFX $8, R1, $8, R16 - BFI $8, R16, $8, R0 - LSL R1, R10, R10 - ADD R1, R11, R15 - BFI $0, R15, $8, R11 - REVW R0, R0 - - // val2 := br0.peekTopBits(peekBits) - MOVD R10, R12 - MOVD R6, R1 - LSR R1, R12, R12 - - // v2 := table[val0&mask] - ADD R12<<1, R8, R15 - MOVHU (R15), R1 + MOVD ctx+0(FP), R0 - // br0.advance(uint8(v2.entry) - UBFX $8, R1, $8, R16 - BFI $8, R16, $8, R0 + // Preload values + MOVBU 8(R0), R2 + MOVD 32(R0), R3 + MOVD 16(R0), R5 + MOVD 24(R0), R1 + ADD R1, R5, R7 + ADD R1<<1, R5, R9 + ADD R1<<1, R1, R11 + ADD R5, R11, R11 + + // Convert each bit reader to sentinel form: bits = value | 1<> 8) - // out[id * dstEvery + 1] = uint8(v1.entry >> 8) - // out[id * dstEvery + 3] = uint8(v2.entry >> 8) - // out[id * dstEvery + 4] = uint8(v3.entry >> 8) - MOVW R0, (R3) - - // update the bitreader structure - MOVD R10, 32(R9) - MOVB R11, 40(R9) - - // br1.fillFast32() - MOVD 80(R9), R10 - MOVBU 88(R9), R11 - CMP $0x20, R11 - BLS skip_fill1 - MOVD 72(R9), R12 - SUB $0x20, R11, R11 - SUB $0x04, R12, R12 - MOVD 48(R9), R13 - - // b.value |= uint64(low) << (b.bitsRead & 63) - ADD R13, R12, R15 - MOVWU (R15), R13 - MOVD R11, R1 - LSL R1, R13, R13 - MOVD R12, 72(R9) - ORR R13, R10, R10 - - // exhausted += (br1.off < 4) - CMP $0x04, R12 - CSINC HS, R2, R2, R16 - BFI $0, R16, $8, R2 - -skip_fill1: - // val0 := br1.peekTopBits(peekBits) - MOVD R10, R12 - MOVD R6, R1 - LSR R1, R12, R12 - - // v0 := table[val0&mask] - ADD R12<<1, R8, R15 - MOVHU (R15), R1 + RBIT R12, R13 + CLZ R13, R13 + MOVD R13, R1 + AND $0x07, R1, R1 + LSR $0x03, R13, R13 + MOVD 88(R0), R12 + SUB R13, R12, R12 + MOVD R12, 88(R0) + MOVD (R12), R12 + ORR $0x01, R12, R12 + LSL R1, R12, R12 + ADD $0x0e, R5, R5 + ADD $0x0e, R7, R7 + ADD $0x0e, R9, R9 + ADD $0x0e, R11, R11 + MOVD 96(R0), R16 + CMP R16, R5 + BLO inner_loop + JMP outer_loop + +done: + // Hand the state back: off = ip - in (negative when the window reached below the stream start), + // value = the sentinel-form container. bitReaderShifted.restoreFromAsm normalizes both. + MOVD (R0), R1 + MOVD 64(R0), R2 + MOVD (R1), R16 + SUB R16, R2, R2 + MOVD R2, 24(R1) + MOVD R6, 32(R1) + MOVD 72(R0), R2 + MOVD 48(R1), R16 + SUB R16, R2, R2 + MOVD R2, 72(R1) + MOVD R8, 80(R1) + MOVD 80(R0), R2 + MOVD 96(R1), R16 + SUB R16, R2, R2 + MOVD R2, 120(R1) + MOVD R10, 128(R1) + MOVD 88(R0), R2 + MOVD 144(R1), R16 + SUB R16, R2, R2 + MOVD R2, 168(R1) + MOVD R12, 176(R1) + MOVD 16(R0), R16 + SUB R16, R5, R5 + LSL $0x02, R5, R5 + MOVD R5, 40(R0) + RET - // br1.advance(uint8(v0.entry) - UBFX $8, R1, $8, R16 - BFI $0, R16, $8, R0 - LSL R1, R10, R10 - ADD R1, R11, R15 - BFI $0, R15, $8, R11 +// skipped decompress4x_main_loop_bmi2 (generic twin preferred on arm64) - // val1 := br1.peekTopBits(peekBits) - MOVD R10, R12 - MOVD R6, R1 - LSR R1, R12, R12 +// skipped decompress4x_8b_main_loop_bmi2 (generic twin preferred on arm64) - // v1 := table[val0&mask] - ADD R12<<1, R8, R15 - MOVHU (R15), R1 +// skipped decompress4x_4b_main_loop_bmi2 (generic twin preferred on arm64) - // br1.advance(uint8(v1.entry) - UBFX $8, R1, $8, R16 - BFI $8, R16, $8, R0 - LSL R1, R10, R10 - ADD R1, R11, R15 - BFI $0, R15, $8, R11 - REVW R0, R0 +// func decompress1x_main_loop_amd64(ctx *decompress1xContext) +// Requires: CMOV +TEXT ·decompress1x_main_loop_arm64(SB), $0-8 + MOVD ctx+0(FP), R0 - // val2 := br1.peekTopBits(peekBits) - MOVD R10, R12 + // Preload values + MOVBU 8(R0), R2 + MOVD 32(R0), R3 + MOVD 16(R0), R5 + MOVD 24(R0), R6 + ADD R5, R6, R6 + MOVD 56(R0), R7 + MOVD 48(R0), R8 + MOVD (R0), R1 + MOVD 32(R1), R9 + MOVBU 40(R1), R10 + +outer_loop: + // Iterations allowed by the output: (limit - op) / 8 MOVD R6, R1 - LSR R1, R12, R12 + SUBS R5, R1, R1 + BLE done + LSR $0x03, R1, R1 + + // Iterations allowed by the input: a refill reads the 8 bytes below the window and + // moves it down by at most 7, so every read stays inside the stream while ip stays + // at least 8 above ilowest. + MOVD R7, R11 + SUB R8, R11, R11 + LSR $0x03, R11, R11 + CMP R1, R11 + CSEL LO, R11, R1, R1 + TST R1, R1 + BEQ done + MOVD $5, R16 + MUL R16, R1, R11 + ADD R5, R11, R11 + +inner_loop: + // symbol 0 + LSR R2, R9, R12 + MOVHU (R3)(R12<<1), R1 + LSL R1, R9, R9 + ADD R1, R10, R10 + LSR $0x08, R1, R1 + MOVB R1, (R5) - // v2 := table[val0&mask] - ADD R12<<1, R8, R15 - MOVHU (R15), R1 + // symbol 1 + LSR R2, R9, R12 + MOVHU (R3)(R12<<1), R1 + LSL R1, R9, R9 + ADD R1, R10, R10 + LSR $0x08, R1, R1 + MOVB R1, 1(R5) - // br1.advance(uint8(v2.entry) - UBFX $8, R1, $8, R16 - BFI $8, R16, $8, R0 - LSL R1, R10, R10 - ADD R1, R11, R15 - BFI $0, R15, $8, R11 + // symbol 2 + LSR R2, R9, R12 + MOVHU (R3)(R12<<1), R1 + LSL R1, R9, R9 + ADD R1, R10, R10 + LSR $0x08, R1, R1 + MOVB R1, 2(R5) - // val3 := br1.peekTopBits(peekBits) - MOVD R10, R12 - MOVD R6, R1 - LSR R1, R12, R12 + // symbol 3 + LSR R2, R9, R12 + MOVHU (R3)(R12<<1), R1 + LSL R1, R9, R9 + ADD R1, R10, R10 + LSR $0x08, R1, R1 + MOVB R1, 3(R5) - // v3 := table[val0&mask] - ADD R12<<1, R8, R15 - MOVHU (R15), R1 + // symbol 4 + LSR R2, R9, R12 + MOVHU (R3)(R12<<1), R1 + LSL R1, R9, R9 + ADD R1, R10, R10 + LSR $0x08, R1, R1 + MOVB R1, 4(R5) - // br1.advance(uint8(v3.entry) - UBFX $8, R1, $8, R16 - BFI $0, R16, $8, R0 - LSL R1, R10, R10 - ADD R1, R11, R15 - BFI $0, R15, $8, R11 - REVW R0, R0 - - // these four writes get coalesced - // out[id * dstEvery + 0] = uint8(v0.entry >> 8) - // out[id * dstEvery + 1] = uint8(v1.entry >> 8) - // out[id * dstEvery + 3] = uint8(v2.entry >> 8) - // out[id * dstEvery + 4] = uint8(v3.entry >> 8) - ADD R7, R3, R15 - MOVW R0, (R15) - - // update the bitreader structure - MOVD R10, 80(R9) - MOVB R11, 88(R9) - - // br2.fillFast32() - MOVD 128(R9), R10 - MOVBU 136(R9), R11 - CMP $0x20, R11 - BLS skip_fill2 - MOVD 120(R9), R12 - SUB $0x20, R11, R11 - SUB $0x04, R12, R12 - MOVD 96(R9), R13 - - // b.value |= uint64(low) << (b.bitsRead & 63) - ADD R13, R12, R15 - MOVWU (R15), R13 - MOVD R11, R1 - LSL R1, R13, R13 - MOVD R12, 120(R9) - ORR R13, R10, R10 - - // exhausted += (br2.off < 4) - CMP $0x04, R12 - CSINC HS, R2, R2, R16 - BFI $0, R16, $8, R2 - -skip_fill2: - // val0 := br2.peekTopBits(peekBits) + // Refill the whole bytes consumed from below the window MOVD R10, R12 - MOVD R6, R1 - LSR R1, R12, R12 - - // v0 := table[val0&mask] - ADD R12<<1, R8, R15 - MOVHU (R15), R1 + AND $0x38, R12, R12 + MOVD -8(R7), R13 + LSR $0x01, R13, R13 + AND $0x07, R10, R10 + MOVD R12, R1 + EOR $0x3f, R1, R1 + LSR R1, R13, R13 + MOVD R10, R1 + LSL R1, R13, R13 + ORR R13, R9, R9 + LSR $0x03, R12, R12 + SUB R12, R7, R7 + ADD $0x05, R5, R5 + CMP R11, R5 + BLO inner_loop + JMP outer_loop + +done: + // Hand the state back in the Go bit reader's own form: off = ip - in, value, bitsRead. + MOVD (R0), R1 + MOVD (R1), R16 + SUB R16, R7, R7 + MOVD R7, 24(R1) + MOVD R9, 32(R1) + MOVB R10, 40(R1) + MOVD 16(R0), R16 + SUB R16, R5, R5 + MOVD R5, 40(R0) + RET - // br2.advance(uint8(v0.entry) - UBFX $8, R1, $8, R16 - BFI $0, R16, $8, R0 - LSL R1, R10, R10 - ADD R1, R11, R15 - BFI $0, R15, $8, R11 +// func decompress1x_8b_main_loop_amd64(ctx *decompress1xContext) +// Requires: CMOV +TEXT ·decompress1x_8b_main_loop_arm64(SB), $0-8 + MOVD ctx+0(FP), R0 - // val1 := br2.peekTopBits(peekBits) - MOVD R10, R12 + // Preload values + MOVBU 8(R0), R2 + MOVD 32(R0), R3 + MOVD 16(R0), R5 + MOVD 24(R0), R6 + ADD R5, R6, R6 + MOVD 56(R0), R7 + MOVD 48(R0), R8 + MOVD (R0), R1 + MOVD 32(R1), R9 + MOVBU 40(R1), R10 + +outer_loop: + // Iterations allowed by the output: (limit - op) / 8 MOVD R6, R1 - LSR R1, R12, R12 - - // v1 := table[val0&mask] - ADD R12<<1, R8, R15 - MOVHU (R15), R1 + SUBS R5, R1, R1 + BLE done + LSR $0x03, R1, R1 + + // Iterations allowed by the input: a refill reads the 8 bytes below the window and + // moves it down by at most 7, so every read stays inside the stream while ip stays + // at least 8 above ilowest. + MOVD R7, R11 + SUB R8, R11, R11 + LSR $0x03, R11, R11 + CMP R1, R11 + CSEL LO, R11, R1, R1 + TST R1, R1 + BEQ done + MOVD $7, R16 + MUL R16, R1, R11 + ADD R5, R11, R11 + +inner_loop: + // symbol 0 + LSR R2, R9, R12 + MOVHU (R3)(R12<<1), R1 + LSL R1, R9, R9 + ADD R1, R10, R10 + LSR $0x08, R1, R1 + MOVB R1, (R5) - // br2.advance(uint8(v1.entry) - UBFX $8, R1, $8, R16 - BFI $8, R16, $8, R0 - LSL R1, R10, R10 - ADD R1, R11, R15 - BFI $0, R15, $8, R11 - REVW R0, R0 + // symbol 1 + LSR R2, R9, R12 + MOVHU (R3)(R12<<1), R1 + LSL R1, R9, R9 + ADD R1, R10, R10 + LSR $0x08, R1, R1 + MOVB R1, 1(R5) - // val2 := br2.peekTopBits(peekBits) - MOVD R10, R12 - MOVD R6, R1 - LSR R1, R12, R12 + // symbol 2 + LSR R2, R9, R12 + MOVHU (R3)(R12<<1), R1 + LSL R1, R9, R9 + ADD R1, R10, R10 + LSR $0x08, R1, R1 + MOVB R1, 2(R5) - // v2 := table[val0&mask] - ADD R12<<1, R8, R15 - MOVHU (R15), R1 + // symbol 3 + LSR R2, R9, R12 + MOVHU (R3)(R12<<1), R1 + LSL R1, R9, R9 + ADD R1, R10, R10 + LSR $0x08, R1, R1 + MOVB R1, 3(R5) - // br2.advance(uint8(v2.entry) - UBFX $8, R1, $8, R16 - BFI $8, R16, $8, R0 - LSL R1, R10, R10 - ADD R1, R11, R15 - BFI $0, R15, $8, R11 + // symbol 4 + LSR R2, R9, R12 + MOVHU (R3)(R12<<1), R1 + LSL R1, R9, R9 + ADD R1, R10, R10 + LSR $0x08, R1, R1 + MOVB R1, 4(R5) - // val3 := br2.peekTopBits(peekBits) - MOVD R10, R12 - MOVD R6, R1 - LSR R1, R12, R12 + // symbol 5 + LSR R2, R9, R12 + MOVHU (R3)(R12<<1), R1 + LSL R1, R9, R9 + ADD R1, R10, R10 + LSR $0x08, R1, R1 + MOVB R1, 5(R5) - // v3 := table[val0&mask] - ADD R12<<1, R8, R15 - MOVHU (R15), R1 + // symbol 6 + LSR R2, R9, R12 + MOVHU (R3)(R12<<1), R1 + LSL R1, R9, R9 + ADD R1, R10, R10 + LSR $0x08, R1, R1 + MOVB R1, 6(R5) - // br2.advance(uint8(v3.entry) - UBFX $8, R1, $8, R16 - BFI $0, R16, $8, R0 - LSL R1, R10, R10 - ADD R1, R11, R15 - BFI $0, R15, $8, R11 - REVW R0, R0 - - // these four writes get coalesced - // out[id * dstEvery + 0] = uint8(v0.entry >> 8) - // out[id * dstEvery + 1] = uint8(v1.entry >> 8) - // out[id * dstEvery + 3] = uint8(v2.entry >> 8) - // out[id * dstEvery + 4] = uint8(v3.entry >> 8) - ADD R7<<1, R3, R15 - MOVW R0, (R15) - - // update the bitreader structure - MOVD R10, 128(R9) - MOVB R11, 136(R9) - - // br3.fillFast32() - MOVD 176(R9), R10 - MOVBU 184(R9), R11 - CMP $0x20, R11 - BLS skip_fill3 - MOVD 168(R9), R12 - SUB $0x20, R11, R11 - SUB $0x04, R12, R12 - MOVD 144(R9), R13 - - // b.value |= uint64(low) << (b.bitsRead & 63) - ADD R13, R12, R15 - MOVWU (R15), R13 - MOVD R11, R1 - LSL R1, R13, R13 - MOVD R12, 168(R9) - ORR R13, R10, R10 - - // exhausted += (br3.off < 4) - CMP $0x04, R12 - CSINC HS, R2, R2, R16 - BFI $0, R16, $8, R2 - -skip_fill3: - // val0 := br3.peekTopBits(peekBits) + // Refill the whole bytes consumed from below the window MOVD R10, R12 - MOVD R6, R1 - LSR R1, R12, R12 - - // v0 := table[val0&mask] - ADD R12<<1, R8, R15 - MOVHU (R15), R1 + AND $0x38, R12, R12 + MOVD -8(R7), R13 + LSR $0x01, R13, R13 + AND $0x07, R10, R10 + MOVD R12, R1 + EOR $0x3f, R1, R1 + LSR R1, R13, R13 + MOVD R10, R1 + LSL R1, R13, R13 + ORR R13, R9, R9 + LSR $0x03, R12, R12 + SUB R12, R7, R7 + ADD $0x07, R5, R5 + CMP R11, R5 + BLO inner_loop + JMP outer_loop + +done: + // Hand the state back in the Go bit reader's own form: off = ip - in, value, bitsRead. + MOVD (R0), R1 + MOVD (R1), R16 + SUB R16, R7, R7 + MOVD R7, 24(R1) + MOVD R9, 32(R1) + MOVB R10, 40(R1) + MOVD 16(R0), R16 + SUB R16, R5, R5 + MOVD R5, 40(R0) + RET - // br3.advance(uint8(v0.entry) - UBFX $8, R1, $8, R16 - BFI $0, R16, $8, R0 - LSL R1, R10, R10 - ADD R1, R11, R15 - BFI $0, R15, $8, R11 +// func decompress1x_4b_main_loop_amd64(ctx *decompress1xContext) +// Requires: CMOV +TEXT ·decompress1x_4b_main_loop_arm64(SB), $0-8 + MOVD ctx+0(FP), R0 - // val1 := br3.peekTopBits(peekBits) - MOVD R10, R12 + // Preload values + MOVBU 8(R0), R2 + MOVD 32(R0), R3 + MOVD 16(R0), R5 + MOVD 24(R0), R6 + ADD R5, R6, R6 + MOVD 56(R0), R7 + MOVD 48(R0), R8 + MOVD (R0), R1 + MOVD 32(R1), R9 + MOVBU 40(R1), R10 + +outer_loop: + // Iterations allowed by the output: (limit - op) / 16 MOVD R6, R1 - LSR R1, R12, R12 - - // v1 := table[val0&mask] - ADD R12<<1, R8, R15 - MOVHU (R15), R1 - - // br3.advance(uint8(v1.entry) - UBFX $8, R1, $8, R16 - BFI $8, R16, $8, R0 - LSL R1, R10, R10 - ADD R1, R11, R15 - BFI $0, R15, $8, R11 - REVW R0, R0 + SUBS R5, R1, R1 + BLE done + LSR $0x04, R1, R1 + + // Iterations allowed by the input: a refill reads the 8 bytes below the window and + // moves it down by at most 7, so every read stays inside the stream while ip stays + // at least 8 above ilowest. + MOVD R7, R11 + SUB R8, R11, R11 + LSR $0x03, R11, R11 + CMP R1, R11 + CSEL LO, R11, R1, R1 + TST R1, R1 + BEQ done + MOVD $14, R16 + MUL R16, R1, R11 + ADD R5, R11, R11 + +inner_loop: + // symbol 0 + LSR R2, R9, R12 + MOVHU (R3)(R12<<1), R1 + LSL R1, R9, R9 + ADD R1, R10, R10 + LSR $0x08, R1, R1 + MOVB R1, (R5) - // val2 := br3.peekTopBits(peekBits) - MOVD R10, R12 - MOVD R6, R1 - LSR R1, R12, R12 + // symbol 1 + LSR R2, R9, R12 + MOVHU (R3)(R12<<1), R1 + LSL R1, R9, R9 + ADD R1, R10, R10 + LSR $0x08, R1, R1 + MOVB R1, 1(R5) - // v2 := table[val0&mask] - ADD R12<<1, R8, R15 - MOVHU (R15), R1 + // symbol 2 + LSR R2, R9, R12 + MOVHU (R3)(R12<<1), R1 + LSL R1, R9, R9 + ADD R1, R10, R10 + LSR $0x08, R1, R1 + MOVB R1, 2(R5) - // br3.advance(uint8(v2.entry) - UBFX $8, R1, $8, R16 - BFI $8, R16, $8, R0 - LSL R1, R10, R10 - ADD R1, R11, R15 - BFI $0, R15, $8, R11 + // symbol 3 + LSR R2, R9, R12 + MOVHU (R3)(R12<<1), R1 + LSL R1, R9, R9 + ADD R1, R10, R10 + LSR $0x08, R1, R1 + MOVB R1, 3(R5) - // val3 := br3.peekTopBits(peekBits) - MOVD R10, R12 - MOVD R6, R1 - LSR R1, R12, R12 + // symbol 4 + LSR R2, R9, R12 + MOVHU (R3)(R12<<1), R1 + LSL R1, R9, R9 + ADD R1, R10, R10 + LSR $0x08, R1, R1 + MOVB R1, 4(R5) - // v3 := table[val0&mask] - ADD R12<<1, R8, R15 - MOVHU (R15), R1 + // symbol 5 + LSR R2, R9, R12 + MOVHU (R3)(R12<<1), R1 + LSL R1, R9, R9 + ADD R1, R10, R10 + LSR $0x08, R1, R1 + MOVB R1, 5(R5) - // br3.advance(uint8(v3.entry) - UBFX $8, R1, $8, R16 - BFI $0, R16, $8, R0 - LSL R1, R10, R10 - ADD R1, R11, R15 - BFI $0, R15, $8, R11 - REVW R0, R0 - - // these four writes get coalesced - // out[id * dstEvery + 0] = uint8(v0.entry >> 8) - // out[id * dstEvery + 1] = uint8(v1.entry >> 8) - // out[id * dstEvery + 3] = uint8(v2.entry >> 8) - // out[id * dstEvery + 4] = uint8(v3.entry >> 8) - ADD R7<<1, R7, R1 - ADD R1, R3, R15 - MOVW R0, (R15) - - // update the bitreader structure - MOVD R10, 176(R9) - MOVB R11, 184(R9) - ADD $0x04, R3, R3 - AND $0xff, R2, R15 - AND $0xff, R2, R16 - TST R16, R15 - BEQ main_loop - MOVD ctx+0(FP), R0 - MOVD 16(R0), R16 - SUB R16, R3, R3 - LSL $0x02, R3, R3 - MOVD R3, 40(R0) - RET + // symbol 6 + LSR R2, R9, R12 + MOVHU (R3)(R12<<1), R1 + LSL R1, R9, R9 + ADD R1, R10, R10 + LSR $0x08, R1, R1 + MOVB R1, 6(R5) -// func decompress1x_main_loop_amd64(ctx *decompress1xContext) -TEXT ·decompress1x_main_loop_arm64(SB), $0-8 - MOVD ctx+0(FP), R1 - MOVD 16(R1), R2 - MOVD 24(R1), R3 - CMP $0x04, R3 - BLO error_max_decoded_size_exceeded - ADD R3, R2, R3 - MOVD (R1), R5 - MOVD (R5), R7 - MOVD 24(R5), R8 - MOVD 32(R5), R9 - MOVBU 40(R5), R10 - MOVD 32(R1), R5 - MOVBU 8(R1), R6 - JMP loop_condition - -main_loop: - // Check if we have room for 4 bytes in the output buffer - ADD $4, R2, R1 - CMP R3, R1 - BGE error_max_decoded_size_exceeded - - // Decode 4 values - CMP $0x20, R10 - BLT bitReader_fillFast_1_end - SUB $0x20, R10, R10 - SUB $0x04, R8, R8 - ADD R8, R7, R15 - MOVWU (R15), R11 - MOVD R10, R1 - LSL R1, R11, R11 - ORR R11, R9, R9 - -bitReader_fillFast_1_end: - MOVD R6, R1 - MOVD R9, R11 - LSR R1, R11, R11 - ADD R11<<1, R5, R15 - MOVHU (R15), R1 - UBFX $8, R1, $8, R16 - BFI $0, R16, $8, R0 - MOVBU R1, R1 + // symbol 7 + LSR R2, R9, R12 + MOVHU (R3)(R12<<1), R1 + LSL R1, R9, R9 ADD R1, R10, R10 + LSR $0x08, R1, R1 + MOVB R1, 7(R5) + + // symbol 8 + LSR R2, R9, R12 + MOVHU (R3)(R12<<1), R1 LSL R1, R9, R9 - MOVD R6, R1 - MOVD R9, R11 - LSR R1, R11, R11 - ADD R11<<1, R5, R15 - MOVHU (R15), R1 - UBFX $8, R1, $8, R16 - BFI $8, R16, $8, R0 - MOVBU R1, R1 ADD R1, R10, R10 + LSR $0x08, R1, R1 + MOVB R1, 8(R5) + + // symbol 9 + LSR R2, R9, R12 + MOVHU (R3)(R12<<1), R1 LSL R1, R9, R9 - REVW R0, R0 - CMP $0x20, R10 - BLT bitReader_fillFast_2_end - SUB $0x20, R10, R10 - SUB $0x04, R8, R8 - ADD R8, R7, R15 - MOVWU (R15), R11 - MOVD R10, R1 - LSL R1, R11, R11 - ORR R11, R9, R9 - -bitReader_fillFast_2_end: - MOVD R6, R1 - MOVD R9, R11 - LSR R1, R11, R11 - ADD R11<<1, R5, R15 - MOVHU (R15), R1 - UBFX $8, R1, $8, R16 - BFI $8, R16, $8, R0 - MOVBU R1, R1 ADD R1, R10, R10 + LSR $0x08, R1, R1 + MOVB R1, 9(R5) + + // symbol 10 + LSR R2, R9, R12 + MOVHU (R3)(R12<<1), R1 LSL R1, R9, R9 - MOVD R6, R1 - MOVD R9, R11 - LSR R1, R11, R11 - ADD R11<<1, R5, R15 - MOVHU (R15), R1 - UBFX $8, R1, $8, R16 - BFI $0, R16, $8, R0 - MOVBU R1, R1 ADD R1, R10, R10 + LSR $0x08, R1, R1 + MOVB R1, 10(R5) + + // symbol 11 + LSR R2, R9, R12 + MOVHU (R3)(R12<<1), R1 LSL R1, R9, R9 - REVW R0, R0 + ADD R1, R10, R10 + LSR $0x08, R1, R1 + MOVB R1, 11(R5) - // Store the decoded values - MOVW R0, (R2) - ADD $0x04, R2, R2 + // symbol 12 + LSR R2, R9, R12 + MOVHU (R3)(R12<<1), R1 + LSL R1, R9, R9 + ADD R1, R10, R10 + LSR $0x08, R1, R1 + MOVB R1, 12(R5) -loop_condition: - CMP $0x08, R8 - BGE main_loop + // symbol 13 + LSR R2, R9, R12 + MOVHU (R3)(R12<<1), R1 + LSL R1, R9, R9 + ADD R1, R10, R10 + LSR $0x08, R1, R1 + MOVB R1, 13(R5) - // Update ctx structure - MOVD ctx+0(FP), R0 + // Refill the whole bytes consumed from below the window + MOVD R10, R12 + AND $0x38, R12, R12 + MOVD -8(R7), R13 + LSR $0x01, R13, R13 + AND $0x07, R10, R10 + MOVD R12, R1 + EOR $0x3f, R1, R1 + LSR R1, R13, R13 + MOVD R10, R1 + LSL R1, R13, R13 + ORR R13, R9, R9 + LSR $0x03, R12, R12 + SUB R12, R7, R7 + ADD $0x0e, R5, R5 + CMP R11, R5 + BLO inner_loop + JMP outer_loop + +done: + // Hand the state back in the Go bit reader's own form: off = ip - in, value, bitsRead. + MOVD (R0), R1 + MOVD (R1), R16 + SUB R16, R7, R7 + MOVD R7, 24(R1) + MOVD R9, 32(R1) + MOVB R10, 40(R1) MOVD 16(R0), R16 - SUB R16, R2, R2 - MOVD R2, 40(R0) - MOVD (R0), R0 - MOVD R8, 24(R0) - MOVD R9, 32(R0) - MOVB R10, 40(R0) - RET - - // Report error -error_max_decoded_size_exceeded: - MOVD ctx+0(FP), R0 - MOVD $-1, R1 - MOVD R1, 40(R0) + SUB R16, R5, R5 + MOVD R5, 40(R0) RET // skipped decompress1x_main_loop_bmi2 (generic twin preferred on arm64) + +// skipped decompress1x_8b_main_loop_bmi2 (generic twin preferred on arm64) + +// skipped decompress1x_4b_main_loop_bmi2 (generic twin preferred on arm64) diff --git a/vendor/github.com/klauspost/compress/huff0/decompress_asm.go b/vendor/github.com/klauspost/compress/huff0/decompress_asm.go index 4854dd43653e..71a80bbd5572 100644 --- a/vendor/github.com/klauspost/compress/huff0/decompress_asm.go +++ b/vendor/github.com/klauspost/compress/huff0/decompress_asm.go @@ -14,6 +14,10 @@ import ( // fallback8BitSize is the size where using Go version is faster. const fallback8BitSize = 800 +// decompress4xContext is the argument block of the Decompress4X asm loops. +// Go fills every field but decoded and inner; the asm advances ip, and on +// return leaves each bit reader in the form bitReaderShifted.restoreFromAsm +// expects. type decompress4xContext struct { pbr *[4]bitReaderShifted peekBits uint8 @@ -21,9 +25,20 @@ type decompress4xContext struct { dstEvery int tbl *dEntrySingle decoded int - limit *byte + limit *byte // stream 0's output pointer must stay below this + ilowest *byte // start of the input block; no read goes below it + ip [4]*byte // each stream's 8-byte input window, advanced by the asm + inner *byte // scratch for the asm: the current inner-loop limit } +// Symbols decoded per stream between reloads by the 4X asm loops; must +// match the constants of the same names in _generate/gen.go. +const ( + fastSymbols = 5 // tablelog 9..11 + fast8bSymbols = 7 // tablelog 5..8 + fast4bSymbols = 14 // tablelog <= 4 +) + // Decompress4X will decompress a 4X encoded stream. // The length of the supplied input must match the end of a block exactly. // The *capacity* of the dst slice must match the destination size of @@ -72,23 +87,55 @@ func (d *Decoder) Decompress4X(dst, src []byte) ([]byte, error) { var decoded int - if len(out) > 4*4 && !(br[0].off < 4 || br[1].off < 4 || br[2].off < 4 || br[3].off < 4) { + nSyms := fastSymbols + if d.actualTableLog <= 4 { + nSyms = fast4bSymbols + } else if use8BitTables { + nSyms = fast8bSymbols + } + // The asm writes nSyms bytes per stream per iteration and only re-checks + // its bounds between batches of iterations (the batch size is derived + // from limit in _generate/gen.go), so stream 0 must stop early enough + // that stream 3 (which may be up to 3 bytes shorter than dstEvery) + // never writes past the end of out. Every stream needs a full 8-byte + // window ahead of its read pointer to enter the loop. + if limit := dstEvery - nSyms - 2; limit > 0 && br[0].canUseAsm() && br[1].canUseAsm() && br[2].canUseAsm() && br[3].canUseAsm() { + for i := range br { + br[i].prepareForAsm() + } ctx := decompress4xContext{ pbr: &br, peekBits: uint8((64 - d.actualTableLog) & 63), // see: bitReaderShifted.peekBitsFast() out: &out[0], dstEvery: dstEvery, tbl: &single[0], - limit: &out[dstEvery-4], // Always stop decoding when first buffer gets here to avoid writing OOB on last. + limit: &out[limit], + // The 6-byte jump table sits below stream 0 inside src, so the + // lowest window may reach into it; restoreFromAsm accounts for + // bytes below a stream's start. Bounding by src rather than by + // stream 0 keeps the asm running about six bytes longer, which + // matters for streams with very short codes. + ilowest: &src[0], } - if use8BitTables { + for i := range br { + ctx.ip[i] = &br[i].in[br[i].off] + } + switch nSyms { + case fast4bSymbols: + decompress4x_4b_main_loop_asm(&ctx) + case fast8bSymbols: decompress4x_8b_main_loop_asm(&ctx) - } else { + default: decompress4x_main_loop_asm(&ctx) } decoded = ctx.decoded out = out[decoded/4:] + for i := range br { + if err := br[i].restoreFromAsm(); err != nil { + return nil, err + } + } } // Decode remaining. @@ -128,18 +175,20 @@ func (d *Decoder) Decompress4X(dst, src []byte) ([]byte, error) { return dst, nil } +// decompress1xContext is the argument block of the Decompress1X asm loops. +// Go fills every field but decoded; the asm advances ip, and on return +// leaves the bit reader in the form bitReaderShifted.restoreFromAsm expects. type decompress1xContext struct { pbr *bitReaderShifted peekBits uint8 out *byte - outCap int + outCap int // no write reaches out[outCap] tbl *dEntrySingle decoded int + ilowest *byte // start of the stream; no read goes below it + ip *byte // the 8-byte input window, advanced by the asm } -// Error reported by asm implementations -const error_max_decoded_size_exeeded = -1 - // Decompress1X will decompress a 1X encoded stream. // The cap of the output buffer will be the maximum decompressed size. // The length of the supplied input must match the end of a block exactly. @@ -158,25 +207,36 @@ func (d *Decoder) Decompress1X(dst, src []byte) ([]byte, error) { const tlSize = 1 << tableLogMax const tlMask = tlSize - 1 - if maxDecodedSize >= 4 { + // The asm decodes whole batches of symbols and only re-checks its + // bounds between them, so it needs at least one batch of room (the + // output bound rounds nSyms up to 16, see _generate/gen.go) and a full + // 8-byte window ahead of the read pointer to enter the loop. It also + // needs at most 7 bits consumed on entry so that a batch cannot run + // the container dry, which prepareForAsm establishes. + decoded := 0 + if maxDecodedSize >= 16 && br.canUseAsm() { + br.prepareForAsm() ctx := decompress1xContext{ pbr: &br, out: &dst[0], outCap: maxDecodedSize, peekBits: uint8((64 - d.actualTableLog) & 63), // see: bitReaderShifted.peekBitsFast() tbl: &d.dt.single[0], + ilowest: &src[0], + ip: &br.in[br.off], } - - decompress1x_main_loop_asm(&ctx) - if ctx.decoded == error_max_decoded_size_exeeded { - return nil, ErrMaxDecodedSizeExceeded + if d.actualTableLog <= 4 { + decompress1x_4b_main_loop_asm(&ctx) + } else if d.actualTableLog <= 8 { + decompress1x_8b_main_loop_asm(&ctx) + } else { + decompress1x_main_loop_asm(&ctx) } - - dst = dst[:ctx.decoded] + decoded = ctx.decoded } + dst = dst[:decoded] - // br < 8, so uint8 is fine - bitsLeft := uint8(br.off)*8 + 64 - br.bitsRead + bitsLeft := br.remaining() for bitsLeft > 0 { br.fill() if len(dst) >= maxDecodedSize { @@ -186,7 +246,7 @@ func (d *Decoder) Decompress1X(dst, src []byte) ([]byte, error) { v := d.dt.single[br.peekBitsFast(d.actualTableLog)&tlMask] nBits := uint8(v.entry) br.advance(nBits) - bitsLeft -= nBits + bitsLeft -= uint(nBits) dst = append(dst, uint8(v.entry>>8)) } return dst, br.close() diff --git a/vendor/github.com/klauspost/compress/zstd/blockdec.go b/vendor/github.com/klauspost/compress/zstd/blockdec.go index 51f9da03875b..89b20d6a8421 100644 --- a/vendor/github.com/klauspost/compress/zstd/blockdec.go +++ b/vendor/github.com/klauspost/compress/zstd/blockdec.go @@ -240,9 +240,11 @@ func (b *blockDec) decodeBuf(hist *history) error { b.dst[i] = v } hist.appendKeep(b.dst) + hist.decoders.consumeSyncLen(len(b.dst)) return nil case blockTypeRaw: hist.appendKeep(b.data) + hist.decoders.consumeSyncLen(len(b.data)) return nil case blockTypeCompressed: saved := b.dst @@ -487,6 +489,7 @@ func (b *blockDec) decodeCompressed(hist *history) error { } if hist.decoders.nSeqs == 0 { b.dst = append(b.dst, hist.decoders.literals...) + hist.decoders.consumeSyncLen(len(hist.decoders.literals)) return nil } before := len(hist.decoders.out) diff --git a/vendor/github.com/klauspost/compress/zstd/decoder.go b/vendor/github.com/klauspost/compress/zstd/decoder.go index c7e500f02a95..90ef3bb03f4a 100644 --- a/vendor/github.com/klauspost/compress/zstd/decoder.go +++ b/vendor/github.com/klauspost/compress/zstd/decoder.go @@ -7,8 +7,11 @@ package zstd import ( "context" "encoding/binary" + "errors" "io" + "maps" "sync" + "sync/atomic" "github.com/klauspost/compress/zstd/internal/xxhash" ) @@ -20,7 +23,8 @@ import ( // A decoder can safely be re-used even if the previous stream failed. // To release the resources, you must call the Close() function on a decoder. type Decoder struct { - o decoderOptions + // publishedOptions contains the current immutable decoder options. + publishedOptions atomic.Pointer[decoderOptions] // Unreferenced decoders, ready for use. decoders chan *blockDec @@ -84,9 +88,10 @@ var ( func NewReader(r io.Reader, opts ...DOption) (*Decoder, error) { initPredefined() var d Decoder - d.o.setDefault() - for _, o := range opts { - err := o(&d.o) + var o decoderOptions + o.setDefault() + for _, opt := range opts { + err := opt(&o) if err != nil { return nil, err } @@ -99,17 +104,19 @@ func NewReader(r io.Reader, opts ...DOption) (*Decoder, error) { } // Initialize dict map if needed. - if d.o.dicts == nil { - d.o.dicts = make(map[uint32]*dict) + if o.dicts == nil { + o.dicts = make(map[uint32]*dict) } // Create decoders - d.decoders = make(chan *blockDec, d.o.concurrent) - for i := 0; i < d.o.concurrent; i++ { - dec := newBlockDec(d.o.lowMem) - dec.localFrame = newFrameDec(d.o) + d.decoders = make(chan *blockDec, o.concurrent) + for i := 0; i < o.concurrent; i++ { + dec := newBlockDec(o.lowMem) + dec.localFrame = newFrameDec(o) + dec.localFrame.o.dicts = nil d.decoders <- dec } + d.publishedOptions.Store(&o) if r == nil { return &d, nil @@ -171,6 +178,9 @@ func (d *Decoder) Reset(r io.Reader) error { d.drainOutput() d.syncStream.br.r = nil + if d.frame != nil { + d.frame.o.dicts = nil + } if r == nil { d.current.err = ErrDecoderNilInput if len(d.current.b) > 0 { @@ -180,8 +190,10 @@ func (d *Decoder) Reset(r io.Reader) error { return nil } + o := *d.publishedOptions.Load() + // If bytes buffer and < 5MB, do sync decoding anyway. - if bb, ok := r.(byter); ok && bb.Len() < d.o.decodeBufsBelow && !d.o.limitToCap { + if bb, ok := r.(byter); ok && bb.Len() < o.decodeBufsBelow && !o.limitToCap { bb2 := bb if debugDecoder { println("*bytes.Buffer detected, doing sync decode, len:", bb.Len()) @@ -193,18 +205,30 @@ func (d *Decoder) Reset(r io.Reader) error { } dst, err := d.DecodeAll(b, dst) - if err == nil { - err = io.EOF + if !errors.Is(err, ErrDecoderSizeExceeded) { + if err == nil { + err = io.EOF + } + // Save output buffer + d.syncStream.dstBuf = dst + d.current.b = dst + d.current.err = err + d.current.flushed = true + if debugDecoder { + println("sync decode to", len(dst), "bytes, err:", err) + } + return nil } - // Save output buffer - d.syncStream.dstBuf = dst - d.current.b = dst - d.current.err = err - d.current.flushed = true + // The output does not fit WithDecoderMaxMemory, which caps the total + // decoded size when decoding in memory. A Reader follows the + // streaming contract, where the option caps the window instead, so + // decode this input as a stream after all; the shortcut has not + // consumed the reader. Drop the oversized buffer rather than keep it + // for reuse. if debugDecoder { - println("sync decode to", len(dst), "bytes, err:", err) + println("sync decode exceeded max memory after", len(dst), "bytes; streaming instead") } - return nil + d.syncStream.dstBuf = nil } // Remove current block. d.stashDecoder() @@ -217,14 +241,16 @@ func (d *Decoder) Reset(r io.Reader) error { // Ensure no-one else is still running... d.streamWg.Wait() if d.frame == nil { - d.frame = newFrameDec(d.o) + d.frame = newFrameDec(o) + } else { + d.frame.setOptions(o) } - if d.o.concurrent == 1 { + if o.concurrent == 1 { return d.startSyncDecoder(r) } - d.current.output = make(chan decodeOutput, d.o.concurrent) + d.current.output = make(chan decodeOutput, o.concurrent) ctx, cancel := context.WithCancel(context.Background()) d.current.cancel = cancel d.streamWg.Add(1) @@ -238,13 +264,21 @@ func (d *Decoder) Reset(r io.Reader) error { // Options are applied on top of the existing options. // Some options cannot be changed on reset and will return an error. func (d *Decoder) ResetWithOptions(r io.Reader, opts ...DOption) error { - d.o.resetOpt = true - defer func() { d.o.resetOpt = false }() + if d.current.err == ErrDecoderClosed { + return d.current.err + } + next := *d.publishedOptions.Load() + next.dicts = maps.Clone(next.dicts) + next.resetOpt = true for _, o := range opts { - if err := o(&d.o); err != nil { + if err := o(&next); err != nil { + next.resetOpt = false + d.publishedOptions.Store(&next) return err } } + next.resetOpt = false + d.publishedOptions.Store(&next) return d.Reset(r) } @@ -324,6 +358,8 @@ func (d *Decoder) DecodeAll(input, dst []byte) ([]byte, error) { // Grab a block decoder and frame decoder. block := <-d.decoders frame := block.localFrame + o := *d.publishedOptions.Load() + frame.setOptions(o) initialSize := len(dst) defer func() { if debugDecoder { @@ -335,6 +371,7 @@ func (d *Decoder) DecodeAll(input, dst []byte) ([]byte, error) { frame.history.decoders.br.in = nil frame.history.decoders.br.cursor = 0 } + frame.o.dicts = nil d.decoders <- block }() frame.bBuf = input @@ -354,20 +391,20 @@ func (d *Decoder) DecodeAll(input, dst []byte) ([]byte, error) { if err = d.setDict(frame); err != nil { return nil, err } - if frame.WindowSize > d.o.maxWindowSize { + if frame.WindowSize > frame.o.maxWindowSize { if debugDecoder { - println("window size exceeded:", frame.WindowSize, ">", d.o.maxWindowSize) + println("window size exceeded:", frame.WindowSize, ">", frame.o.maxWindowSize) } return dst, ErrWindowSizeExceeded } if frame.FrameContentSize != fcsUnknown { - if frame.FrameContentSize > d.o.maxDecodedSize-uint64(len(dst)-initialSize) { + if frame.FrameContentSize > frame.o.maxDecodedSize-uint64(len(dst)-initialSize) { if debugDecoder { - println("decoder size exceeded; fcs:", frame.FrameContentSize, "> mcs:", d.o.maxDecodedSize-uint64(len(dst)-initialSize), "len:", len(dst)) + println("decoder size exceeded; fcs:", frame.FrameContentSize, "> mcs:", frame.o.maxDecodedSize-uint64(len(dst)-initialSize), "len:", len(dst)) } return dst, ErrDecoderSizeExceeded } - if d.o.limitToCap && frame.FrameContentSize > uint64(cap(dst)-len(dst)) { + if frame.o.limitToCap && frame.FrameContentSize > uint64(cap(dst)-len(dst)) { if debugDecoder { println("decoder size exceeded; fcs:", frame.FrameContentSize, "> (cap-len)", cap(dst)-len(dst)) } @@ -380,14 +417,14 @@ func (d *Decoder) DecodeAll(input, dst []byte) ([]byte, error) { } } - if cap(dst) == 0 && !d.o.limitToCap { + if cap(dst) == 0 && !frame.o.limitToCap { // Allocate len(input) * 2 by default if nothing is provided // and we didn't get frame content size. size := min( // Cap to 1 MB. len(input)*2, 1<<20) - if uint64(size) > d.o.maxDecodedSize { - size = int(d.o.maxDecodedSize) + if uint64(size) > frame.o.maxDecodedSize { + size = int(frame.o.maxDecodedSize) } dst = make([]byte, 0, size) } @@ -396,7 +433,7 @@ func (d *Decoder) DecodeAll(input, dst []byte) ([]byte, error) { if err != nil { return dst, err } - if uint64(len(dst)-initialSize) > d.o.maxDecodedSize { + if uint64(len(dst)-initialSize) > frame.o.maxDecodedSize { return dst, ErrDecoderSizeExceeded } if len(frame.bBuf) == 0 { @@ -459,7 +496,7 @@ func (d *Decoder) nextBlock(blocking bool) (ok bool) { println("got", len(d.current.b), "bytes, error:", d.current.err, "data crc:", tmp) } - if d.o.ignoreChecksum { + if d.frame.o.ignoreChecksum { return true } @@ -497,7 +534,7 @@ func (d *Decoder) nextBlockSync() (ok bool) { if d.current.err != nil { return false } - if d.frame.WindowSize > d.o.maxDecodedSize || d.frame.WindowSize > d.o.maxWindowSize { + if d.frame.WindowSize > d.frame.o.maxDecodedSize || d.frame.WindowSize > d.frame.o.maxWindowSize { d.current.err = ErrDecoderSizeExceeded return false } @@ -546,11 +583,11 @@ func (d *Decoder) nextBlockSync() (ok bool) { // Update/Check CRC if d.frame.HasCheckSum { - if !d.o.ignoreChecksum { + if !d.frame.o.ignoreChecksum { d.frame.crc.Write(d.current.b) } if d.current.d.Last { - if !d.o.ignoreChecksum { + if !d.frame.o.ignoreChecksum { d.current.err = d.frame.checkCRC() } else { d.current.err = d.frame.consumeCRC() @@ -656,8 +693,8 @@ func (d *Decoder) startStreamDecoder(ctx context.Context, r io.Reader, output ch defer d.streamWg.Done() br := readerWrapper{r: r} - var seqDecode = make(chan *blockDec, d.o.concurrent) - var seqExecute = make(chan *blockDec, d.o.concurrent) + var seqDecode = make(chan *blockDec, d.frame.o.concurrent) + var seqExecute = make(chan *blockDec, d.frame.o.concurrent) // Async 1: Decode sequences... go func() { @@ -858,9 +895,9 @@ decodeStream: if err == nil { err = d.setDict(frame) } - if err == nil && d.frame.WindowSize > d.o.maxWindowSize { + if err == nil && d.frame.WindowSize > d.frame.o.maxWindowSize { if debugDecoder { - println("decoder size exceeded, fws:", d.frame.WindowSize, "> mws:", d.o.maxWindowSize) + println("decoder size exceeded, fws:", d.frame.WindowSize, "> mws:", d.frame.o.maxWindowSize) } err = ErrDecoderSizeExceeded @@ -940,7 +977,7 @@ decodeStream: } func (d *Decoder) setDict(frame *frameDec) (err error) { - dict, ok := d.o.dicts[frame.DictionaryID] + dict, ok := frame.o.dicts[frame.DictionaryID] if ok { if debugDecoder { println("setting dict", frame.DictionaryID) diff --git a/vendor/github.com/klauspost/compress/zstd/framedec.go b/vendor/github.com/klauspost/compress/zstd/framedec.go index d88f067e5c2e..81d8b9c922a5 100644 --- a/vendor/github.com/klauspost/compress/zstd/framedec.go +++ b/vendor/github.com/klauspost/compress/zstd/framedec.go @@ -49,13 +49,16 @@ const ( ) func newFrameDec(o decoderOptions) *frameDec { + d := frameDec{} + d.setOptions(o) + return &d +} + +func (d *frameDec) setOptions(o decoderOptions) { if o.maxWindowSize > o.maxDecodedSize { o.maxWindowSize = o.maxDecodedSize } - d := frameDec{ - o: o, - } - return &d + d.o = o } // reset will read the frame header and prepare for block decoding. diff --git a/vendor/github.com/klauspost/compress/zstd/fse_decoder_arm64.s b/vendor/github.com/klauspost/compress/zstd/fse_decoder_arm64.s index 77ee3913f056..8ca913eaec69 100644 --- a/vendor/github.com/klauspost/compress/zstd/fse_decoder_arm64.s +++ b/vendor/github.com/klauspost/compress/zstd/fse_decoder_arm64.s @@ -1,7 +1,7 @@ // Code generated by command: go run gen_fse.go -out ../fse_decoder.s -arch amd64,arm64 -pkg=zstd. DO NOT EDIT. // EXPERIMENTAL arm64 output lowered from an amd64 avo program. -//go:build arm64 && !appengine && !noasm && gc && !noasm +//go:build arm64 && (!appengine && !noasm && gc && !noasm) // func buildDtable_asm(s *fseDecoder, ctx *buildDtableAsmContext) int TEXT ·buildDtable_asm(SB), $0-24 @@ -26,11 +26,10 @@ TEXT ·buildDtable_asm(SB), $0-24 JMP init_main_loop_condition init_main_loop: - ADD R8<<1, R1, R15 - MOVH (R15), R9 - AND $0xffff, R9, R15 + MOVH (R1)(R8<<1), R9 MOVD $-1, R16 AND $0xffff, R16, R16 + AND $0xffff, R9, R15 CMP R16, R15 BNE do_not_update_high_threshold ADD R7<<3, R5, R15 @@ -39,8 +38,7 @@ init_main_loop: MOVD $0x0000000000000001, R9 do_not_update_high_threshold: - ADD R8<<1, R3, R15 - MOVH R9, (R15) + MOVH R9, (R3)(R8<<1) ADD $1, R8, R8 init_main_loop_condition: @@ -49,12 +47,10 @@ init_main_loop_condition: // Spread symbols // Calculate table step - MOVD R0, R8 - LSR $0x01, R8, R8 - MOVD R0, R9 - LSR $0x03, R9, R9 - ADD R9, R8, R8 - ADD $3, R8, R8 + LSR $0x01, R0, R8 + LSR $0x03, R0, R9 + ADD R9, R8, R8 + ADD $3, R8, R8 // Fill add bits values SUB $1, R0, R9 @@ -64,8 +60,7 @@ init_main_loop_condition: spread_main_loop: MOVD $0, R12 - ADD R11<<1, R1, R15 - MOVH (R15), R13 + MOVH (R1)(R11<<1), R13 JMP spread_inner_loop_condition spread_inner_loop: @@ -102,11 +97,9 @@ spread_check_ok: build_table_main_table: ADD R6<<3, R5, R15 MOVBU 1(R15), R1 - ADD R1<<1, R3, R15 - MOVHU (R15), R7 + MOVHU (R3)(R1<<1), R7 ADD $1, R7, R8 - ADD R1<<1, R3, R15 - MOVH R8, (R15) + MOVH R8, (R3)(R1<<1) MOVD R7, R8 CLZ R8, R16 MOVD $63, R8 @@ -129,12 +122,10 @@ build_table_main_table: RET build_table_check1_ok: - AND $0xff, R1, R15 - AND $0xff, R1, R16 - TST R16, R15 + TST $0xff, R1 BNE build_table_check2_ok - AND $0xffff, R7, R15 AND $0xffff, R6, R16 + AND $0xffff, R7, R15 CMP R16, R15 BNE build_table_check2_ok MOVD ctx+8(FP), R0 diff --git a/vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash.go b/vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash.go index fc40c820016c..9c0cbc1681db 100644 --- a/vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash.go +++ b/vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash.go @@ -52,6 +52,11 @@ func (d *Digest) Reset() { d.n = 0 } +// maxAsmSize bounds the input handed to a single writeBlocks call. Assembly is +// never preemptible, so an unbounded call holds every P in stop-the-world for +// its duration. Whole 32-byte blocks, so Write can feed it in pieces. +const maxAsmSize = 128 << 10 // 4096 whole 32-byte blocks + // Size always returns 8 bytes. func (d *Digest) Size() int { return 8 } @@ -83,6 +88,13 @@ func (d *Digest) Write(b []byte) (n int, err error) { d.n = 0 } + for len(b) >= maxAsmSize { + // Assembly is not preemptible, so hashing a large buffer in one call + // blocks every stop-the-world for the whole call. Feed it in chunks. + writeBlocks(d, b[:maxAsmSize]) + b = b[maxAsmSize:] + } + if len(b) >= 32 { // One or more full blocks left. nw := writeBlocks(d, b) diff --git a/vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash_amd64.s b/vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash_amd64.s index ddb63aa91b14..e4c688efd32a 100644 --- a/vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash_amd64.s +++ b/vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash_amd64.s @@ -175,7 +175,11 @@ finalize: RET // func writeBlocks(d *Digest, b []byte) int -TEXT ·writeBlocks(SB), NOSPLIT|NOFRAME, $0-40 +// +// Deliberately not NOSPLIT: the stack check is the preemption point that lets a +// stop-the-world proceed between chunks. The frame must be >= abi.StackSmall +// (128), or the assembler marks this leaf NOSPLIT anyway and drops that check. +TEXT ·writeBlocks(SB), $128-40 // Load fixed primes needed for round. MOVQ ·primes+0(SB), prime1 MOVQ ·primes+8(SB), prime2 diff --git a/vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash_arm64.s b/vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash_arm64.s index ae7d4d3295a4..e88cc2fc9172 100644 --- a/vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash_arm64.s +++ b/vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash_arm64.s @@ -163,7 +163,11 @@ finalize: RET // func writeBlocks(s *Digest, b []byte) int -TEXT ·writeBlocks(SB), NOSPLIT|NOFRAME, $0-40 +// +// Deliberately not NOSPLIT: the stack check is the preemption point that lets a +// stop-the-world proceed between chunks. The frame must be >= abi.StackSmall +// (128), or the assembler marks this leaf NOSPLIT anyway and drops that check. +TEXT ·writeBlocks(SB), $128-40 LDP ·primes+0(SB), (prime1, prime2) // Load state. Assume v[1-4] are stored contiguously. diff --git a/vendor/github.com/klauspost/compress/zstd/seqdec.go b/vendor/github.com/klauspost/compress/zstd/seqdec.go index 0bfb0e43c7b6..d1a7732fcc64 100644 --- a/vendor/github.com/klauspost/compress/zstd/seqdec.go +++ b/vendor/github.com/klauspost/compress/zstd/seqdec.go @@ -99,6 +99,29 @@ func (s *sequenceDecs) initialize(br *bitReader, hist *history, out []byte) erro return nil } +// consumeSyncLen accounts for n bytes of frame output produced outside +// decodeSync. maxSyncLen bounds how much the frame may still produce and +// decodeSyncSimple compares it with the output buffer's slack to decide +// whether the extended 16-byte copies are safe. Blocks with sequences +// subtracted their output; raw blocks, RLE blocks and compressed blocks +// holding only literals did not, so a frame that opened with a raw block +// overstated its remaining size for every block after it and ran the +// bounds-exact copies for the rest of the frame. +// +// Zero means "no bound" and selects the conservative paths, so a block that +// meets or exceeds the bound lands there; the frame-size checks in +// runDecoder reject the excess afterwards. +func (s *sequenceDecs) consumeSyncLen(n int) { + if s.maxSyncLen == 0 { + return + } + if uint64(n) >= s.maxSyncLen { + s.maxSyncLen = 0 + return + } + s.maxSyncLen -= uint64(n) +} + func (s *sequenceDecs) freeDecoders() { if f := s.litLengths.fse; f != nil && !f.preDefined { fseDecoderPool.Put(f) diff --git a/vendor/github.com/klauspost/compress/zstd/seqdec_amd64.s b/vendor/github.com/klauspost/compress/zstd/seqdec_amd64.s index 4d3188ff49b9..798d940231fe 100644 --- a/vendor/github.com/klauspost/compress/zstd/seqdec_amd64.s +++ b/vendor/github.com/klauspost/compress/zstd/seqdec_amd64.s @@ -4,7 +4,7 @@ // func sequenceDecs_decode_amd64(s *sequenceDecs, br *bitReader, ctx *decodeAsmContext) int // Requires: CMOV -TEXT ·sequenceDecs_decode_amd64(SB), $8-32 +TEXT ·sequenceDecs_decode_amd64(SB), $40-32 MOVQ br+8(FP), CX MOVQ 24(CX), DX MOVBQZX 40(CX), BX @@ -16,6 +16,14 @@ TEXT ·sequenceDecs_decode_amd64(SB), $8-32 MOVQ 72(AX), DI MOVQ 80(AX), R8 MOVQ 88(AX), R9 + MOVQ (AX), CX + MOVQ CX, 8(SP) + MOVQ 24(AX), CX + MOVQ CX, 16(SP) + MOVQ 48(AX), CX + MOVQ CX, 24(SP) + MOVQ 96(AX), CX + MOVQ CX, 32(SP) MOVQ 104(AX), R10 MOVQ s+0(FP), AX MOVQ 144(AX), R11 @@ -55,46 +63,32 @@ sequenceDecs_decode_amd64_fill_check_overread: sequenceDecs_decode_amd64_fill_end: // Update offset - MOVQ R9, AX - MOVQ BX, CX - MOVQ DX, R15 - SHLQ CL, R15 - MOVB AH, CL - SHRQ $0x20, AX - TESTQ CX, CX - JZ sequenceDecs_decode_amd64_of_update_zero - ADDQ CX, BX - CMPQ BX, $0x40 - JA sequenceDecs_decode_amd64_of_update_zero - CMPQ CX, $0x40 - JAE sequenceDecs_decode_amd64_of_update_zero - NEGQ CX - SHRQ CL, R15 - ADDQ R15, AX - -sequenceDecs_decode_amd64_of_update_zero: - MOVQ AX, 16(R10) + MOVQ R9, AX + MOVQ BX, CX + MOVQ DX, R15 + SHLQ CL, R15 + SHRQ $0x01, R15 + MOVBLZX AH, CX + SHRQ $0x20, AX + ADDQ CX, BX + XORQ $0x3f, CX + SHRQ CL, R15 + ADDQ R15, AX + MOVQ AX, 16(R10) // Update match length - MOVQ R8, AX - MOVQ BX, CX - MOVQ DX, R15 - SHLQ CL, R15 - MOVB AH, CL - SHRQ $0x20, AX - TESTQ CX, CX - JZ sequenceDecs_decode_amd64_ml_update_zero - ADDQ CX, BX - CMPQ BX, $0x40 - JA sequenceDecs_decode_amd64_ml_update_zero - CMPQ CX, $0x40 - JAE sequenceDecs_decode_amd64_ml_update_zero - NEGQ CX - SHRQ CL, R15 - ADDQ R15, AX - -sequenceDecs_decode_amd64_ml_update_zero: - MOVQ AX, 8(R10) + MOVQ R8, AX + MOVQ BX, CX + MOVQ DX, R15 + SHLQ CL, R15 + SHRQ $0x01, R15 + MOVBLZX AH, CX + SHRQ $0x20, AX + ADDQ CX, BX + XORQ $0x3f, CX + SHRQ CL, R15 + ADDQ R15, AX + MOVQ AX, 8(R10) // Fill bitreader to have enough for the remaining CMPQ SI, $0x08 @@ -126,90 +120,76 @@ sequenceDecs_decode_amd64_fill_2_check_overread: sequenceDecs_decode_amd64_fill_2_end: // Update literal length - MOVQ DI, AX - MOVQ BX, CX - MOVQ DX, R15 - SHLQ CL, R15 - MOVB AH, CL - SHRQ $0x20, AX - TESTQ CX, CX - JZ sequenceDecs_decode_amd64_ll_update_zero - ADDQ CX, BX - CMPQ BX, $0x40 - JA sequenceDecs_decode_amd64_ll_update_zero - CMPQ CX, $0x40 - JAE sequenceDecs_decode_amd64_ll_update_zero - NEGQ CX - SHRQ CL, R15 - ADDQ R15, AX - -sequenceDecs_decode_amd64_ll_update_zero: - MOVQ AX, (R10) + MOVQ DI, AX + MOVQ BX, CX + MOVQ DX, R15 + SHLQ CL, R15 + SHRQ $0x01, R15 + MOVBLZX AH, CX + SHRQ $0x20, AX + ADDQ CX, BX + XORQ $0x3f, CX + SHRQ CL, R15 + ADDQ R15, AX + MOVQ AX, (R10) // Fill bitreader for state updates MOVQ R14, (SP) MOVQ R9, AX SHRQ $0x08, AX MOVBQZX AL, AX - MOVQ ctx+16(FP), CX - CMPQ 96(CX), $0x00 + CMPQ 32(SP), $0x00 JZ sequenceDecs_decode_amd64_skip_update // Update Literal Length State MOVBQZX DI, R14 SHRL $0x10, DI - LEAQ (BX)(R14*1), CX + MOVQ BX, CX MOVQ DX, R15 - MOVQ CX, BX - ROLQ CL, R15 - MOVL $0x00000001, BP - MOVB R14, CL - SHLL CL, BP - DECL BP - ANDQ BP, R15 + SHLQ CL, R15 + SHRQ $0x01, R15 + ADDQ R14, BX + XORQ $0x3f, R14 + MOVQ R14, CX + SHRQ CL, R15 ADDQ R15, DI // Load ctx.llTable - MOVQ ctx+16(FP), CX - MOVQ (CX), CX + MOVQ 8(SP), CX MOVQ (CX)(DI*8), DI // Update Match Length State MOVBQZX R8, R14 SHRL $0x10, R8 - LEAQ (BX)(R14*1), CX + MOVQ BX, CX MOVQ DX, R15 - MOVQ CX, BX - ROLQ CL, R15 - MOVL $0x00000001, BP - MOVB R14, CL - SHLL CL, BP - DECL BP - ANDQ BP, R15 + SHLQ CL, R15 + SHRQ $0x01, R15 + ADDQ R14, BX + XORQ $0x3f, R14 + MOVQ R14, CX + SHRQ CL, R15 ADDQ R15, R8 // Load ctx.mlTable - MOVQ ctx+16(FP), CX - MOVQ 24(CX), CX + MOVQ 16(SP), CX MOVQ (CX)(R8*8), R8 // Update Offset State MOVBQZX R9, R14 SHRL $0x10, R9 - LEAQ (BX)(R14*1), CX + MOVQ BX, CX MOVQ DX, R15 - MOVQ CX, BX - ROLQ CL, R15 - MOVL $0x00000001, BP - MOVB R14, CL - SHLL CL, BP - DECL BP - ANDQ BP, R15 + SHLQ CL, R15 + SHRQ $0x01, R15 + ADDQ R14, BX + XORQ $0x3f, R14 + MOVQ R14, CX + SHRQ CL, R15 ADDQ R15, R9 // Load ctx.ofTable - MOVQ ctx+16(FP), CX - MOVQ 48(CX), CX + MOVQ 24(SP), CX MOVQ (CX)(R9*8), R9 sequenceDecs_decode_amd64_skip_update: @@ -290,8 +270,7 @@ sequenceDecs_decode_amd64_after_adjust: sequenceDecs_decode_amd64_match_len_ofs_ok: ADDQ $0x18, R10 - MOVQ ctx+16(FP), AX - DECQ 96(AX) + DECQ 32(SP) JNS sequenceDecs_decode_amd64_main_loop MOVQ s+0(FP), AX MOVQ R11, 144(AX) @@ -303,36 +282,54 @@ sequenceDecs_decode_amd64_match_len_ofs_ok: MOVQ SI, 32(AX) // Return success + MOVQ 32(SP), AX + MOVQ ctx+16(FP), CX + MOVQ AX, 96(CX) MOVQ $0x00000000, ret+24(FP) RET // Return with match length error sequenceDecs_decode_amd64_error_match_len_ofs_mismatch: + MOVQ 32(SP), AX + MOVQ ctx+16(FP), CX + MOVQ AX, 96(CX) MOVQ $0x00000001, ret+24(FP) RET // Return with match too long error sequenceDecs_decode_amd64_error_match_len_too_big: + MOVQ 32(SP), AX + MOVQ ctx+16(FP), CX + MOVQ AX, 96(CX) MOVQ $0x00000002, ret+24(FP) RET // Return with match offset too long error + MOVQ 32(SP), AX + MOVQ ctx+16(FP), CX + MOVQ AX, 96(CX) MOVQ $0x00000003, ret+24(FP) RET // Return with not enough literals error error_not_enough_literals: + MOVQ 32(SP), AX + MOVQ ctx+16(FP), CX + MOVQ AX, 96(CX) MOVQ $0x00000004, ret+24(FP) RET // Return with overread error error_overread: + MOVQ 32(SP), AX + MOVQ ctx+16(FP), CX + MOVQ AX, 96(CX) MOVQ $0x00000006, ret+24(FP) RET // func sequenceDecs_decode_56_amd64(s *sequenceDecs, br *bitReader, ctx *decodeAsmContext) int // Requires: CMOV -TEXT ·sequenceDecs_decode_56_amd64(SB), $8-32 +TEXT ·sequenceDecs_decode_56_amd64(SB), $40-32 MOVQ br+8(FP), CX MOVQ 24(CX), DX MOVBQZX 40(CX), BX @@ -344,6 +341,14 @@ TEXT ·sequenceDecs_decode_56_amd64(SB), $8-32 MOVQ 72(AX), DI MOVQ 80(AX), R8 MOVQ 88(AX), R9 + MOVQ (AX), CX + MOVQ CX, 8(SP) + MOVQ 24(AX), CX + MOVQ CX, 16(SP) + MOVQ 48(AX), CX + MOVQ CX, 24(SP) + MOVQ 96(AX), CX + MOVQ CX, 32(SP) MOVQ 104(AX), R10 MOVQ s+0(FP), AX MOVQ 144(AX), R11 @@ -383,132 +388,104 @@ sequenceDecs_decode_56_amd64_fill_check_overread: sequenceDecs_decode_56_amd64_fill_end: // Update offset - MOVQ R9, AX - MOVQ BX, CX - MOVQ DX, R15 - SHLQ CL, R15 - MOVB AH, CL - SHRQ $0x20, AX - TESTQ CX, CX - JZ sequenceDecs_decode_56_amd64_of_update_zero - ADDQ CX, BX - CMPQ BX, $0x40 - JA sequenceDecs_decode_56_amd64_of_update_zero - CMPQ CX, $0x40 - JAE sequenceDecs_decode_56_amd64_of_update_zero - NEGQ CX - SHRQ CL, R15 - ADDQ R15, AX - -sequenceDecs_decode_56_amd64_of_update_zero: - MOVQ AX, 16(R10) + MOVQ R9, AX + MOVQ BX, CX + MOVQ DX, R15 + SHLQ CL, R15 + SHRQ $0x01, R15 + MOVBLZX AH, CX + SHRQ $0x20, AX + ADDQ CX, BX + XORQ $0x3f, CX + SHRQ CL, R15 + ADDQ R15, AX + MOVQ AX, 16(R10) // Update match length - MOVQ R8, AX - MOVQ BX, CX - MOVQ DX, R15 - SHLQ CL, R15 - MOVB AH, CL - SHRQ $0x20, AX - TESTQ CX, CX - JZ sequenceDecs_decode_56_amd64_ml_update_zero - ADDQ CX, BX - CMPQ BX, $0x40 - JA sequenceDecs_decode_56_amd64_ml_update_zero - CMPQ CX, $0x40 - JAE sequenceDecs_decode_56_amd64_ml_update_zero - NEGQ CX - SHRQ CL, R15 - ADDQ R15, AX - -sequenceDecs_decode_56_amd64_ml_update_zero: - MOVQ AX, 8(R10) + MOVQ R8, AX + MOVQ BX, CX + MOVQ DX, R15 + SHLQ CL, R15 + SHRQ $0x01, R15 + MOVBLZX AH, CX + SHRQ $0x20, AX + ADDQ CX, BX + XORQ $0x3f, CX + SHRQ CL, R15 + ADDQ R15, AX + MOVQ AX, 8(R10) // Update literal length - MOVQ DI, AX - MOVQ BX, CX - MOVQ DX, R15 - SHLQ CL, R15 - MOVB AH, CL - SHRQ $0x20, AX - TESTQ CX, CX - JZ sequenceDecs_decode_56_amd64_ll_update_zero - ADDQ CX, BX - CMPQ BX, $0x40 - JA sequenceDecs_decode_56_amd64_ll_update_zero - CMPQ CX, $0x40 - JAE sequenceDecs_decode_56_amd64_ll_update_zero - NEGQ CX - SHRQ CL, R15 - ADDQ R15, AX - -sequenceDecs_decode_56_amd64_ll_update_zero: - MOVQ AX, (R10) + MOVQ DI, AX + MOVQ BX, CX + MOVQ DX, R15 + SHLQ CL, R15 + SHRQ $0x01, R15 + MOVBLZX AH, CX + SHRQ $0x20, AX + ADDQ CX, BX + XORQ $0x3f, CX + SHRQ CL, R15 + ADDQ R15, AX + MOVQ AX, (R10) // Fill bitreader for state updates MOVQ R14, (SP) MOVQ R9, AX SHRQ $0x08, AX MOVBQZX AL, AX - MOVQ ctx+16(FP), CX - CMPQ 96(CX), $0x00 + CMPQ 32(SP), $0x00 JZ sequenceDecs_decode_56_amd64_skip_update // Update Literal Length State MOVBQZX DI, R14 SHRL $0x10, DI - LEAQ (BX)(R14*1), CX + MOVQ BX, CX MOVQ DX, R15 - MOVQ CX, BX - ROLQ CL, R15 - MOVL $0x00000001, BP - MOVB R14, CL - SHLL CL, BP - DECL BP - ANDQ BP, R15 + SHLQ CL, R15 + SHRQ $0x01, R15 + ADDQ R14, BX + XORQ $0x3f, R14 + MOVQ R14, CX + SHRQ CL, R15 ADDQ R15, DI // Load ctx.llTable - MOVQ ctx+16(FP), CX - MOVQ (CX), CX + MOVQ 8(SP), CX MOVQ (CX)(DI*8), DI // Update Match Length State MOVBQZX R8, R14 SHRL $0x10, R8 - LEAQ (BX)(R14*1), CX + MOVQ BX, CX MOVQ DX, R15 - MOVQ CX, BX - ROLQ CL, R15 - MOVL $0x00000001, BP - MOVB R14, CL - SHLL CL, BP - DECL BP - ANDQ BP, R15 + SHLQ CL, R15 + SHRQ $0x01, R15 + ADDQ R14, BX + XORQ $0x3f, R14 + MOVQ R14, CX + SHRQ CL, R15 ADDQ R15, R8 // Load ctx.mlTable - MOVQ ctx+16(FP), CX - MOVQ 24(CX), CX + MOVQ 16(SP), CX MOVQ (CX)(R8*8), R8 // Update Offset State MOVBQZX R9, R14 SHRL $0x10, R9 - LEAQ (BX)(R14*1), CX + MOVQ BX, CX MOVQ DX, R15 - MOVQ CX, BX - ROLQ CL, R15 - MOVL $0x00000001, BP - MOVB R14, CL - SHLL CL, BP - DECL BP - ANDQ BP, R15 + SHLQ CL, R15 + SHRQ $0x01, R15 + ADDQ R14, BX + XORQ $0x3f, R14 + MOVQ R14, CX + SHRQ CL, R15 ADDQ R15, R9 // Load ctx.ofTable - MOVQ ctx+16(FP), CX - MOVQ 48(CX), CX + MOVQ 24(SP), CX MOVQ (CX)(R9*8), R9 sequenceDecs_decode_56_amd64_skip_update: @@ -589,8 +566,7 @@ sequenceDecs_decode_56_amd64_after_adjust: sequenceDecs_decode_56_amd64_match_len_ofs_ok: ADDQ $0x18, R10 - MOVQ ctx+16(FP), AX - DECQ 96(AX) + DECQ 32(SP) JNS sequenceDecs_decode_56_amd64_main_loop MOVQ s+0(FP), AX MOVQ R11, 144(AX) @@ -602,36 +578,54 @@ sequenceDecs_decode_56_amd64_match_len_ofs_ok: MOVQ SI, 32(AX) // Return success + MOVQ 32(SP), AX + MOVQ ctx+16(FP), CX + MOVQ AX, 96(CX) MOVQ $0x00000000, ret+24(FP) RET // Return with match length error sequenceDecs_decode_56_amd64_error_match_len_ofs_mismatch: + MOVQ 32(SP), AX + MOVQ ctx+16(FP), CX + MOVQ AX, 96(CX) MOVQ $0x00000001, ret+24(FP) RET // Return with match too long error sequenceDecs_decode_56_amd64_error_match_len_too_big: + MOVQ 32(SP), AX + MOVQ ctx+16(FP), CX + MOVQ AX, 96(CX) MOVQ $0x00000002, ret+24(FP) RET // Return with match offset too long error + MOVQ 32(SP), AX + MOVQ ctx+16(FP), CX + MOVQ AX, 96(CX) MOVQ $0x00000003, ret+24(FP) RET // Return with not enough literals error error_not_enough_literals: + MOVQ 32(SP), AX + MOVQ ctx+16(FP), CX + MOVQ AX, 96(CX) MOVQ $0x00000004, ret+24(FP) RET // Return with overread error error_overread: + MOVQ 32(SP), AX + MOVQ ctx+16(FP), CX + MOVQ AX, 96(CX) MOVQ $0x00000006, ret+24(FP) RET // func sequenceDecs_decode_bmi2(s *sequenceDecs, br *bitReader, ctx *decodeAsmContext) int // Requires: BMI, BMI2, CMOV -TEXT ·sequenceDecs_decode_bmi2(SB), $8-32 +TEXT ·sequenceDecs_decode_bmi2(SB), $40-32 MOVQ br+8(FP), BX MOVQ 24(BX), AX MOVBQZX 40(BX), DX @@ -643,6 +637,14 @@ TEXT ·sequenceDecs_decode_bmi2(SB), $8-32 MOVQ 72(CX), SI MOVQ 80(CX), DI MOVQ 88(CX), R8 + MOVQ (CX), R9 + MOVQ R9, 8(SP) + MOVQ 24(CX), R9 + MOVQ R9, 16(SP) + MOVQ 48(CX), R9 + MOVQ R9, 24(SP) + MOVQ 96(CX), R9 + MOVQ R9, 32(SP) MOVQ 104(CX), R9 MOVQ s+0(FP), CX MOVQ 144(CX), R10 @@ -753,8 +755,7 @@ sequenceDecs_decode_bmi2_fill_2_end: MOVQ R13, (SP) MOVQ $0x00000808, CX BEXTRQ CX, R8, R13 - MOVQ ctx+16(FP), CX - CMPQ 96(CX), $0x00 + CMPQ 32(SP), $0x00 JZ sequenceDecs_decode_bmi2_skip_update LEAQ (SI)(DI*1), R14 ADDQ R8, R14 @@ -772,8 +773,7 @@ sequenceDecs_decode_bmi2_fill_2_end: ADDQ CX, R8 // Load ctx.ofTable - MOVQ ctx+16(FP), CX - MOVQ 48(CX), CX + MOVQ 24(SP), CX MOVQ (CX)(R8*8), R8 // Update Match Length State @@ -783,8 +783,7 @@ sequenceDecs_decode_bmi2_fill_2_end: ADDQ CX, DI // Load ctx.mlTable - MOVQ ctx+16(FP), CX - MOVQ 24(CX), CX + MOVQ 16(SP), CX MOVQ (CX)(DI*8), DI // Update Literal Length State @@ -793,8 +792,7 @@ sequenceDecs_decode_bmi2_fill_2_end: ADDQ CX, SI // Load ctx.llTable - MOVQ ctx+16(FP), CX - MOVQ (CX), CX + MOVQ 8(SP), CX MOVQ (CX)(SI*8), SI sequenceDecs_decode_bmi2_skip_update: @@ -875,8 +873,7 @@ sequenceDecs_decode_bmi2_after_adjust: sequenceDecs_decode_bmi2_match_len_ofs_ok: ADDQ $0x18, R9 - MOVQ ctx+16(FP), CX - DECQ 96(CX) + DECQ 32(SP) JNS sequenceDecs_decode_bmi2_main_loop MOVQ s+0(FP), CX MOVQ R10, 144(CX) @@ -888,36 +885,54 @@ sequenceDecs_decode_bmi2_match_len_ofs_ok: MOVQ BX, 32(CX) // Return success + MOVQ 32(SP), AX + MOVQ ctx+16(FP), CX + MOVQ AX, 96(CX) MOVQ $0x00000000, ret+24(FP) RET // Return with match length error sequenceDecs_decode_bmi2_error_match_len_ofs_mismatch: + MOVQ 32(SP), AX + MOVQ ctx+16(FP), CX + MOVQ AX, 96(CX) MOVQ $0x00000001, ret+24(FP) RET // Return with match too long error sequenceDecs_decode_bmi2_error_match_len_too_big: + MOVQ 32(SP), AX + MOVQ ctx+16(FP), CX + MOVQ AX, 96(CX) MOVQ $0x00000002, ret+24(FP) RET // Return with match offset too long error + MOVQ 32(SP), AX + MOVQ ctx+16(FP), CX + MOVQ AX, 96(CX) MOVQ $0x00000003, ret+24(FP) RET // Return with not enough literals error error_not_enough_literals: + MOVQ 32(SP), AX + MOVQ ctx+16(FP), CX + MOVQ AX, 96(CX) MOVQ $0x00000004, ret+24(FP) RET // Return with overread error error_overread: + MOVQ 32(SP), AX + MOVQ ctx+16(FP), CX + MOVQ AX, 96(CX) MOVQ $0x00000006, ret+24(FP) RET // func sequenceDecs_decode_56_bmi2(s *sequenceDecs, br *bitReader, ctx *decodeAsmContext) int // Requires: BMI, BMI2, CMOV -TEXT ·sequenceDecs_decode_56_bmi2(SB), $8-32 +TEXT ·sequenceDecs_decode_56_bmi2(SB), $40-32 MOVQ br+8(FP), BX MOVQ 24(BX), AX MOVBQZX 40(BX), DX @@ -929,6 +944,14 @@ TEXT ·sequenceDecs_decode_56_bmi2(SB), $8-32 MOVQ 72(CX), SI MOVQ 80(CX), DI MOVQ 88(CX), R8 + MOVQ (CX), R9 + MOVQ R9, 8(SP) + MOVQ 24(CX), R9 + MOVQ R9, 16(SP) + MOVQ 48(CX), R9 + MOVQ R9, 24(SP) + MOVQ 96(CX), R9 + MOVQ R9, 32(SP) MOVQ 104(CX), R9 MOVQ s+0(FP), CX MOVQ 144(CX), R10 @@ -1010,8 +1033,7 @@ sequenceDecs_decode_56_bmi2_fill_end: MOVQ R13, (SP) MOVQ $0x00000808, CX BEXTRQ CX, R8, R13 - MOVQ ctx+16(FP), CX - CMPQ 96(CX), $0x00 + CMPQ 32(SP), $0x00 JZ sequenceDecs_decode_56_bmi2_skip_update LEAQ (SI)(DI*1), R14 ADDQ R8, R14 @@ -1029,8 +1051,7 @@ sequenceDecs_decode_56_bmi2_fill_end: ADDQ CX, R8 // Load ctx.ofTable - MOVQ ctx+16(FP), CX - MOVQ 48(CX), CX + MOVQ 24(SP), CX MOVQ (CX)(R8*8), R8 // Update Match Length State @@ -1040,8 +1061,7 @@ sequenceDecs_decode_56_bmi2_fill_end: ADDQ CX, DI // Load ctx.mlTable - MOVQ ctx+16(FP), CX - MOVQ 24(CX), CX + MOVQ 16(SP), CX MOVQ (CX)(DI*8), DI // Update Literal Length State @@ -1050,8 +1070,7 @@ sequenceDecs_decode_56_bmi2_fill_end: ADDQ CX, SI // Load ctx.llTable - MOVQ ctx+16(FP), CX - MOVQ (CX), CX + MOVQ 8(SP), CX MOVQ (CX)(SI*8), SI sequenceDecs_decode_56_bmi2_skip_update: @@ -1132,8 +1151,7 @@ sequenceDecs_decode_56_bmi2_after_adjust: sequenceDecs_decode_56_bmi2_match_len_ofs_ok: ADDQ $0x18, R9 - MOVQ ctx+16(FP), CX - DECQ 96(CX) + DECQ 32(SP) JNS sequenceDecs_decode_56_bmi2_main_loop MOVQ s+0(FP), CX MOVQ R10, 144(CX) @@ -1145,30 +1163,48 @@ sequenceDecs_decode_56_bmi2_match_len_ofs_ok: MOVQ BX, 32(CX) // Return success + MOVQ 32(SP), AX + MOVQ ctx+16(FP), CX + MOVQ AX, 96(CX) MOVQ $0x00000000, ret+24(FP) RET // Return with match length error sequenceDecs_decode_56_bmi2_error_match_len_ofs_mismatch: + MOVQ 32(SP), AX + MOVQ ctx+16(FP), CX + MOVQ AX, 96(CX) MOVQ $0x00000001, ret+24(FP) RET // Return with match too long error sequenceDecs_decode_56_bmi2_error_match_len_too_big: + MOVQ 32(SP), AX + MOVQ ctx+16(FP), CX + MOVQ AX, 96(CX) MOVQ $0x00000002, ret+24(FP) RET // Return with match offset too long error + MOVQ 32(SP), AX + MOVQ ctx+16(FP), CX + MOVQ AX, 96(CX) MOVQ $0x00000003, ret+24(FP) RET // Return with not enough literals error error_not_enough_literals: + MOVQ 32(SP), AX + MOVQ ctx+16(FP), CX + MOVQ AX, 96(CX) MOVQ $0x00000004, ret+24(FP) RET // Return with overread error error_overread: + MOVQ 32(SP), AX + MOVQ ctx+16(FP), CX + MOVQ AX, 96(CX) MOVQ $0x00000006, ret+24(FP) RET @@ -1203,9 +1239,11 @@ main_loop: MOVQ 8(AX), R13 // Copy literals - TESTQ R11, R11 - JZ check_offset - XORQ R14, R14 + MOVUPS (SI), X0 + MOVUPS X0, (BX) + CMPQ R11, $0x10 + JBE copy_1_end + MOVQ $0x00000010, R14 copy_1: MOVUPS (SI)(R14*1), X0 @@ -1213,12 +1251,13 @@ copy_1: ADDQ $0x10, R14 CMPQ R14, R11 JB copy_1 - ADDQ R11, SI - ADDQ R11, BX - ADDQ R11, DI + +copy_1_end: + ADDQ R11, SI + ADDQ R11, BX + ADDQ R11, DI // Malformed input if seq.mo > t+len(hist) || seq.mo > s.windowSize) -check_offset: LEAQ (DI)(R10*1), R11 CMPQ R12, R11 JG error_match_off_too_big @@ -1258,13 +1297,13 @@ copy_4_small: JMP copy_4_move_8through16 copy_4_move_3: - MOVW (R14), R11 - MOVB 2(R14), R12 - MOVW R11, (BX) - MOVB R12, 2(BX) - ADDQ R13, R14 - ADDQ R13, BX - JMP copy_4_end + MOVWQZX (R14), R11 + MOVB 2(R14), R12 + MOVW R11, (BX) + MOVB R12, 2(BX) + ADDQ R13, R14 + ADDQ R13, BX + JMP copy_4_end copy_4_move_4through7: MOVL (R14), R11 @@ -1327,13 +1366,13 @@ copy_5_move_1or2: JMP copy_5_end copy_5_move_3: - MOVW (R14), R15 - MOVB 2(R14), BP - MOVW R15, (BX) - MOVB BP, 2(BX) - ADDQ R11, R14 - ADDQ R11, BX - JMP copy_5_end + MOVWQZX (R14), R15 + MOVB 2(R14), BP + MOVW R15, (BX) + MOVB BP, 2(BX) + ADDQ R11, R14 + ADDQ R11, BX + JMP copy_5_end copy_5_move_4through7: MOVL (R14), R15 @@ -1366,9 +1405,19 @@ copy_match: JA copy_overlapping_match // Copy non-overlapping match - ADDQ R13, DI - MOVQ BX, R12 + ADDQ R13, DI + MOVUPS (R11), X0 + MOVUPS X0, (BX) + CMPQ R13, $0x10 + JA copy_2_long + ADDQ R13, BX + JMP handle_loop + +copy_2_long: + LEAQ 16(BX), R12 ADDQ R13, BX + ADDQ $0x10, R11 + SUBQ $0x10, R13 copy_2: MOVUPS (R11), X0 @@ -1384,12 +1433,12 @@ copy_overlapping_match: ADDQ R13, DI copy_slow_3: - MOVB (R11), R12 - MOVB R12, (BX) - INCQ R11 - INCQ BX - DECQ R13 - JNZ copy_slow_3 + MOVBQZX (R11), R12 + MOVB R12, (BX) + INCQ R11 + INCQ BX + DECQ R13 + JNZ copy_slow_3 handle_loop: ADDQ $0x18, AX @@ -1494,13 +1543,13 @@ copy_1_move_1or2: JMP copy_1_end copy_1_move_3: - MOVW (SI), R14 - MOVB 2(SI), R15 - MOVW R14, (BX) - MOVB R15, 2(BX) - ADDQ R11, SI - ADDQ R11, BX - JMP copy_1_end + MOVWQZX (SI), R14 + MOVB 2(SI), R15 + MOVW R14, (BX) + MOVB R15, 2(BX) + ADDQ R11, SI + ADDQ R11, BX + JMP copy_1_end copy_1_move_4through7: MOVL (SI), R14 @@ -1563,13 +1612,13 @@ copy_4_small: JMP copy_4_move_8through16 copy_4_move_3: - MOVW (R14), R11 - MOVB 2(R14), R12 - MOVW R11, (BX) - MOVB R12, 2(BX) - ADDQ R13, R14 - ADDQ R13, BX - JMP copy_4_end + MOVWQZX (R14), R11 + MOVB 2(R14), R12 + MOVW R11, (BX) + MOVB R12, 2(BX) + ADDQ R13, R14 + ADDQ R13, BX + JMP copy_4_end copy_4_move_4through7: MOVL (R14), R11 @@ -1632,13 +1681,13 @@ copy_5_move_1or2: JMP copy_5_end copy_5_move_3: - MOVW (R14), R15 - MOVB 2(R14), BP - MOVW R15, (BX) - MOVB BP, 2(BX) - ADDQ R11, R14 - ADDQ R11, BX - JMP copy_5_end + MOVWQZX (R14), R15 + MOVB 2(R14), BP + MOVW R15, (BX) + MOVB BP, 2(BX) + ADDQ R11, R14 + ADDQ R11, BX + JMP copy_5_end copy_5_move_4through7: MOVL (R14), R15 @@ -1707,13 +1756,13 @@ copy_2_move_1or2: JMP copy_2_end copy_2_move_3: - MOVW (R11), R12 - MOVB 2(R11), R14 - MOVW R12, (BX) - MOVB R14, 2(BX) - ADDQ R13, R11 - ADDQ R13, BX - JMP copy_2_end + MOVWQZX (R11), R12 + MOVB 2(R11), R14 + MOVW R12, (BX) + MOVB R14, 2(BX) + ADDQ R13, R11 + ADDQ R13, BX + JMP copy_2_end copy_2_move_4through7: MOVL (R11), R12 @@ -1740,12 +1789,12 @@ copy_overlapping_match: ADDQ R13, DI copy_slow_3: - MOVB (R11), R12 - MOVB R12, (BX) - INCQ R11 - INCQ BX - DECQ R13 - JNZ copy_slow_3 + MOVBQZX (R11), R12 + MOVB R12, (BX) + INCQ R11 + INCQ BX + DECQ R13 + JNZ copy_slow_3 handle_loop: ADDQ $0x18, AX @@ -1784,7 +1833,7 @@ empty_seqs: // func sequenceDecs_decodeSync_amd64(s *sequenceDecs, br *bitReader, ctx *decodeSyncAsmContext) int // Requires: CMOV, SSE -TEXT ·sequenceDecs_decodeSync_amd64(SB), $64-32 +TEXT ·sequenceDecs_decodeSync_amd64(SB), $96-32 MOVQ br+8(FP), CX MOVQ 24(CX), DX MOVBQZX 40(CX), BX @@ -1796,26 +1845,34 @@ TEXT ·sequenceDecs_decodeSync_amd64(SB), $64-32 MOVQ 72(AX), DI MOVQ 80(AX), R8 MOVQ 88(AX), R9 - XORQ CX, CX + MOVQ (AX), CX MOVQ CX, 8(SP) + MOVQ 24(AX), CX MOVQ CX, 16(SP) + MOVQ 48(AX), CX MOVQ CX, 24(SP) + MOVQ 96(AX), CX + MOVQ CX, 32(SP) + XORQ CX, CX + MOVQ CX, 40(SP) + MOVQ CX, 48(SP) + MOVQ CX, 56(SP) MOVQ 112(AX), R10 MOVQ 128(AX), CX - MOVQ CX, 32(SP) + MOVQ CX, 64(SP) MOVQ 144(AX), R11 MOVQ 136(AX), R12 MOVQ 200(AX), CX - MOVQ CX, 56(SP) + MOVQ CX, 88(SP) MOVQ 176(AX), CX - MOVQ CX, 48(SP) + MOVQ CX, 80(SP) MOVQ 184(AX), AX - MOVQ AX, 40(SP) - MOVQ 40(SP), AX - ADDQ AX, 48(SP) + MOVQ AX, 72(SP) + MOVQ 72(SP), AX + ADDQ AX, 80(SP) // Calculate pointer to s.out[cap(s.out)] (a past-end pointer) - ADDQ R10, 32(SP) + ADDQ R10, 64(SP) // outBase += outPosition ADDQ R12, R10 @@ -1853,46 +1910,32 @@ sequenceDecs_decodeSync_amd64_fill_check_overread: sequenceDecs_decodeSync_amd64_fill_end: // Update offset - MOVQ R9, AX - MOVQ BX, CX - MOVQ DX, R14 - SHLQ CL, R14 - MOVB AH, CL - SHRQ $0x20, AX - TESTQ CX, CX - JZ sequenceDecs_decodeSync_amd64_of_update_zero - ADDQ CX, BX - CMPQ BX, $0x40 - JA sequenceDecs_decodeSync_amd64_of_update_zero - CMPQ CX, $0x40 - JAE sequenceDecs_decodeSync_amd64_of_update_zero - NEGQ CX - SHRQ CL, R14 - ADDQ R14, AX - -sequenceDecs_decodeSync_amd64_of_update_zero: - MOVQ AX, 8(SP) + MOVQ R9, AX + MOVQ BX, CX + MOVQ DX, R14 + SHLQ CL, R14 + SHRQ $0x01, R14 + MOVBLZX AH, CX + SHRQ $0x20, AX + ADDQ CX, BX + XORQ $0x3f, CX + SHRQ CL, R14 + ADDQ R14, AX + MOVQ AX, 40(SP) // Update match length - MOVQ R8, AX - MOVQ BX, CX - MOVQ DX, R14 - SHLQ CL, R14 - MOVB AH, CL - SHRQ $0x20, AX - TESTQ CX, CX - JZ sequenceDecs_decodeSync_amd64_ml_update_zero - ADDQ CX, BX - CMPQ BX, $0x40 - JA sequenceDecs_decodeSync_amd64_ml_update_zero - CMPQ CX, $0x40 - JAE sequenceDecs_decodeSync_amd64_ml_update_zero - NEGQ CX - SHRQ CL, R14 - ADDQ R14, AX - -sequenceDecs_decodeSync_amd64_ml_update_zero: - MOVQ AX, 16(SP) + MOVQ R8, AX + MOVQ BX, CX + MOVQ DX, R14 + SHLQ CL, R14 + SHRQ $0x01, R14 + MOVBLZX AH, CX + SHRQ $0x20, AX + ADDQ CX, BX + XORQ $0x3f, CX + SHRQ CL, R14 + ADDQ R14, AX + MOVQ AX, 48(SP) // Fill bitreader to have enough for the remaining CMPQ SI, $0x08 @@ -1924,96 +1967,82 @@ sequenceDecs_decodeSync_amd64_fill_2_check_overread: sequenceDecs_decodeSync_amd64_fill_2_end: // Update literal length - MOVQ DI, AX - MOVQ BX, CX - MOVQ DX, R14 - SHLQ CL, R14 - MOVB AH, CL - SHRQ $0x20, AX - TESTQ CX, CX - JZ sequenceDecs_decodeSync_amd64_ll_update_zero - ADDQ CX, BX - CMPQ BX, $0x40 - JA sequenceDecs_decodeSync_amd64_ll_update_zero - CMPQ CX, $0x40 - JAE sequenceDecs_decodeSync_amd64_ll_update_zero - NEGQ CX - SHRQ CL, R14 - ADDQ R14, AX - -sequenceDecs_decodeSync_amd64_ll_update_zero: - MOVQ AX, 24(SP) + MOVQ DI, AX + MOVQ BX, CX + MOVQ DX, R14 + SHLQ CL, R14 + SHRQ $0x01, R14 + MOVBLZX AH, CX + SHRQ $0x20, AX + ADDQ CX, BX + XORQ $0x3f, CX + SHRQ CL, R14 + ADDQ R14, AX + MOVQ AX, 56(SP) // Fill bitreader for state updates MOVQ R13, (SP) MOVQ R9, AX SHRQ $0x08, AX MOVBQZX AL, AX - MOVQ ctx+16(FP), CX - CMPQ 96(CX), $0x00 + CMPQ 32(SP), $0x00 JZ sequenceDecs_decodeSync_amd64_skip_update // Update Literal Length State MOVBQZX DI, R13 SHRL $0x10, DI - LEAQ (BX)(R13*1), CX + MOVQ BX, CX MOVQ DX, R14 - MOVQ CX, BX - ROLQ CL, R14 - MOVL $0x00000001, R15 - MOVB R13, CL - SHLL CL, R15 - DECL R15 - ANDQ R15, R14 + SHLQ CL, R14 + SHRQ $0x01, R14 + ADDQ R13, BX + XORQ $0x3f, R13 + MOVQ R13, CX + SHRQ CL, R14 ADDQ R14, DI // Load ctx.llTable - MOVQ ctx+16(FP), CX - MOVQ (CX), CX + MOVQ 8(SP), CX MOVQ (CX)(DI*8), DI // Update Match Length State MOVBQZX R8, R13 SHRL $0x10, R8 - LEAQ (BX)(R13*1), CX + MOVQ BX, CX MOVQ DX, R14 - MOVQ CX, BX - ROLQ CL, R14 - MOVL $0x00000001, R15 - MOVB R13, CL - SHLL CL, R15 - DECL R15 - ANDQ R15, R14 + SHLQ CL, R14 + SHRQ $0x01, R14 + ADDQ R13, BX + XORQ $0x3f, R13 + MOVQ R13, CX + SHRQ CL, R14 ADDQ R14, R8 // Load ctx.mlTable - MOVQ ctx+16(FP), CX - MOVQ 24(CX), CX + MOVQ 16(SP), CX MOVQ (CX)(R8*8), R8 // Update Offset State MOVBQZX R9, R13 SHRL $0x10, R9 - LEAQ (BX)(R13*1), CX + MOVQ BX, CX MOVQ DX, R14 - MOVQ CX, BX - ROLQ CL, R14 - MOVL $0x00000001, R15 - MOVB R13, CL - SHLL CL, R15 - DECL R15 - ANDQ R15, R14 + SHLQ CL, R14 + SHRQ $0x01, R14 + ADDQ R13, BX + XORQ $0x3f, R13 + MOVQ R13, CX + SHRQ CL, R14 ADDQ R14, R9 // Load ctx.ofTable - MOVQ ctx+16(FP), CX - MOVQ 48(CX), CX + MOVQ 24(SP), CX MOVQ (CX)(R9*8), R9 sequenceDecs_decodeSync_amd64_skip_update: // Adjust offset MOVQ s+0(FP), CX - MOVQ 8(SP), R13 + MOVQ 40(SP), R13 CMPQ AX, $0x01 JBE sequenceDecs_decodeSync_amd64_adjust_offsetB_1_or_0 MOVUPS 144(CX), X0 @@ -2022,7 +2051,7 @@ sequenceDecs_decodeSync_amd64_skip_update: JMP sequenceDecs_decodeSync_amd64_after_adjust sequenceDecs_decodeSync_amd64_adjust_offsetB_1_or_0: - CMPQ 24(SP), $0x00000000 + CMPQ 56(SP), $0x00000000 JNE sequenceDecs_decodeSync_amd64_adjust_offset_maybezero INCQ R13 JMP sequenceDecs_decodeSync_amd64_adjust_offset_nonzero @@ -2057,14 +2086,11 @@ sequenceDecs_decodeSync_amd64_adjust_skip: MOVQ R14, R13 sequenceDecs_decodeSync_amd64_after_adjust: - MOVQ R13, 8(SP) + MOVQ R13, 40(SP) // Check values - MOVQ 16(SP), AX - MOVQ 24(SP), CX - LEAQ (AX)(CX*1), R14 - MOVQ s+0(FP), R15 - ADDQ R14, 256(R15) + MOVQ 48(SP), AX + MOVQ 56(SP), CX MOVQ ctx+16(FP), R14 SUBQ CX, 104(R14) JS error_not_enough_literals @@ -2076,50 +2102,49 @@ sequenceDecs_decodeSync_amd64_after_adjust: JNZ sequenceDecs_decodeSync_amd64_error_match_len_ofs_mismatch sequenceDecs_decodeSync_amd64_match_len_ofs_ok: - MOVQ 24(SP), AX - MOVQ 8(SP), CX - MOVQ 16(SP), R13 - // Check if we have enough space in s.out - LEAQ 16(AX)(R13*1), R14 + LEAQ 16(CX)(AX*1), R14 ADDQ R10, R14 - CMPQ R14, 32(SP) + CMPQ R14, 64(SP) JA error_not_enough_space // Copy literals - TESTQ AX, AX - JZ check_offset - XORQ R14, R14 + MOVUPS (R11), X0 + MOVUPS X0, (R10) + CMPQ CX, $0x10 + JBE copy_1_end + MOVQ $0x00000010, R14 copy_1: MOVUPS (R11)(R14*1), X0 MOVUPS X0, (R10)(R14*1) ADDQ $0x10, R14 - CMPQ R14, AX + CMPQ R14, CX JB copy_1 - ADDQ AX, R11 - ADDQ AX, R10 - ADDQ AX, R12 + +copy_1_end: + ADDQ CX, R11 + ADDQ CX, R10 + ADDQ CX, R12 // Malformed input if seq.mo > t+len(hist) || seq.mo > s.windowSize) -check_offset: - MOVQ R12, AX - ADDQ 40(SP), AX - CMPQ CX, AX + MOVQ R12, CX + ADDQ 72(SP), CX + CMPQ R13, CX JG error_match_off_too_big - CMPQ CX, 56(SP) + CMPQ R13, 88(SP) JG error_match_off_too_big // Copy match from history - MOVQ CX, AX - SUBQ R12, AX + MOVQ R13, CX + SUBQ R12, CX JLS copy_match - MOVQ 48(SP), R14 - SUBQ AX, R14 - CMPQ R13, AX + MOVQ 80(SP), R14 + SUBQ CX, R14 + CMPQ AX, CX JG copy_all_from_history - MOVQ R13, AX - SUBQ $0x10, AX + MOVQ AX, CX + SUBQ $0x10, CX JB copy_4_small copy_4_loop: @@ -2127,54 +2152,54 @@ copy_4_loop: MOVUPS X0, (R10) ADDQ $0x10, R14 ADDQ $0x10, R10 - SUBQ $0x10, AX + SUBQ $0x10, CX JAE copy_4_loop - LEAQ 16(R14)(AX*1), R14 - LEAQ 16(R10)(AX*1), R10 + LEAQ 16(R14)(CX*1), R14 + LEAQ 16(R10)(CX*1), R10 MOVUPS -16(R14), X0 MOVUPS X0, -16(R10) JMP copy_4_end copy_4_small: - CMPQ R13, $0x03 + CMPQ AX, $0x03 JE copy_4_move_3 - CMPQ R13, $0x08 + CMPQ AX, $0x08 JB copy_4_move_4through7 JMP copy_4_move_8through16 copy_4_move_3: - MOVW (R14), AX - MOVB 2(R14), CL - MOVW AX, (R10) - MOVB CL, 2(R10) - ADDQ R13, R14 - ADDQ R13, R10 - JMP copy_4_end + MOVWQZX (R14), CX + MOVB 2(R14), R13 + MOVW CX, (R10) + MOVB R13, 2(R10) + ADDQ AX, R14 + ADDQ AX, R10 + JMP copy_4_end copy_4_move_4through7: - MOVL (R14), AX - MOVL -4(R14)(R13*1), CX - MOVL AX, (R10) - MOVL CX, -4(R10)(R13*1) - ADDQ R13, R14 - ADDQ R13, R10 + MOVL (R14), CX + MOVL -4(R14)(AX*1), R13 + MOVL CX, (R10) + MOVL R13, -4(R10)(AX*1) + ADDQ AX, R14 + ADDQ AX, R10 JMP copy_4_end copy_4_move_8through16: - MOVQ (R14), AX - MOVQ -8(R14)(R13*1), CX - MOVQ AX, (R10) - MOVQ CX, -8(R10)(R13*1) - ADDQ R13, R14 - ADDQ R13, R10 + MOVQ (R14), CX + MOVQ -8(R14)(AX*1), R13 + MOVQ CX, (R10) + MOVQ R13, -8(R10)(AX*1) + ADDQ AX, R14 + ADDQ AX, R10 copy_4_end: - ADDQ R13, R12 + ADDQ AX, R12 JMP handle_loop JMP loop_finished copy_all_from_history: - MOVQ AX, R15 + MOVQ CX, R15 SUBQ $0x10, R15 JB copy_5_small @@ -2192,90 +2217,99 @@ copy_5_loop: JMP copy_5_end copy_5_small: - CMPQ AX, $0x03 + CMPQ CX, $0x03 JE copy_5_move_3 JB copy_5_move_1or2 - CMPQ AX, $0x08 + CMPQ CX, $0x08 JB copy_5_move_4through7 JMP copy_5_move_8through16 copy_5_move_1or2: MOVB (R14), R15 - MOVB -1(R14)(AX*1), BP + MOVB -1(R14)(CX*1), BP MOVB R15, (R10) - MOVB BP, -1(R10)(AX*1) - ADDQ AX, R14 - ADDQ AX, R10 + MOVB BP, -1(R10)(CX*1) + ADDQ CX, R14 + ADDQ CX, R10 JMP copy_5_end copy_5_move_3: - MOVW (R14), R15 - MOVB 2(R14), BP - MOVW R15, (R10) - MOVB BP, 2(R10) - ADDQ AX, R14 - ADDQ AX, R10 - JMP copy_5_end + MOVWQZX (R14), R15 + MOVB 2(R14), BP + MOVW R15, (R10) + MOVB BP, 2(R10) + ADDQ CX, R14 + ADDQ CX, R10 + JMP copy_5_end copy_5_move_4through7: MOVL (R14), R15 - MOVL -4(R14)(AX*1), BP + MOVL -4(R14)(CX*1), BP MOVL R15, (R10) - MOVL BP, -4(R10)(AX*1) - ADDQ AX, R14 - ADDQ AX, R10 + MOVL BP, -4(R10)(CX*1) + ADDQ CX, R14 + ADDQ CX, R10 JMP copy_5_end copy_5_move_8through16: MOVQ (R14), R15 - MOVQ -8(R14)(AX*1), BP + MOVQ -8(R14)(CX*1), BP MOVQ R15, (R10) - MOVQ BP, -8(R10)(AX*1) - ADDQ AX, R14 - ADDQ AX, R10 + MOVQ BP, -8(R10)(CX*1) + ADDQ CX, R14 + ADDQ CX, R10 copy_5_end: - ADDQ AX, R12 - SUBQ AX, R13 + ADDQ CX, R12 + SUBQ CX, AX // Copy match from the current buffer copy_match: - MOVQ R10, AX - SUBQ CX, AX + MOVQ R10, CX + SUBQ R13, CX // ml <= mo - CMPQ R13, CX + CMPQ AX, R13 JA copy_overlapping_match // Copy non-overlapping match - ADDQ R13, R12 - MOVQ R10, CX - ADDQ R13, R10 + ADDQ AX, R12 + MOVUPS (CX), X0 + MOVUPS X0, (R10) + CMPQ AX, $0x10 + JA copy_2_long + ADDQ AX, R10 + JMP handle_loop + +copy_2_long: + LEAQ 16(R10), R13 + ADDQ AX, R10 + ADDQ $0x10, CX + SUBQ $0x10, AX copy_2: - MOVUPS (AX), X0 - MOVUPS X0, (CX) - ADDQ $0x10, AX + MOVUPS (CX), X0 + MOVUPS X0, (R13) ADDQ $0x10, CX - SUBQ $0x10, R13 + ADDQ $0x10, R13 + SUBQ $0x10, AX JHI copy_2 JMP handle_loop // Copy overlapping match copy_overlapping_match: - ADDQ R13, R12 + ADDQ AX, R12 copy_slow_3: - MOVB (AX), CL - MOVB CL, (R10) - INCQ AX - INCQ R10 - DECQ R13 - JNZ copy_slow_3 + MOVBQZX (CX), R13 + MOVB R13, (R10) + INCQ CX + INCQ R10 + DECQ AX + JNZ copy_slow_3 handle_loop: - MOVQ ctx+16(FP), AX - DECQ 96(AX) + DECQ 32(SP) JNS sequenceDecs_decodeSync_amd64_main_loop loop_finished: @@ -2284,6 +2318,13 @@ loop_finished: MOVB BL, 40(AX) MOVQ SI, 32(AX) + // s.seqSize += outPosition - ctx.outPosition + MOVQ ctx+16(FP), AX + MOVQ s+0(FP), CX + MOVQ R12, DX + SUBQ 136(AX), DX + ADDQ DX, 256(CX) + // Update the context MOVQ ctx+16(FP), AX MOVQ R12, 136(AX) @@ -2297,7 +2338,7 @@ loop_finished: // Return with match length error sequenceDecs_decodeSync_amd64_error_match_len_ofs_mismatch: - MOVQ 16(SP), AX + MOVQ 48(SP), AX MOVQ ctx+16(FP), CX MOVQ AX, 216(CX) MOVQ $0x00000001, ret+24(FP) @@ -2306,7 +2347,7 @@ sequenceDecs_decodeSync_amd64_error_match_len_ofs_mismatch: // Return with match too long error sequenceDecs_decodeSync_amd64_error_match_len_too_big: MOVQ ctx+16(FP), AX - MOVQ 16(SP), CX + MOVQ 48(SP), CX MOVQ CX, 216(AX) MOVQ $0x00000002, ret+24(FP) RET @@ -2314,7 +2355,7 @@ sequenceDecs_decodeSync_amd64_error_match_len_too_big: // Return with match offset too long error error_match_off_too_big: MOVQ ctx+16(FP), AX - MOVQ 8(SP), CX + MOVQ 40(SP), CX MOVQ CX, 224(AX) MOVQ R12, 136(AX) MOVQ $0x00000003, ret+24(FP) @@ -2323,7 +2364,7 @@ error_match_off_too_big: // Return with not enough literals error error_not_enough_literals: MOVQ ctx+16(FP), AX - MOVQ 24(SP), CX + MOVQ 56(SP), CX MOVQ CX, 208(AX) MOVQ $0x00000004, ret+24(FP) RET @@ -2336,9 +2377,9 @@ error_overread: // Return with not enough output space error error_not_enough_space: MOVQ ctx+16(FP), AX - MOVQ 24(SP), CX + MOVQ 56(SP), CX MOVQ CX, 208(AX) - MOVQ 16(SP), CX + MOVQ 48(SP), CX MOVQ CX, 216(AX) MOVQ R12, 136(AX) MOVQ $0x00000005, ret+24(FP) @@ -2346,7 +2387,7 @@ error_not_enough_space: // func sequenceDecs_decodeSync_bmi2(s *sequenceDecs, br *bitReader, ctx *decodeSyncAsmContext) int // Requires: BMI, BMI2, CMOV, SSE -TEXT ·sequenceDecs_decodeSync_bmi2(SB), $64-32 +TEXT ·sequenceDecs_decodeSync_bmi2(SB), $96-32 MOVQ br+8(FP), BX MOVQ 24(BX), AX MOVBQZX 40(BX), DX @@ -2358,26 +2399,34 @@ TEXT ·sequenceDecs_decodeSync_bmi2(SB), $64-32 MOVQ 72(CX), SI MOVQ 80(CX), DI MOVQ 88(CX), R8 - XORQ R9, R9 + MOVQ (CX), R9 MOVQ R9, 8(SP) + MOVQ 24(CX), R9 MOVQ R9, 16(SP) + MOVQ 48(CX), R9 MOVQ R9, 24(SP) + MOVQ 96(CX), R9 + MOVQ R9, 32(SP) + XORQ R9, R9 + MOVQ R9, 40(SP) + MOVQ R9, 48(SP) + MOVQ R9, 56(SP) MOVQ 112(CX), R9 MOVQ 128(CX), R10 - MOVQ R10, 32(SP) + MOVQ R10, 64(SP) MOVQ 144(CX), R10 MOVQ 136(CX), R11 MOVQ 200(CX), R12 - MOVQ R12, 56(SP) + MOVQ R12, 88(SP) MOVQ 176(CX), R12 - MOVQ R12, 48(SP) + MOVQ R12, 80(SP) MOVQ 184(CX), CX - MOVQ CX, 40(SP) - MOVQ 40(SP), CX - ADDQ CX, 48(SP) + MOVQ CX, 72(SP) + MOVQ 72(SP), CX + ADDQ CX, 80(SP) // Calculate pointer to s.out[cap(s.out)] (a past-end pointer) - ADDQ R9, 32(SP) + ADDQ R9, 64(SP) // outBase += outPosition ADDQ R11, R9 @@ -2425,7 +2474,7 @@ sequenceDecs_decodeSync_bmi2_fill_end: MOVQ R8, CX SHRQ $0x20, CX ADDQ R14, CX - MOVQ CX, 8(SP) + MOVQ CX, 40(SP) // Update match length MOVQ $0x00000808, CX @@ -2438,7 +2487,7 @@ sequenceDecs_decodeSync_bmi2_fill_end: MOVQ DI, CX SHRQ $0x20, CX ADDQ R14, CX - MOVQ CX, 16(SP) + MOVQ CX, 48(SP) // Fill bitreader to have enough for the remaining CMPQ BX, $0x08 @@ -2480,14 +2529,13 @@ sequenceDecs_decodeSync_bmi2_fill_2_end: MOVQ SI, CX SHRQ $0x20, CX ADDQ R14, CX - MOVQ CX, 24(SP) + MOVQ CX, 56(SP) // Fill bitreader for state updates MOVQ R12, (SP) MOVQ $0x00000808, CX BEXTRQ CX, R8, R12 - MOVQ ctx+16(FP), CX - CMPQ 96(CX), $0x00 + CMPQ 32(SP), $0x00 JZ sequenceDecs_decodeSync_bmi2_skip_update LEAQ (SI)(DI*1), R13 ADDQ R8, R13 @@ -2505,8 +2553,7 @@ sequenceDecs_decodeSync_bmi2_fill_2_end: ADDQ CX, R8 // Load ctx.ofTable - MOVQ ctx+16(FP), CX - MOVQ 48(CX), CX + MOVQ 24(SP), CX MOVQ (CX)(R8*8), R8 // Update Match Length State @@ -2516,8 +2563,7 @@ sequenceDecs_decodeSync_bmi2_fill_2_end: ADDQ CX, DI // Load ctx.mlTable - MOVQ ctx+16(FP), CX - MOVQ 24(CX), CX + MOVQ 16(SP), CX MOVQ (CX)(DI*8), DI // Update Literal Length State @@ -2526,14 +2572,13 @@ sequenceDecs_decodeSync_bmi2_fill_2_end: ADDQ CX, SI // Load ctx.llTable - MOVQ ctx+16(FP), CX - MOVQ (CX), CX + MOVQ 8(SP), CX MOVQ (CX)(SI*8), SI sequenceDecs_decodeSync_bmi2_skip_update: // Adjust offset MOVQ s+0(FP), CX - MOVQ 8(SP), R13 + MOVQ 40(SP), R13 CMPQ R12, $0x01 JBE sequenceDecs_decodeSync_bmi2_adjust_offsetB_1_or_0 MOVUPS 144(CX), X0 @@ -2542,7 +2587,7 @@ sequenceDecs_decodeSync_bmi2_skip_update: JMP sequenceDecs_decodeSync_bmi2_after_adjust sequenceDecs_decodeSync_bmi2_adjust_offsetB_1_or_0: - CMPQ 24(SP), $0x00000000 + CMPQ 56(SP), $0x00000000 JNE sequenceDecs_decodeSync_bmi2_adjust_offset_maybezero INCQ R13 JMP sequenceDecs_decodeSync_bmi2_adjust_offset_nonzero @@ -2577,14 +2622,11 @@ sequenceDecs_decodeSync_bmi2_adjust_skip: MOVQ R14, R13 sequenceDecs_decodeSync_bmi2_after_adjust: - MOVQ R13, 8(SP) + MOVQ R13, 40(SP) // Check values - MOVQ 16(SP), CX - MOVQ 24(SP), R12 - LEAQ (CX)(R12*1), R14 - MOVQ s+0(FP), R15 - ADDQ R14, 256(R15) + MOVQ 48(SP), CX + MOVQ 56(SP), R12 MOVQ ctx+16(FP), R14 SUBQ R12, 104(R14) JS error_not_enough_literals @@ -2596,50 +2638,49 @@ sequenceDecs_decodeSync_bmi2_after_adjust: JNZ sequenceDecs_decodeSync_bmi2_error_match_len_ofs_mismatch sequenceDecs_decodeSync_bmi2_match_len_ofs_ok: - MOVQ 24(SP), CX - MOVQ 8(SP), R12 - MOVQ 16(SP), R13 - // Check if we have enough space in s.out - LEAQ 16(CX)(R13*1), R14 + LEAQ 16(R12)(CX*1), R14 ADDQ R9, R14 - CMPQ R14, 32(SP) + CMPQ R14, 64(SP) JA error_not_enough_space // Copy literals - TESTQ CX, CX - JZ check_offset - XORQ R14, R14 + MOVUPS (R10), X0 + MOVUPS X0, (R9) + CMPQ R12, $0x10 + JBE copy_1_end + MOVQ $0x00000010, R14 copy_1: MOVUPS (R10)(R14*1), X0 MOVUPS X0, (R9)(R14*1) ADDQ $0x10, R14 - CMPQ R14, CX + CMPQ R14, R12 JB copy_1 - ADDQ CX, R10 - ADDQ CX, R9 - ADDQ CX, R11 + +copy_1_end: + ADDQ R12, R10 + ADDQ R12, R9 + ADDQ R12, R11 // Malformed input if seq.mo > t+len(hist) || seq.mo > s.windowSize) -check_offset: - MOVQ R11, CX - ADDQ 40(SP), CX - CMPQ R12, CX + MOVQ R11, R12 + ADDQ 72(SP), R12 + CMPQ R13, R12 JG error_match_off_too_big - CMPQ R12, 56(SP) + CMPQ R13, 88(SP) JG error_match_off_too_big // Copy match from history - MOVQ R12, CX - SUBQ R11, CX + MOVQ R13, R12 + SUBQ R11, R12 JLS copy_match - MOVQ 48(SP), R14 - SUBQ CX, R14 - CMPQ R13, CX + MOVQ 80(SP), R14 + SUBQ R12, R14 + CMPQ CX, R12 JG copy_all_from_history - MOVQ R13, CX - SUBQ $0x10, CX + MOVQ CX, R12 + SUBQ $0x10, R12 JB copy_4_small copy_4_loop: @@ -2647,54 +2688,54 @@ copy_4_loop: MOVUPS X0, (R9) ADDQ $0x10, R14 ADDQ $0x10, R9 - SUBQ $0x10, CX + SUBQ $0x10, R12 JAE copy_4_loop - LEAQ 16(R14)(CX*1), R14 - LEAQ 16(R9)(CX*1), R9 + LEAQ 16(R14)(R12*1), R14 + LEAQ 16(R9)(R12*1), R9 MOVUPS -16(R14), X0 MOVUPS X0, -16(R9) JMP copy_4_end copy_4_small: - CMPQ R13, $0x03 + CMPQ CX, $0x03 JE copy_4_move_3 - CMPQ R13, $0x08 + CMPQ CX, $0x08 JB copy_4_move_4through7 JMP copy_4_move_8through16 copy_4_move_3: - MOVW (R14), CX - MOVB 2(R14), R12 - MOVW CX, (R9) - MOVB R12, 2(R9) - ADDQ R13, R14 - ADDQ R13, R9 - JMP copy_4_end + MOVWQZX (R14), R12 + MOVB 2(R14), R13 + MOVW R12, (R9) + MOVB R13, 2(R9) + ADDQ CX, R14 + ADDQ CX, R9 + JMP copy_4_end copy_4_move_4through7: - MOVL (R14), CX - MOVL -4(R14)(R13*1), R12 - MOVL CX, (R9) - MOVL R12, -4(R9)(R13*1) - ADDQ R13, R14 - ADDQ R13, R9 + MOVL (R14), R12 + MOVL -4(R14)(CX*1), R13 + MOVL R12, (R9) + MOVL R13, -4(R9)(CX*1) + ADDQ CX, R14 + ADDQ CX, R9 JMP copy_4_end copy_4_move_8through16: - MOVQ (R14), CX - MOVQ -8(R14)(R13*1), R12 - MOVQ CX, (R9) - MOVQ R12, -8(R9)(R13*1) - ADDQ R13, R14 - ADDQ R13, R9 + MOVQ (R14), R12 + MOVQ -8(R14)(CX*1), R13 + MOVQ R12, (R9) + MOVQ R13, -8(R9)(CX*1) + ADDQ CX, R14 + ADDQ CX, R9 copy_4_end: - ADDQ R13, R11 + ADDQ CX, R11 JMP handle_loop JMP loop_finished copy_all_from_history: - MOVQ CX, R15 + MOVQ R12, R15 SUBQ $0x10, R15 JB copy_5_small @@ -2712,90 +2753,99 @@ copy_5_loop: JMP copy_5_end copy_5_small: - CMPQ CX, $0x03 + CMPQ R12, $0x03 JE copy_5_move_3 JB copy_5_move_1or2 - CMPQ CX, $0x08 + CMPQ R12, $0x08 JB copy_5_move_4through7 JMP copy_5_move_8through16 copy_5_move_1or2: MOVB (R14), R15 - MOVB -1(R14)(CX*1), BP + MOVB -1(R14)(R12*1), BP MOVB R15, (R9) - MOVB BP, -1(R9)(CX*1) - ADDQ CX, R14 - ADDQ CX, R9 + MOVB BP, -1(R9)(R12*1) + ADDQ R12, R14 + ADDQ R12, R9 JMP copy_5_end copy_5_move_3: - MOVW (R14), R15 - MOVB 2(R14), BP - MOVW R15, (R9) - MOVB BP, 2(R9) - ADDQ CX, R14 - ADDQ CX, R9 - JMP copy_5_end + MOVWQZX (R14), R15 + MOVB 2(R14), BP + MOVW R15, (R9) + MOVB BP, 2(R9) + ADDQ R12, R14 + ADDQ R12, R9 + JMP copy_5_end copy_5_move_4through7: MOVL (R14), R15 - MOVL -4(R14)(CX*1), BP + MOVL -4(R14)(R12*1), BP MOVL R15, (R9) - MOVL BP, -4(R9)(CX*1) - ADDQ CX, R14 - ADDQ CX, R9 + MOVL BP, -4(R9)(R12*1) + ADDQ R12, R14 + ADDQ R12, R9 JMP copy_5_end copy_5_move_8through16: MOVQ (R14), R15 - MOVQ -8(R14)(CX*1), BP + MOVQ -8(R14)(R12*1), BP MOVQ R15, (R9) - MOVQ BP, -8(R9)(CX*1) - ADDQ CX, R14 - ADDQ CX, R9 + MOVQ BP, -8(R9)(R12*1) + ADDQ R12, R14 + ADDQ R12, R9 copy_5_end: - ADDQ CX, R11 - SUBQ CX, R13 + ADDQ R12, R11 + SUBQ R12, CX // Copy match from the current buffer copy_match: - MOVQ R9, CX - SUBQ R12, CX + MOVQ R9, R12 + SUBQ R13, R12 // ml <= mo - CMPQ R13, R12 + CMPQ CX, R13 JA copy_overlapping_match // Copy non-overlapping match - ADDQ R13, R11 - MOVQ R9, R12 - ADDQ R13, R9 + ADDQ CX, R11 + MOVUPS (R12), X0 + MOVUPS X0, (R9) + CMPQ CX, $0x10 + JA copy_2_long + ADDQ CX, R9 + JMP handle_loop + +copy_2_long: + LEAQ 16(R9), R13 + ADDQ CX, R9 + ADDQ $0x10, R12 + SUBQ $0x10, CX copy_2: - MOVUPS (CX), X0 - MOVUPS X0, (R12) - ADDQ $0x10, CX + MOVUPS (R12), X0 + MOVUPS X0, (R13) ADDQ $0x10, R12 - SUBQ $0x10, R13 + ADDQ $0x10, R13 + SUBQ $0x10, CX JHI copy_2 JMP handle_loop // Copy overlapping match copy_overlapping_match: - ADDQ R13, R11 + ADDQ CX, R11 copy_slow_3: - MOVB (CX), R12 - MOVB R12, (R9) - INCQ CX - INCQ R9 - DECQ R13 - JNZ copy_slow_3 + MOVBQZX (R12), R13 + MOVB R13, (R9) + INCQ R12 + INCQ R9 + DECQ CX + JNZ copy_slow_3 handle_loop: - MOVQ ctx+16(FP), CX - DECQ 96(CX) + DECQ 32(SP) JNS sequenceDecs_decodeSync_bmi2_main_loop loop_finished: @@ -2804,6 +2854,13 @@ loop_finished: MOVB DL, 40(CX) MOVQ BX, 32(CX) + // s.seqSize += outPosition - ctx.outPosition + MOVQ ctx+16(FP), AX + MOVQ s+0(FP), CX + MOVQ R11, DX + SUBQ 136(AX), DX + ADDQ DX, 256(CX) + // Update the context MOVQ ctx+16(FP), AX MOVQ R11, 136(AX) @@ -2817,7 +2874,7 @@ loop_finished: // Return with match length error sequenceDecs_decodeSync_bmi2_error_match_len_ofs_mismatch: - MOVQ 16(SP), AX + MOVQ 48(SP), AX MOVQ ctx+16(FP), CX MOVQ AX, 216(CX) MOVQ $0x00000001, ret+24(FP) @@ -2826,7 +2883,7 @@ sequenceDecs_decodeSync_bmi2_error_match_len_ofs_mismatch: // Return with match too long error sequenceDecs_decodeSync_bmi2_error_match_len_too_big: MOVQ ctx+16(FP), AX - MOVQ 16(SP), CX + MOVQ 48(SP), CX MOVQ CX, 216(AX) MOVQ $0x00000002, ret+24(FP) RET @@ -2834,7 +2891,7 @@ sequenceDecs_decodeSync_bmi2_error_match_len_too_big: // Return with match offset too long error error_match_off_too_big: MOVQ ctx+16(FP), AX - MOVQ 8(SP), CX + MOVQ 40(SP), CX MOVQ CX, 224(AX) MOVQ R11, 136(AX) MOVQ $0x00000003, ret+24(FP) @@ -2843,7 +2900,7 @@ error_match_off_too_big: // Return with not enough literals error error_not_enough_literals: MOVQ ctx+16(FP), AX - MOVQ 24(SP), CX + MOVQ 56(SP), CX MOVQ CX, 208(AX) MOVQ $0x00000004, ret+24(FP) RET @@ -2856,9 +2913,9 @@ error_overread: // Return with not enough output space error error_not_enough_space: MOVQ ctx+16(FP), AX - MOVQ 24(SP), CX + MOVQ 56(SP), CX MOVQ CX, 208(AX) - MOVQ 16(SP), CX + MOVQ 48(SP), CX MOVQ CX, 216(AX) MOVQ R11, 136(AX) MOVQ $0x00000005, ret+24(FP) @@ -2866,7 +2923,7 @@ error_not_enough_space: // func sequenceDecs_decodeSync_safe_amd64(s *sequenceDecs, br *bitReader, ctx *decodeSyncAsmContext) int // Requires: CMOV, SSE -TEXT ·sequenceDecs_decodeSync_safe_amd64(SB), $64-32 +TEXT ·sequenceDecs_decodeSync_safe_amd64(SB), $96-32 MOVQ br+8(FP), CX MOVQ 24(CX), DX MOVBQZX 40(CX), BX @@ -2878,26 +2935,34 @@ TEXT ·sequenceDecs_decodeSync_safe_amd64(SB), $64-32 MOVQ 72(AX), DI MOVQ 80(AX), R8 MOVQ 88(AX), R9 - XORQ CX, CX + MOVQ (AX), CX MOVQ CX, 8(SP) + MOVQ 24(AX), CX MOVQ CX, 16(SP) + MOVQ 48(AX), CX MOVQ CX, 24(SP) + MOVQ 96(AX), CX + MOVQ CX, 32(SP) + XORQ CX, CX + MOVQ CX, 40(SP) + MOVQ CX, 48(SP) + MOVQ CX, 56(SP) MOVQ 112(AX), R10 MOVQ 128(AX), CX - MOVQ CX, 32(SP) + MOVQ CX, 64(SP) MOVQ 144(AX), R11 MOVQ 136(AX), R12 MOVQ 200(AX), CX - MOVQ CX, 56(SP) + MOVQ CX, 88(SP) MOVQ 176(AX), CX - MOVQ CX, 48(SP) + MOVQ CX, 80(SP) MOVQ 184(AX), AX - MOVQ AX, 40(SP) - MOVQ 40(SP), AX - ADDQ AX, 48(SP) + MOVQ AX, 72(SP) + MOVQ 72(SP), AX + ADDQ AX, 80(SP) // Calculate pointer to s.out[cap(s.out)] (a past-end pointer) - ADDQ R10, 32(SP) + ADDQ R10, 64(SP) // outBase += outPosition ADDQ R12, R10 @@ -2935,46 +3000,32 @@ sequenceDecs_decodeSync_safe_amd64_fill_check_overread: sequenceDecs_decodeSync_safe_amd64_fill_end: // Update offset - MOVQ R9, AX - MOVQ BX, CX - MOVQ DX, R14 - SHLQ CL, R14 - MOVB AH, CL - SHRQ $0x20, AX - TESTQ CX, CX - JZ sequenceDecs_decodeSync_safe_amd64_of_update_zero - ADDQ CX, BX - CMPQ BX, $0x40 - JA sequenceDecs_decodeSync_safe_amd64_of_update_zero - CMPQ CX, $0x40 - JAE sequenceDecs_decodeSync_safe_amd64_of_update_zero - NEGQ CX - SHRQ CL, R14 - ADDQ R14, AX - -sequenceDecs_decodeSync_safe_amd64_of_update_zero: - MOVQ AX, 8(SP) + MOVQ R9, AX + MOVQ BX, CX + MOVQ DX, R14 + SHLQ CL, R14 + SHRQ $0x01, R14 + MOVBLZX AH, CX + SHRQ $0x20, AX + ADDQ CX, BX + XORQ $0x3f, CX + SHRQ CL, R14 + ADDQ R14, AX + MOVQ AX, 40(SP) // Update match length - MOVQ R8, AX - MOVQ BX, CX - MOVQ DX, R14 - SHLQ CL, R14 - MOVB AH, CL - SHRQ $0x20, AX - TESTQ CX, CX - JZ sequenceDecs_decodeSync_safe_amd64_ml_update_zero - ADDQ CX, BX - CMPQ BX, $0x40 - JA sequenceDecs_decodeSync_safe_amd64_ml_update_zero - CMPQ CX, $0x40 - JAE sequenceDecs_decodeSync_safe_amd64_ml_update_zero - NEGQ CX - SHRQ CL, R14 - ADDQ R14, AX - -sequenceDecs_decodeSync_safe_amd64_ml_update_zero: - MOVQ AX, 16(SP) + MOVQ R8, AX + MOVQ BX, CX + MOVQ DX, R14 + SHLQ CL, R14 + SHRQ $0x01, R14 + MOVBLZX AH, CX + SHRQ $0x20, AX + ADDQ CX, BX + XORQ $0x3f, CX + SHRQ CL, R14 + ADDQ R14, AX + MOVQ AX, 48(SP) // Fill bitreader to have enough for the remaining CMPQ SI, $0x08 @@ -3006,96 +3057,82 @@ sequenceDecs_decodeSync_safe_amd64_fill_2_check_overread: sequenceDecs_decodeSync_safe_amd64_fill_2_end: // Update literal length - MOVQ DI, AX - MOVQ BX, CX - MOVQ DX, R14 - SHLQ CL, R14 - MOVB AH, CL - SHRQ $0x20, AX - TESTQ CX, CX - JZ sequenceDecs_decodeSync_safe_amd64_ll_update_zero - ADDQ CX, BX - CMPQ BX, $0x40 - JA sequenceDecs_decodeSync_safe_amd64_ll_update_zero - CMPQ CX, $0x40 - JAE sequenceDecs_decodeSync_safe_amd64_ll_update_zero - NEGQ CX - SHRQ CL, R14 - ADDQ R14, AX - -sequenceDecs_decodeSync_safe_amd64_ll_update_zero: - MOVQ AX, 24(SP) + MOVQ DI, AX + MOVQ BX, CX + MOVQ DX, R14 + SHLQ CL, R14 + SHRQ $0x01, R14 + MOVBLZX AH, CX + SHRQ $0x20, AX + ADDQ CX, BX + XORQ $0x3f, CX + SHRQ CL, R14 + ADDQ R14, AX + MOVQ AX, 56(SP) // Fill bitreader for state updates MOVQ R13, (SP) MOVQ R9, AX SHRQ $0x08, AX MOVBQZX AL, AX - MOVQ ctx+16(FP), CX - CMPQ 96(CX), $0x00 + CMPQ 32(SP), $0x00 JZ sequenceDecs_decodeSync_safe_amd64_skip_update // Update Literal Length State MOVBQZX DI, R13 SHRL $0x10, DI - LEAQ (BX)(R13*1), CX + MOVQ BX, CX MOVQ DX, R14 - MOVQ CX, BX - ROLQ CL, R14 - MOVL $0x00000001, R15 - MOVB R13, CL - SHLL CL, R15 - DECL R15 - ANDQ R15, R14 + SHLQ CL, R14 + SHRQ $0x01, R14 + ADDQ R13, BX + XORQ $0x3f, R13 + MOVQ R13, CX + SHRQ CL, R14 ADDQ R14, DI // Load ctx.llTable - MOVQ ctx+16(FP), CX - MOVQ (CX), CX + MOVQ 8(SP), CX MOVQ (CX)(DI*8), DI // Update Match Length State MOVBQZX R8, R13 SHRL $0x10, R8 - LEAQ (BX)(R13*1), CX + MOVQ BX, CX MOVQ DX, R14 - MOVQ CX, BX - ROLQ CL, R14 - MOVL $0x00000001, R15 - MOVB R13, CL - SHLL CL, R15 - DECL R15 - ANDQ R15, R14 + SHLQ CL, R14 + SHRQ $0x01, R14 + ADDQ R13, BX + XORQ $0x3f, R13 + MOVQ R13, CX + SHRQ CL, R14 ADDQ R14, R8 // Load ctx.mlTable - MOVQ ctx+16(FP), CX - MOVQ 24(CX), CX + MOVQ 16(SP), CX MOVQ (CX)(R8*8), R8 // Update Offset State MOVBQZX R9, R13 SHRL $0x10, R9 - LEAQ (BX)(R13*1), CX + MOVQ BX, CX MOVQ DX, R14 - MOVQ CX, BX - ROLQ CL, R14 - MOVL $0x00000001, R15 - MOVB R13, CL - SHLL CL, R15 - DECL R15 - ANDQ R15, R14 + SHLQ CL, R14 + SHRQ $0x01, R14 + ADDQ R13, BX + XORQ $0x3f, R13 + MOVQ R13, CX + SHRQ CL, R14 ADDQ R14, R9 // Load ctx.ofTable - MOVQ ctx+16(FP), CX - MOVQ 48(CX), CX + MOVQ 24(SP), CX MOVQ (CX)(R9*8), R9 sequenceDecs_decodeSync_safe_amd64_skip_update: // Adjust offset MOVQ s+0(FP), CX - MOVQ 8(SP), R13 + MOVQ 40(SP), R13 CMPQ AX, $0x01 JBE sequenceDecs_decodeSync_safe_amd64_adjust_offsetB_1_or_0 MOVUPS 144(CX), X0 @@ -3104,7 +3141,7 @@ sequenceDecs_decodeSync_safe_amd64_skip_update: JMP sequenceDecs_decodeSync_safe_amd64_after_adjust sequenceDecs_decodeSync_safe_amd64_adjust_offsetB_1_or_0: - CMPQ 24(SP), $0x00000000 + CMPQ 56(SP), $0x00000000 JNE sequenceDecs_decodeSync_safe_amd64_adjust_offset_maybezero INCQ R13 JMP sequenceDecs_decodeSync_safe_amd64_adjust_offset_nonzero @@ -3139,14 +3176,11 @@ sequenceDecs_decodeSync_safe_amd64_adjust_skip: MOVQ R14, R13 sequenceDecs_decodeSync_safe_amd64_after_adjust: - MOVQ R13, 8(SP) + MOVQ R13, 40(SP) // Check values - MOVQ 16(SP), AX - MOVQ 24(SP), CX - LEAQ (AX)(CX*1), R14 - MOVQ s+0(FP), R15 - ADDQ R14, 256(R15) + MOVQ 48(SP), AX + MOVQ 56(SP), CX MOVQ ctx+16(FP), R14 SUBQ CX, 104(R14) JS error_not_enough_literals @@ -3158,20 +3192,16 @@ sequenceDecs_decodeSync_safe_amd64_after_adjust: JNZ sequenceDecs_decodeSync_safe_amd64_error_match_len_ofs_mismatch sequenceDecs_decodeSync_safe_amd64_match_len_ofs_ok: - MOVQ 24(SP), AX - MOVQ 8(SP), CX - MOVQ 16(SP), R13 - // Check if we have enough space in s.out - LEAQ (AX)(R13*1), R14 + LEAQ (CX)(AX*1), R14 ADDQ R10, R14 - CMPQ R14, 32(SP) + CMPQ R14, 64(SP) JA error_not_enough_space // Copy literals - TESTQ AX, AX + TESTQ CX, CX JZ check_offset - MOVQ AX, R14 + MOVQ CX, R14 SUBQ $0x10, R14 JB copy_1_small @@ -3189,70 +3219,70 @@ copy_1_loop: JMP copy_1_end copy_1_small: - CMPQ AX, $0x03 + CMPQ CX, $0x03 JE copy_1_move_3 JB copy_1_move_1or2 - CMPQ AX, $0x08 + CMPQ CX, $0x08 JB copy_1_move_4through7 JMP copy_1_move_8through16 copy_1_move_1or2: MOVB (R11), R14 - MOVB -1(R11)(AX*1), R15 + MOVB -1(R11)(CX*1), R15 MOVB R14, (R10) - MOVB R15, -1(R10)(AX*1) - ADDQ AX, R11 - ADDQ AX, R10 + MOVB R15, -1(R10)(CX*1) + ADDQ CX, R11 + ADDQ CX, R10 JMP copy_1_end copy_1_move_3: - MOVW (R11), R14 - MOVB 2(R11), R15 - MOVW R14, (R10) - MOVB R15, 2(R10) - ADDQ AX, R11 - ADDQ AX, R10 - JMP copy_1_end + MOVWQZX (R11), R14 + MOVB 2(R11), R15 + MOVW R14, (R10) + MOVB R15, 2(R10) + ADDQ CX, R11 + ADDQ CX, R10 + JMP copy_1_end copy_1_move_4through7: MOVL (R11), R14 - MOVL -4(R11)(AX*1), R15 + MOVL -4(R11)(CX*1), R15 MOVL R14, (R10) - MOVL R15, -4(R10)(AX*1) - ADDQ AX, R11 - ADDQ AX, R10 + MOVL R15, -4(R10)(CX*1) + ADDQ CX, R11 + ADDQ CX, R10 JMP copy_1_end copy_1_move_8through16: MOVQ (R11), R14 - MOVQ -8(R11)(AX*1), R15 + MOVQ -8(R11)(CX*1), R15 MOVQ R14, (R10) - MOVQ R15, -8(R10)(AX*1) - ADDQ AX, R11 - ADDQ AX, R10 + MOVQ R15, -8(R10)(CX*1) + ADDQ CX, R11 + ADDQ CX, R10 copy_1_end: - ADDQ AX, R12 + ADDQ CX, R12 // Malformed input if seq.mo > t+len(hist) || seq.mo > s.windowSize) check_offset: - MOVQ R12, AX - ADDQ 40(SP), AX - CMPQ CX, AX + MOVQ R12, CX + ADDQ 72(SP), CX + CMPQ R13, CX JG error_match_off_too_big - CMPQ CX, 56(SP) + CMPQ R13, 88(SP) JG error_match_off_too_big // Copy match from history - MOVQ CX, AX - SUBQ R12, AX + MOVQ R13, CX + SUBQ R12, CX JLS copy_match - MOVQ 48(SP), R14 - SUBQ AX, R14 - CMPQ R13, AX + MOVQ 80(SP), R14 + SUBQ CX, R14 + CMPQ AX, CX JG copy_all_from_history - MOVQ R13, AX - SUBQ $0x10, AX + MOVQ AX, CX + SUBQ $0x10, CX JB copy_4_small copy_4_loop: @@ -3260,54 +3290,54 @@ copy_4_loop: MOVUPS X0, (R10) ADDQ $0x10, R14 ADDQ $0x10, R10 - SUBQ $0x10, AX + SUBQ $0x10, CX JAE copy_4_loop - LEAQ 16(R14)(AX*1), R14 - LEAQ 16(R10)(AX*1), R10 + LEAQ 16(R14)(CX*1), R14 + LEAQ 16(R10)(CX*1), R10 MOVUPS -16(R14), X0 MOVUPS X0, -16(R10) JMP copy_4_end copy_4_small: - CMPQ R13, $0x03 + CMPQ AX, $0x03 JE copy_4_move_3 - CMPQ R13, $0x08 + CMPQ AX, $0x08 JB copy_4_move_4through7 JMP copy_4_move_8through16 copy_4_move_3: - MOVW (R14), AX - MOVB 2(R14), CL - MOVW AX, (R10) - MOVB CL, 2(R10) - ADDQ R13, R14 - ADDQ R13, R10 - JMP copy_4_end + MOVWQZX (R14), CX + MOVB 2(R14), R13 + MOVW CX, (R10) + MOVB R13, 2(R10) + ADDQ AX, R14 + ADDQ AX, R10 + JMP copy_4_end copy_4_move_4through7: - MOVL (R14), AX - MOVL -4(R14)(R13*1), CX - MOVL AX, (R10) - MOVL CX, -4(R10)(R13*1) - ADDQ R13, R14 - ADDQ R13, R10 + MOVL (R14), CX + MOVL -4(R14)(AX*1), R13 + MOVL CX, (R10) + MOVL R13, -4(R10)(AX*1) + ADDQ AX, R14 + ADDQ AX, R10 JMP copy_4_end copy_4_move_8through16: - MOVQ (R14), AX - MOVQ -8(R14)(R13*1), CX - MOVQ AX, (R10) - MOVQ CX, -8(R10)(R13*1) - ADDQ R13, R14 - ADDQ R13, R10 + MOVQ (R14), CX + MOVQ -8(R14)(AX*1), R13 + MOVQ CX, (R10) + MOVQ R13, -8(R10)(AX*1) + ADDQ AX, R14 + ADDQ AX, R10 copy_4_end: - ADDQ R13, R12 + ADDQ AX, R12 JMP handle_loop JMP loop_finished copy_all_from_history: - MOVQ AX, R15 + MOVQ CX, R15 SUBQ $0x10, R15 JB copy_5_small @@ -3325,141 +3355,140 @@ copy_5_loop: JMP copy_5_end copy_5_small: - CMPQ AX, $0x03 + CMPQ CX, $0x03 JE copy_5_move_3 JB copy_5_move_1or2 - CMPQ AX, $0x08 + CMPQ CX, $0x08 JB copy_5_move_4through7 JMP copy_5_move_8through16 copy_5_move_1or2: MOVB (R14), R15 - MOVB -1(R14)(AX*1), BP + MOVB -1(R14)(CX*1), BP MOVB R15, (R10) - MOVB BP, -1(R10)(AX*1) - ADDQ AX, R14 - ADDQ AX, R10 + MOVB BP, -1(R10)(CX*1) + ADDQ CX, R14 + ADDQ CX, R10 JMP copy_5_end copy_5_move_3: - MOVW (R14), R15 - MOVB 2(R14), BP - MOVW R15, (R10) - MOVB BP, 2(R10) - ADDQ AX, R14 - ADDQ AX, R10 - JMP copy_5_end + MOVWQZX (R14), R15 + MOVB 2(R14), BP + MOVW R15, (R10) + MOVB BP, 2(R10) + ADDQ CX, R14 + ADDQ CX, R10 + JMP copy_5_end copy_5_move_4through7: MOVL (R14), R15 - MOVL -4(R14)(AX*1), BP + MOVL -4(R14)(CX*1), BP MOVL R15, (R10) - MOVL BP, -4(R10)(AX*1) - ADDQ AX, R14 - ADDQ AX, R10 + MOVL BP, -4(R10)(CX*1) + ADDQ CX, R14 + ADDQ CX, R10 JMP copy_5_end copy_5_move_8through16: MOVQ (R14), R15 - MOVQ -8(R14)(AX*1), BP + MOVQ -8(R14)(CX*1), BP MOVQ R15, (R10) - MOVQ BP, -8(R10)(AX*1) - ADDQ AX, R14 - ADDQ AX, R10 + MOVQ BP, -8(R10)(CX*1) + ADDQ CX, R14 + ADDQ CX, R10 copy_5_end: - ADDQ AX, R12 - SUBQ AX, R13 + ADDQ CX, R12 + SUBQ CX, AX // Copy match from the current buffer copy_match: - MOVQ R10, AX - SUBQ CX, AX + MOVQ R10, CX + SUBQ R13, CX // ml <= mo - CMPQ R13, CX + CMPQ AX, R13 JA copy_overlapping_match // Copy non-overlapping match - ADDQ R13, R12 - MOVQ R13, CX - SUBQ $0x10, CX + ADDQ AX, R12 + MOVQ AX, R13 + SUBQ $0x10, R13 JB copy_2_small copy_2_loop: - MOVUPS (AX), X0 + MOVUPS (CX), X0 MOVUPS X0, (R10) - ADDQ $0x10, AX + ADDQ $0x10, CX ADDQ $0x10, R10 - SUBQ $0x10, CX + SUBQ $0x10, R13 JAE copy_2_loop - LEAQ 16(AX)(CX*1), AX - LEAQ 16(R10)(CX*1), R10 - MOVUPS -16(AX), X0 + LEAQ 16(CX)(R13*1), CX + LEAQ 16(R10)(R13*1), R10 + MOVUPS -16(CX), X0 MOVUPS X0, -16(R10) JMP copy_2_end copy_2_small: - CMPQ R13, $0x03 + CMPQ AX, $0x03 JE copy_2_move_3 JB copy_2_move_1or2 - CMPQ R13, $0x08 + CMPQ AX, $0x08 JB copy_2_move_4through7 JMP copy_2_move_8through16 copy_2_move_1or2: - MOVB (AX), CL - MOVB -1(AX)(R13*1), R14 - MOVB CL, (R10) - MOVB R14, -1(R10)(R13*1) - ADDQ R13, AX - ADDQ R13, R10 + MOVB (CX), R13 + MOVB -1(CX)(AX*1), R14 + MOVB R13, (R10) + MOVB R14, -1(R10)(AX*1) + ADDQ AX, CX + ADDQ AX, R10 JMP copy_2_end copy_2_move_3: - MOVW (AX), CX - MOVB 2(AX), R14 - MOVW CX, (R10) - MOVB R14, 2(R10) - ADDQ R13, AX - ADDQ R13, R10 - JMP copy_2_end + MOVWQZX (CX), R13 + MOVB 2(CX), R14 + MOVW R13, (R10) + MOVB R14, 2(R10) + ADDQ AX, CX + ADDQ AX, R10 + JMP copy_2_end copy_2_move_4through7: - MOVL (AX), CX - MOVL -4(AX)(R13*1), R14 - MOVL CX, (R10) - MOVL R14, -4(R10)(R13*1) - ADDQ R13, AX - ADDQ R13, R10 + MOVL (CX), R13 + MOVL -4(CX)(AX*1), R14 + MOVL R13, (R10) + MOVL R14, -4(R10)(AX*1) + ADDQ AX, CX + ADDQ AX, R10 JMP copy_2_end copy_2_move_8through16: - MOVQ (AX), CX - MOVQ -8(AX)(R13*1), R14 - MOVQ CX, (R10) - MOVQ R14, -8(R10)(R13*1) - ADDQ R13, AX - ADDQ R13, R10 + MOVQ (CX), R13 + MOVQ -8(CX)(AX*1), R14 + MOVQ R13, (R10) + MOVQ R14, -8(R10)(AX*1) + ADDQ AX, CX + ADDQ AX, R10 copy_2_end: JMP handle_loop // Copy overlapping match copy_overlapping_match: - ADDQ R13, R12 + ADDQ AX, R12 copy_slow_3: - MOVB (AX), CL - MOVB CL, (R10) - INCQ AX - INCQ R10 - DECQ R13 - JNZ copy_slow_3 + MOVBQZX (CX), R13 + MOVB R13, (R10) + INCQ CX + INCQ R10 + DECQ AX + JNZ copy_slow_3 handle_loop: - MOVQ ctx+16(FP), AX - DECQ 96(AX) + DECQ 32(SP) JNS sequenceDecs_decodeSync_safe_amd64_main_loop loop_finished: @@ -3468,6 +3497,13 @@ loop_finished: MOVB BL, 40(AX) MOVQ SI, 32(AX) + // s.seqSize += outPosition - ctx.outPosition + MOVQ ctx+16(FP), AX + MOVQ s+0(FP), CX + MOVQ R12, DX + SUBQ 136(AX), DX + ADDQ DX, 256(CX) + // Update the context MOVQ ctx+16(FP), AX MOVQ R12, 136(AX) @@ -3481,7 +3517,7 @@ loop_finished: // Return with match length error sequenceDecs_decodeSync_safe_amd64_error_match_len_ofs_mismatch: - MOVQ 16(SP), AX + MOVQ 48(SP), AX MOVQ ctx+16(FP), CX MOVQ AX, 216(CX) MOVQ $0x00000001, ret+24(FP) @@ -3490,7 +3526,7 @@ sequenceDecs_decodeSync_safe_amd64_error_match_len_ofs_mismatch: // Return with match too long error sequenceDecs_decodeSync_safe_amd64_error_match_len_too_big: MOVQ ctx+16(FP), AX - MOVQ 16(SP), CX + MOVQ 48(SP), CX MOVQ CX, 216(AX) MOVQ $0x00000002, ret+24(FP) RET @@ -3498,7 +3534,7 @@ sequenceDecs_decodeSync_safe_amd64_error_match_len_too_big: // Return with match offset too long error error_match_off_too_big: MOVQ ctx+16(FP), AX - MOVQ 8(SP), CX + MOVQ 40(SP), CX MOVQ CX, 224(AX) MOVQ R12, 136(AX) MOVQ $0x00000003, ret+24(FP) @@ -3507,7 +3543,7 @@ error_match_off_too_big: // Return with not enough literals error error_not_enough_literals: MOVQ ctx+16(FP), AX - MOVQ 24(SP), CX + MOVQ 56(SP), CX MOVQ CX, 208(AX) MOVQ $0x00000004, ret+24(FP) RET @@ -3520,9 +3556,9 @@ error_overread: // Return with not enough output space error error_not_enough_space: MOVQ ctx+16(FP), AX - MOVQ 24(SP), CX + MOVQ 56(SP), CX MOVQ CX, 208(AX) - MOVQ 16(SP), CX + MOVQ 48(SP), CX MOVQ CX, 216(AX) MOVQ R12, 136(AX) MOVQ $0x00000005, ret+24(FP) @@ -3530,7 +3566,7 @@ error_not_enough_space: // func sequenceDecs_decodeSync_safe_bmi2(s *sequenceDecs, br *bitReader, ctx *decodeSyncAsmContext) int // Requires: BMI, BMI2, CMOV, SSE -TEXT ·sequenceDecs_decodeSync_safe_bmi2(SB), $64-32 +TEXT ·sequenceDecs_decodeSync_safe_bmi2(SB), $96-32 MOVQ br+8(FP), BX MOVQ 24(BX), AX MOVBQZX 40(BX), DX @@ -3542,26 +3578,34 @@ TEXT ·sequenceDecs_decodeSync_safe_bmi2(SB), $64-32 MOVQ 72(CX), SI MOVQ 80(CX), DI MOVQ 88(CX), R8 - XORQ R9, R9 + MOVQ (CX), R9 MOVQ R9, 8(SP) + MOVQ 24(CX), R9 MOVQ R9, 16(SP) + MOVQ 48(CX), R9 MOVQ R9, 24(SP) + MOVQ 96(CX), R9 + MOVQ R9, 32(SP) + XORQ R9, R9 + MOVQ R9, 40(SP) + MOVQ R9, 48(SP) + MOVQ R9, 56(SP) MOVQ 112(CX), R9 MOVQ 128(CX), R10 - MOVQ R10, 32(SP) + MOVQ R10, 64(SP) MOVQ 144(CX), R10 MOVQ 136(CX), R11 MOVQ 200(CX), R12 - MOVQ R12, 56(SP) + MOVQ R12, 88(SP) MOVQ 176(CX), R12 - MOVQ R12, 48(SP) + MOVQ R12, 80(SP) MOVQ 184(CX), CX - MOVQ CX, 40(SP) - MOVQ 40(SP), CX - ADDQ CX, 48(SP) + MOVQ CX, 72(SP) + MOVQ 72(SP), CX + ADDQ CX, 80(SP) // Calculate pointer to s.out[cap(s.out)] (a past-end pointer) - ADDQ R9, 32(SP) + ADDQ R9, 64(SP) // outBase += outPosition ADDQ R11, R9 @@ -3609,7 +3653,7 @@ sequenceDecs_decodeSync_safe_bmi2_fill_end: MOVQ R8, CX SHRQ $0x20, CX ADDQ R14, CX - MOVQ CX, 8(SP) + MOVQ CX, 40(SP) // Update match length MOVQ $0x00000808, CX @@ -3622,7 +3666,7 @@ sequenceDecs_decodeSync_safe_bmi2_fill_end: MOVQ DI, CX SHRQ $0x20, CX ADDQ R14, CX - MOVQ CX, 16(SP) + MOVQ CX, 48(SP) // Fill bitreader to have enough for the remaining CMPQ BX, $0x08 @@ -3664,14 +3708,13 @@ sequenceDecs_decodeSync_safe_bmi2_fill_2_end: MOVQ SI, CX SHRQ $0x20, CX ADDQ R14, CX - MOVQ CX, 24(SP) + MOVQ CX, 56(SP) // Fill bitreader for state updates MOVQ R12, (SP) MOVQ $0x00000808, CX BEXTRQ CX, R8, R12 - MOVQ ctx+16(FP), CX - CMPQ 96(CX), $0x00 + CMPQ 32(SP), $0x00 JZ sequenceDecs_decodeSync_safe_bmi2_skip_update LEAQ (SI)(DI*1), R13 ADDQ R8, R13 @@ -3689,8 +3732,7 @@ sequenceDecs_decodeSync_safe_bmi2_fill_2_end: ADDQ CX, R8 // Load ctx.ofTable - MOVQ ctx+16(FP), CX - MOVQ 48(CX), CX + MOVQ 24(SP), CX MOVQ (CX)(R8*8), R8 // Update Match Length State @@ -3700,8 +3742,7 @@ sequenceDecs_decodeSync_safe_bmi2_fill_2_end: ADDQ CX, DI // Load ctx.mlTable - MOVQ ctx+16(FP), CX - MOVQ 24(CX), CX + MOVQ 16(SP), CX MOVQ (CX)(DI*8), DI // Update Literal Length State @@ -3710,14 +3751,13 @@ sequenceDecs_decodeSync_safe_bmi2_fill_2_end: ADDQ CX, SI // Load ctx.llTable - MOVQ ctx+16(FP), CX - MOVQ (CX), CX + MOVQ 8(SP), CX MOVQ (CX)(SI*8), SI sequenceDecs_decodeSync_safe_bmi2_skip_update: // Adjust offset MOVQ s+0(FP), CX - MOVQ 8(SP), R13 + MOVQ 40(SP), R13 CMPQ R12, $0x01 JBE sequenceDecs_decodeSync_safe_bmi2_adjust_offsetB_1_or_0 MOVUPS 144(CX), X0 @@ -3726,7 +3766,7 @@ sequenceDecs_decodeSync_safe_bmi2_skip_update: JMP sequenceDecs_decodeSync_safe_bmi2_after_adjust sequenceDecs_decodeSync_safe_bmi2_adjust_offsetB_1_or_0: - CMPQ 24(SP), $0x00000000 + CMPQ 56(SP), $0x00000000 JNE sequenceDecs_decodeSync_safe_bmi2_adjust_offset_maybezero INCQ R13 JMP sequenceDecs_decodeSync_safe_bmi2_adjust_offset_nonzero @@ -3761,14 +3801,11 @@ sequenceDecs_decodeSync_safe_bmi2_adjust_skip: MOVQ R14, R13 sequenceDecs_decodeSync_safe_bmi2_after_adjust: - MOVQ R13, 8(SP) + MOVQ R13, 40(SP) // Check values - MOVQ 16(SP), CX - MOVQ 24(SP), R12 - LEAQ (CX)(R12*1), R14 - MOVQ s+0(FP), R15 - ADDQ R14, 256(R15) + MOVQ 48(SP), CX + MOVQ 56(SP), R12 MOVQ ctx+16(FP), R14 SUBQ R12, 104(R14) JS error_not_enough_literals @@ -3780,20 +3817,16 @@ sequenceDecs_decodeSync_safe_bmi2_after_adjust: JNZ sequenceDecs_decodeSync_safe_bmi2_error_match_len_ofs_mismatch sequenceDecs_decodeSync_safe_bmi2_match_len_ofs_ok: - MOVQ 24(SP), CX - MOVQ 8(SP), R12 - MOVQ 16(SP), R13 - // Check if we have enough space in s.out - LEAQ (CX)(R13*1), R14 + LEAQ (R12)(CX*1), R14 ADDQ R9, R14 - CMPQ R14, 32(SP) + CMPQ R14, 64(SP) JA error_not_enough_space // Copy literals - TESTQ CX, CX + TESTQ R12, R12 JZ check_offset - MOVQ CX, R14 + MOVQ R12, R14 SUBQ $0x10, R14 JB copy_1_small @@ -3811,70 +3844,70 @@ copy_1_loop: JMP copy_1_end copy_1_small: - CMPQ CX, $0x03 + CMPQ R12, $0x03 JE copy_1_move_3 JB copy_1_move_1or2 - CMPQ CX, $0x08 + CMPQ R12, $0x08 JB copy_1_move_4through7 JMP copy_1_move_8through16 copy_1_move_1or2: MOVB (R10), R14 - MOVB -1(R10)(CX*1), R15 + MOVB -1(R10)(R12*1), R15 MOVB R14, (R9) - MOVB R15, -1(R9)(CX*1) - ADDQ CX, R10 - ADDQ CX, R9 + MOVB R15, -1(R9)(R12*1) + ADDQ R12, R10 + ADDQ R12, R9 JMP copy_1_end copy_1_move_3: - MOVW (R10), R14 - MOVB 2(R10), R15 - MOVW R14, (R9) - MOVB R15, 2(R9) - ADDQ CX, R10 - ADDQ CX, R9 - JMP copy_1_end + MOVWQZX (R10), R14 + MOVB 2(R10), R15 + MOVW R14, (R9) + MOVB R15, 2(R9) + ADDQ R12, R10 + ADDQ R12, R9 + JMP copy_1_end copy_1_move_4through7: MOVL (R10), R14 - MOVL -4(R10)(CX*1), R15 + MOVL -4(R10)(R12*1), R15 MOVL R14, (R9) - MOVL R15, -4(R9)(CX*1) - ADDQ CX, R10 - ADDQ CX, R9 + MOVL R15, -4(R9)(R12*1) + ADDQ R12, R10 + ADDQ R12, R9 JMP copy_1_end copy_1_move_8through16: MOVQ (R10), R14 - MOVQ -8(R10)(CX*1), R15 + MOVQ -8(R10)(R12*1), R15 MOVQ R14, (R9) - MOVQ R15, -8(R9)(CX*1) - ADDQ CX, R10 - ADDQ CX, R9 + MOVQ R15, -8(R9)(R12*1) + ADDQ R12, R10 + ADDQ R12, R9 copy_1_end: - ADDQ CX, R11 + ADDQ R12, R11 // Malformed input if seq.mo > t+len(hist) || seq.mo > s.windowSize) check_offset: - MOVQ R11, CX - ADDQ 40(SP), CX - CMPQ R12, CX + MOVQ R11, R12 + ADDQ 72(SP), R12 + CMPQ R13, R12 JG error_match_off_too_big - CMPQ R12, 56(SP) + CMPQ R13, 88(SP) JG error_match_off_too_big // Copy match from history - MOVQ R12, CX - SUBQ R11, CX + MOVQ R13, R12 + SUBQ R11, R12 JLS copy_match - MOVQ 48(SP), R14 - SUBQ CX, R14 - CMPQ R13, CX + MOVQ 80(SP), R14 + SUBQ R12, R14 + CMPQ CX, R12 JG copy_all_from_history - MOVQ R13, CX - SUBQ $0x10, CX + MOVQ CX, R12 + SUBQ $0x10, R12 JB copy_4_small copy_4_loop: @@ -3882,54 +3915,54 @@ copy_4_loop: MOVUPS X0, (R9) ADDQ $0x10, R14 ADDQ $0x10, R9 - SUBQ $0x10, CX + SUBQ $0x10, R12 JAE copy_4_loop - LEAQ 16(R14)(CX*1), R14 - LEAQ 16(R9)(CX*1), R9 + LEAQ 16(R14)(R12*1), R14 + LEAQ 16(R9)(R12*1), R9 MOVUPS -16(R14), X0 MOVUPS X0, -16(R9) JMP copy_4_end copy_4_small: - CMPQ R13, $0x03 + CMPQ CX, $0x03 JE copy_4_move_3 - CMPQ R13, $0x08 + CMPQ CX, $0x08 JB copy_4_move_4through7 JMP copy_4_move_8through16 copy_4_move_3: - MOVW (R14), CX - MOVB 2(R14), R12 - MOVW CX, (R9) - MOVB R12, 2(R9) - ADDQ R13, R14 - ADDQ R13, R9 - JMP copy_4_end + MOVWQZX (R14), R12 + MOVB 2(R14), R13 + MOVW R12, (R9) + MOVB R13, 2(R9) + ADDQ CX, R14 + ADDQ CX, R9 + JMP copy_4_end copy_4_move_4through7: - MOVL (R14), CX - MOVL -4(R14)(R13*1), R12 - MOVL CX, (R9) - MOVL R12, -4(R9)(R13*1) - ADDQ R13, R14 - ADDQ R13, R9 + MOVL (R14), R12 + MOVL -4(R14)(CX*1), R13 + MOVL R12, (R9) + MOVL R13, -4(R9)(CX*1) + ADDQ CX, R14 + ADDQ CX, R9 JMP copy_4_end copy_4_move_8through16: - MOVQ (R14), CX - MOVQ -8(R14)(R13*1), R12 - MOVQ CX, (R9) - MOVQ R12, -8(R9)(R13*1) - ADDQ R13, R14 - ADDQ R13, R9 + MOVQ (R14), R12 + MOVQ -8(R14)(CX*1), R13 + MOVQ R12, (R9) + MOVQ R13, -8(R9)(CX*1) + ADDQ CX, R14 + ADDQ CX, R9 copy_4_end: - ADDQ R13, R11 + ADDQ CX, R11 JMP handle_loop JMP loop_finished copy_all_from_history: - MOVQ CX, R15 + MOVQ R12, R15 SUBQ $0x10, R15 JB copy_5_small @@ -3947,141 +3980,140 @@ copy_5_loop: JMP copy_5_end copy_5_small: - CMPQ CX, $0x03 + CMPQ R12, $0x03 JE copy_5_move_3 JB copy_5_move_1or2 - CMPQ CX, $0x08 + CMPQ R12, $0x08 JB copy_5_move_4through7 JMP copy_5_move_8through16 copy_5_move_1or2: MOVB (R14), R15 - MOVB -1(R14)(CX*1), BP + MOVB -1(R14)(R12*1), BP MOVB R15, (R9) - MOVB BP, -1(R9)(CX*1) - ADDQ CX, R14 - ADDQ CX, R9 + MOVB BP, -1(R9)(R12*1) + ADDQ R12, R14 + ADDQ R12, R9 JMP copy_5_end copy_5_move_3: - MOVW (R14), R15 - MOVB 2(R14), BP - MOVW R15, (R9) - MOVB BP, 2(R9) - ADDQ CX, R14 - ADDQ CX, R9 - JMP copy_5_end + MOVWQZX (R14), R15 + MOVB 2(R14), BP + MOVW R15, (R9) + MOVB BP, 2(R9) + ADDQ R12, R14 + ADDQ R12, R9 + JMP copy_5_end copy_5_move_4through7: MOVL (R14), R15 - MOVL -4(R14)(CX*1), BP + MOVL -4(R14)(R12*1), BP MOVL R15, (R9) - MOVL BP, -4(R9)(CX*1) - ADDQ CX, R14 - ADDQ CX, R9 + MOVL BP, -4(R9)(R12*1) + ADDQ R12, R14 + ADDQ R12, R9 JMP copy_5_end copy_5_move_8through16: MOVQ (R14), R15 - MOVQ -8(R14)(CX*1), BP + MOVQ -8(R14)(R12*1), BP MOVQ R15, (R9) - MOVQ BP, -8(R9)(CX*1) - ADDQ CX, R14 - ADDQ CX, R9 + MOVQ BP, -8(R9)(R12*1) + ADDQ R12, R14 + ADDQ R12, R9 copy_5_end: - ADDQ CX, R11 - SUBQ CX, R13 + ADDQ R12, R11 + SUBQ R12, CX // Copy match from the current buffer copy_match: - MOVQ R9, CX - SUBQ R12, CX + MOVQ R9, R12 + SUBQ R13, R12 // ml <= mo - CMPQ R13, R12 + CMPQ CX, R13 JA copy_overlapping_match // Copy non-overlapping match - ADDQ R13, R11 - MOVQ R13, R12 - SUBQ $0x10, R12 + ADDQ CX, R11 + MOVQ CX, R13 + SUBQ $0x10, R13 JB copy_2_small copy_2_loop: - MOVUPS (CX), X0 + MOVUPS (R12), X0 MOVUPS X0, (R9) - ADDQ $0x10, CX + ADDQ $0x10, R12 ADDQ $0x10, R9 - SUBQ $0x10, R12 + SUBQ $0x10, R13 JAE copy_2_loop - LEAQ 16(CX)(R12*1), CX - LEAQ 16(R9)(R12*1), R9 - MOVUPS -16(CX), X0 + LEAQ 16(R12)(R13*1), R12 + LEAQ 16(R9)(R13*1), R9 + MOVUPS -16(R12), X0 MOVUPS X0, -16(R9) JMP copy_2_end copy_2_small: - CMPQ R13, $0x03 + CMPQ CX, $0x03 JE copy_2_move_3 JB copy_2_move_1or2 - CMPQ R13, $0x08 + CMPQ CX, $0x08 JB copy_2_move_4through7 JMP copy_2_move_8through16 copy_2_move_1or2: - MOVB (CX), R12 - MOVB -1(CX)(R13*1), R14 - MOVB R12, (R9) - MOVB R14, -1(R9)(R13*1) - ADDQ R13, CX - ADDQ R13, R9 + MOVB (R12), R13 + MOVB -1(R12)(CX*1), R14 + MOVB R13, (R9) + MOVB R14, -1(R9)(CX*1) + ADDQ CX, R12 + ADDQ CX, R9 JMP copy_2_end copy_2_move_3: - MOVW (CX), R12 - MOVB 2(CX), R14 - MOVW R12, (R9) - MOVB R14, 2(R9) - ADDQ R13, CX - ADDQ R13, R9 - JMP copy_2_end + MOVWQZX (R12), R13 + MOVB 2(R12), R14 + MOVW R13, (R9) + MOVB R14, 2(R9) + ADDQ CX, R12 + ADDQ CX, R9 + JMP copy_2_end copy_2_move_4through7: - MOVL (CX), R12 - MOVL -4(CX)(R13*1), R14 - MOVL R12, (R9) - MOVL R14, -4(R9)(R13*1) - ADDQ R13, CX - ADDQ R13, R9 + MOVL (R12), R13 + MOVL -4(R12)(CX*1), R14 + MOVL R13, (R9) + MOVL R14, -4(R9)(CX*1) + ADDQ CX, R12 + ADDQ CX, R9 JMP copy_2_end copy_2_move_8through16: - MOVQ (CX), R12 - MOVQ -8(CX)(R13*1), R14 - MOVQ R12, (R9) - MOVQ R14, -8(R9)(R13*1) - ADDQ R13, CX - ADDQ R13, R9 + MOVQ (R12), R13 + MOVQ -8(R12)(CX*1), R14 + MOVQ R13, (R9) + MOVQ R14, -8(R9)(CX*1) + ADDQ CX, R12 + ADDQ CX, R9 copy_2_end: JMP handle_loop // Copy overlapping match copy_overlapping_match: - ADDQ R13, R11 + ADDQ CX, R11 copy_slow_3: - MOVB (CX), R12 - MOVB R12, (R9) - INCQ CX - INCQ R9 - DECQ R13 - JNZ copy_slow_3 + MOVBQZX (R12), R13 + MOVB R13, (R9) + INCQ R12 + INCQ R9 + DECQ CX + JNZ copy_slow_3 handle_loop: - MOVQ ctx+16(FP), CX - DECQ 96(CX) + DECQ 32(SP) JNS sequenceDecs_decodeSync_safe_bmi2_main_loop loop_finished: @@ -4090,6 +4122,13 @@ loop_finished: MOVB DL, 40(CX) MOVQ BX, 32(CX) + // s.seqSize += outPosition - ctx.outPosition + MOVQ ctx+16(FP), AX + MOVQ s+0(FP), CX + MOVQ R11, DX + SUBQ 136(AX), DX + ADDQ DX, 256(CX) + // Update the context MOVQ ctx+16(FP), AX MOVQ R11, 136(AX) @@ -4103,7 +4142,7 @@ loop_finished: // Return with match length error sequenceDecs_decodeSync_safe_bmi2_error_match_len_ofs_mismatch: - MOVQ 16(SP), AX + MOVQ 48(SP), AX MOVQ ctx+16(FP), CX MOVQ AX, 216(CX) MOVQ $0x00000001, ret+24(FP) @@ -4112,7 +4151,7 @@ sequenceDecs_decodeSync_safe_bmi2_error_match_len_ofs_mismatch: // Return with match too long error sequenceDecs_decodeSync_safe_bmi2_error_match_len_too_big: MOVQ ctx+16(FP), AX - MOVQ 16(SP), CX + MOVQ 48(SP), CX MOVQ CX, 216(AX) MOVQ $0x00000002, ret+24(FP) RET @@ -4120,7 +4159,7 @@ sequenceDecs_decodeSync_safe_bmi2_error_match_len_too_big: // Return with match offset too long error error_match_off_too_big: MOVQ ctx+16(FP), AX - MOVQ 8(SP), CX + MOVQ 40(SP), CX MOVQ CX, 224(AX) MOVQ R11, 136(AX) MOVQ $0x00000003, ret+24(FP) @@ -4129,7 +4168,7 @@ error_match_off_too_big: // Return with not enough literals error error_not_enough_literals: MOVQ ctx+16(FP), AX - MOVQ 24(SP), CX + MOVQ 56(SP), CX MOVQ CX, 208(AX) MOVQ $0x00000004, ret+24(FP) RET @@ -4142,9 +4181,9 @@ error_overread: // Return with not enough output space error error_not_enough_space: MOVQ ctx+16(FP), AX - MOVQ 24(SP), CX + MOVQ 56(SP), CX MOVQ CX, 208(AX) - MOVQ 16(SP), CX + MOVQ 48(SP), CX MOVQ CX, 216(AX) MOVQ R11, 136(AX) MOVQ $0x00000005, ret+24(FP) diff --git a/vendor/github.com/klauspost/compress/zstd/seqdec_arm64.s b/vendor/github.com/klauspost/compress/zstd/seqdec_arm64.s index 6f54ef21ba12..56391bfc1839 100644 --- a/vendor/github.com/klauspost/compress/zstd/seqdec_arm64.s +++ b/vendor/github.com/klauspost/compress/zstd/seqdec_arm64.s @@ -1,11 +1,11 @@ // Code generated by command: go run gen.go -out ../seqdec.s -arch amd64,arm64 -pkg=zstd. DO NOT EDIT. // EXPERIMENTAL arm64 output lowered from an amd64 avo program. -//go:build arm64 && !appengine && !noasm && gc && !noasm +//go:build arm64 && (!appengine && !noasm && gc && !noasm) // func sequenceDecs_decode_amd64(s *sequenceDecs, br *bitReader, ctx *decodeAsmContext) int // Requires: CMOV -TEXT ·sequenceDecs_decode_arm64(SB), $8-32 +TEXT ·sequenceDecs_decode_arm64(SB), $40-32 MOVD br+8(FP), R1 MOVD 24(R1), R2 MOVBU 40(R1), R3 @@ -17,6 +17,14 @@ TEXT ·sequenceDecs_decode_arm64(SB), $8-32 MOVD 72(R0), R6 MOVD 80(R0), R7 MOVD 88(R0), R8 + MOVD (R0), R1 + MOVD R1, 16(RSP) + MOVD 24(R0), R1 + MOVD R1, 24(RSP) + MOVD 48(R0), R1 + MOVD R1, 32(RSP) + MOVD 96(R0), R1 + MOVD R1, 40(RSP) MOVD 104(R0), R9 MOVD s+0(FP), R0 MOVD 144(R0), R10 @@ -29,8 +37,7 @@ sequenceDecs_decode_amd64_main_loop: // Fill bitreader to have enough for the offset and match length. CMP $0x08, R5 BLT sequenceDecs_decode_amd64_fill_byte_by_byte - MOVD R3, R0 - LSR $0x03, R0, R0 + LSR $0x03, R3, R0 SUB R0, R13, R13 MOVD (R13), R2 SUB R0, R5, R5 @@ -57,53 +64,32 @@ sequenceDecs_decode_amd64_fill_check_overread: sequenceDecs_decode_amd64_fill_end: // Update offset MOVD R8, R0 - MOVD R3, R1 - MOVD R2, R14 - LSL R1, R14, R14 - UBFX $8, R0, $8, R16 - BFI $0, R16, $8, R1 + LSL R3, R2, R14 + LSR $0x01, R14, R14 + UBFX $8, R0, $8, R1 LSR $0x20, R0, R0 - TST R1, R1 - BEQ sequenceDecs_decode_amd64_of_update_zero ADD R1, R3, R3 - CMP $0x40, R3 - BHI sequenceDecs_decode_amd64_of_update_zero - CMP $0x40, R1 - BHS sequenceDecs_decode_amd64_of_update_zero - NEG R1, R1 + EOR $0x3f, R1, R1 LSR R1, R14, R14 ADD R14, R0, R0 - -sequenceDecs_decode_amd64_of_update_zero: MOVD R0, 16(R9) // Update match length MOVD R7, R0 - MOVD R3, R1 - MOVD R2, R14 - LSL R1, R14, R14 - UBFX $8, R0, $8, R16 - BFI $0, R16, $8, R1 + LSL R3, R2, R14 + LSR $0x01, R14, R14 + UBFX $8, R0, $8, R1 LSR $0x20, R0, R0 - TST R1, R1 - BEQ sequenceDecs_decode_amd64_ml_update_zero ADD R1, R3, R3 - CMP $0x40, R3 - BHI sequenceDecs_decode_amd64_ml_update_zero - CMP $0x40, R1 - BHS sequenceDecs_decode_amd64_ml_update_zero - NEG R1, R1 + EOR $0x3f, R1, R1 LSR R1, R14, R14 ADD R14, R0, R0 - -sequenceDecs_decode_amd64_ml_update_zero: MOVD R0, 8(R9) // Fill bitreader to have enough for the remaining CMP $0x08, R5 BLT sequenceDecs_decode_amd64_fill_2_byte_by_byte - MOVD R3, R0 - LSR $0x03, R0, R0 + LSR $0x03, R3, R0 SUB R0, R13, R13 MOVD (R13), R2 SUB R0, R5, R5 @@ -130,98 +116,64 @@ sequenceDecs_decode_amd64_fill_2_check_overread: sequenceDecs_decode_amd64_fill_2_end: // Update literal length MOVD R6, R0 - MOVD R3, R1 - MOVD R2, R14 - LSL R1, R14, R14 - UBFX $8, R0, $8, R16 - BFI $0, R16, $8, R1 + LSL R3, R2, R14 + LSR $0x01, R14, R14 + UBFX $8, R0, $8, R1 LSR $0x20, R0, R0 - TST R1, R1 - BEQ sequenceDecs_decode_amd64_ll_update_zero ADD R1, R3, R3 - CMP $0x40, R3 - BHI sequenceDecs_decode_amd64_ll_update_zero - CMP $0x40, R1 - BHS sequenceDecs_decode_amd64_ll_update_zero - NEG R1, R1 + EOR $0x3f, R1, R1 LSR R1, R14, R14 ADD R14, R0, R0 - -sequenceDecs_decode_amd64_ll_update_zero: MOVD R0, (R9) // Fill bitreader for state updates - MOVD R13, 8(RSP) - MOVD R8, R0 - LSR $0x08, R0, R0 - MOVBU R0, R0 - MOVD ctx+16(FP), R1 - MOVD 96(R1), R16 - CMP $0x00, R16 - BEQ sequenceDecs_decode_amd64_skip_update + MOVD R13, 8(RSP) + UBFX $8, R8, $8, R0 + MOVD 40(RSP), R16 + CMP $0x00, R16 + BEQ sequenceDecs_decode_amd64_skip_update // Update Literal Length State MOVBU R6, R13 LSRW $0x10, R6, R6 - ADD R13, R3, R1 - MOVD R2, R14 - MOVD R1, R3 - NEG R1, R16 - ROR R16, R14, R14 - MOVD $0x00000001, R4 - BFI $0, R13, $8, R1 - LSLW R1, R4, R4 - SUBW $1, R4, R4 - AND R4, R14, R14 + LSL R3, R2, R14 + LSR $0x01, R14, R14 + ADD R13, R3, R3 + EOR $0x3f, R13, R13 + LSR R13, R14, R14 ADD R14, R6, R6 // Load ctx.llTable - MOVD ctx+16(FP), R1 - MOVD (R1), R1 - ADD R6<<3, R1, R15 - MOVD (R15), R6 + MOVD 16(RSP), R1 + MOVD (R1)(R6<<3), R6 // Update Match Length State MOVBU R7, R13 LSRW $0x10, R7, R7 - ADD R13, R3, R1 - MOVD R2, R14 - MOVD R1, R3 - NEG R1, R16 - ROR R16, R14, R14 - MOVD $0x00000001, R4 - BFI $0, R13, $8, R1 - LSLW R1, R4, R4 - SUBW $1, R4, R4 - AND R4, R14, R14 + LSL R3, R2, R14 + LSR $0x01, R14, R14 + ADD R13, R3, R3 + EOR $0x3f, R13, R13 + LSR R13, R14, R14 ADD R14, R7, R7 // Load ctx.mlTable - MOVD ctx+16(FP), R1 - MOVD 24(R1), R1 - ADD R7<<3, R1, R15 - MOVD (R15), R7 + MOVD 24(RSP), R1 + MOVD (R1)(R7<<3), R7 // Update Offset State MOVBU R8, R13 LSRW $0x10, R8, R8 - ADD R13, R3, R1 - MOVD R2, R14 - MOVD R1, R3 - NEG R1, R16 - ROR R16, R14, R14 - MOVD $0x00000001, R4 - BFI $0, R13, $8, R1 - LSLW R1, R4, R4 - SUBW $1, R4, R4 - AND R4, R14, R14 + LSL R3, R2, R14 + LSR $0x01, R14, R14 + ADD R13, R3, R3 + EOR $0x3f, R13, R13 + LSR R13, R14, R14 ADD R14, R8, R8 // Load ctx.ofTable - MOVD ctx+16(FP), R1 - MOVD 48(R1), R1 - ADD R8<<3, R1, R15 - MOVD (R15), R8 + MOVD 32(RSP), R1 + MOVD (R1)(R8<<3), R8 sequenceDecs_decode_amd64_skip_update: // Adjust offset @@ -306,10 +258,9 @@ sequenceDecs_decode_amd64_after_adjust: sequenceDecs_decode_amd64_match_len_ofs_ok: ADD $0x18, R9, R9 - MOVD ctx+16(FP), R0 - MOVD 96(R0), R16 + MOVD 40(RSP), R16 SUBS $1, R16, R16 - MOVD R16, 96(R0) + MOVD R16, 40(RSP) BPL sequenceDecs_decode_amd64_main_loop MOVD s+0(FP), R0 MOVD R10, 144(R0) @@ -321,42 +272,60 @@ sequenceDecs_decode_amd64_match_len_ofs_ok: MOVD R5, 32(R0) // Return success + MOVD 40(RSP), R0 + MOVD ctx+16(FP), R1 + MOVD R0, 96(R1) MOVD $0x00000000, R16 MOVD R16, ret+24(FP) RET // Return with match length error sequenceDecs_decode_amd64_error_match_len_ofs_mismatch: + MOVD 40(RSP), R0 + MOVD ctx+16(FP), R1 + MOVD R0, 96(R1) MOVD $0x00000001, R16 MOVD R16, ret+24(FP) RET // Return with match too long error sequenceDecs_decode_amd64_error_match_len_too_big: + MOVD 40(RSP), R0 + MOVD ctx+16(FP), R1 + MOVD R0, 96(R1) MOVD $0x00000002, R16 MOVD R16, ret+24(FP) RET // Return with match offset too long error + MOVD 40(RSP), R0 + MOVD ctx+16(FP), R1 + MOVD R0, 96(R1) MOVD $0x00000003, R16 MOVD R16, ret+24(FP) RET // Return with not enough literals error error_not_enough_literals: + MOVD 40(RSP), R0 + MOVD ctx+16(FP), R1 + MOVD R0, 96(R1) MOVD $0x00000004, R16 MOVD R16, ret+24(FP) RET // Return with overread error error_overread: + MOVD 40(RSP), R0 + MOVD ctx+16(FP), R1 + MOVD R0, 96(R1) MOVD $0x00000006, R16 MOVD R16, ret+24(FP) RET // func sequenceDecs_decode_56_amd64(s *sequenceDecs, br *bitReader, ctx *decodeAsmContext) int // Requires: CMOV -TEXT ·sequenceDecs_decode_56_arm64(SB), $8-32 +TEXT ·sequenceDecs_decode_56_arm64(SB), $40-32 MOVD br+8(FP), R1 MOVD 24(R1), R2 MOVBU 40(R1), R3 @@ -368,6 +337,14 @@ TEXT ·sequenceDecs_decode_56_arm64(SB), $8-32 MOVD 72(R0), R6 MOVD 80(R0), R7 MOVD 88(R0), R8 + MOVD (R0), R1 + MOVD R1, 16(RSP) + MOVD 24(R0), R1 + MOVD R1, 24(RSP) + MOVD 48(R0), R1 + MOVD R1, 32(RSP) + MOVD 96(R0), R1 + MOVD R1, 40(RSP) MOVD 104(R0), R9 MOVD s+0(FP), R0 MOVD 144(R0), R10 @@ -380,8 +357,7 @@ sequenceDecs_decode_56_amd64_main_loop: // Fill bitreader to have enough for the offset and match length. CMP $0x08, R5 BLT sequenceDecs_decode_56_amd64_fill_byte_by_byte - MOVD R3, R0 - LSR $0x03, R0, R0 + LSR $0x03, R3, R0 SUB R0, R13, R13 MOVD (R13), R2 SUB R0, R5, R5 @@ -408,142 +384,88 @@ sequenceDecs_decode_56_amd64_fill_check_overread: sequenceDecs_decode_56_amd64_fill_end: // Update offset MOVD R8, R0 - MOVD R3, R1 - MOVD R2, R14 - LSL R1, R14, R14 - UBFX $8, R0, $8, R16 - BFI $0, R16, $8, R1 + LSL R3, R2, R14 + LSR $0x01, R14, R14 + UBFX $8, R0, $8, R1 LSR $0x20, R0, R0 - TST R1, R1 - BEQ sequenceDecs_decode_56_amd64_of_update_zero ADD R1, R3, R3 - CMP $0x40, R3 - BHI sequenceDecs_decode_56_amd64_of_update_zero - CMP $0x40, R1 - BHS sequenceDecs_decode_56_amd64_of_update_zero - NEG R1, R1 + EOR $0x3f, R1, R1 LSR R1, R14, R14 ADD R14, R0, R0 - -sequenceDecs_decode_56_amd64_of_update_zero: MOVD R0, 16(R9) // Update match length MOVD R7, R0 - MOVD R3, R1 - MOVD R2, R14 - LSL R1, R14, R14 - UBFX $8, R0, $8, R16 - BFI $0, R16, $8, R1 + LSL R3, R2, R14 + LSR $0x01, R14, R14 + UBFX $8, R0, $8, R1 LSR $0x20, R0, R0 - TST R1, R1 - BEQ sequenceDecs_decode_56_amd64_ml_update_zero ADD R1, R3, R3 - CMP $0x40, R3 - BHI sequenceDecs_decode_56_amd64_ml_update_zero - CMP $0x40, R1 - BHS sequenceDecs_decode_56_amd64_ml_update_zero - NEG R1, R1 + EOR $0x3f, R1, R1 LSR R1, R14, R14 ADD R14, R0, R0 - -sequenceDecs_decode_56_amd64_ml_update_zero: MOVD R0, 8(R9) // Update literal length MOVD R6, R0 - MOVD R3, R1 - MOVD R2, R14 - LSL R1, R14, R14 - UBFX $8, R0, $8, R16 - BFI $0, R16, $8, R1 + LSL R3, R2, R14 + LSR $0x01, R14, R14 + UBFX $8, R0, $8, R1 LSR $0x20, R0, R0 - TST R1, R1 - BEQ sequenceDecs_decode_56_amd64_ll_update_zero ADD R1, R3, R3 - CMP $0x40, R3 - BHI sequenceDecs_decode_56_amd64_ll_update_zero - CMP $0x40, R1 - BHS sequenceDecs_decode_56_amd64_ll_update_zero - NEG R1, R1 + EOR $0x3f, R1, R1 LSR R1, R14, R14 ADD R14, R0, R0 - -sequenceDecs_decode_56_amd64_ll_update_zero: MOVD R0, (R9) // Fill bitreader for state updates - MOVD R13, 8(RSP) - MOVD R8, R0 - LSR $0x08, R0, R0 - MOVBU R0, R0 - MOVD ctx+16(FP), R1 - MOVD 96(R1), R16 - CMP $0x00, R16 - BEQ sequenceDecs_decode_56_amd64_skip_update + MOVD R13, 8(RSP) + UBFX $8, R8, $8, R0 + MOVD 40(RSP), R16 + CMP $0x00, R16 + BEQ sequenceDecs_decode_56_amd64_skip_update // Update Literal Length State MOVBU R6, R13 LSRW $0x10, R6, R6 - ADD R13, R3, R1 - MOVD R2, R14 - MOVD R1, R3 - NEG R1, R16 - ROR R16, R14, R14 - MOVD $0x00000001, R4 - BFI $0, R13, $8, R1 - LSLW R1, R4, R4 - SUBW $1, R4, R4 - AND R4, R14, R14 + LSL R3, R2, R14 + LSR $0x01, R14, R14 + ADD R13, R3, R3 + EOR $0x3f, R13, R13 + LSR R13, R14, R14 ADD R14, R6, R6 // Load ctx.llTable - MOVD ctx+16(FP), R1 - MOVD (R1), R1 - ADD R6<<3, R1, R15 - MOVD (R15), R6 + MOVD 16(RSP), R1 + MOVD (R1)(R6<<3), R6 // Update Match Length State MOVBU R7, R13 LSRW $0x10, R7, R7 - ADD R13, R3, R1 - MOVD R2, R14 - MOVD R1, R3 - NEG R1, R16 - ROR R16, R14, R14 - MOVD $0x00000001, R4 - BFI $0, R13, $8, R1 - LSLW R1, R4, R4 - SUBW $1, R4, R4 - AND R4, R14, R14 + LSL R3, R2, R14 + LSR $0x01, R14, R14 + ADD R13, R3, R3 + EOR $0x3f, R13, R13 + LSR R13, R14, R14 ADD R14, R7, R7 // Load ctx.mlTable - MOVD ctx+16(FP), R1 - MOVD 24(R1), R1 - ADD R7<<3, R1, R15 - MOVD (R15), R7 + MOVD 24(RSP), R1 + MOVD (R1)(R7<<3), R7 // Update Offset State MOVBU R8, R13 LSRW $0x10, R8, R8 - ADD R13, R3, R1 - MOVD R2, R14 - MOVD R1, R3 - NEG R1, R16 - ROR R16, R14, R14 - MOVD $0x00000001, R4 - BFI $0, R13, $8, R1 - LSLW R1, R4, R4 - SUBW $1, R4, R4 - AND R4, R14, R14 + LSL R3, R2, R14 + LSR $0x01, R14, R14 + ADD R13, R3, R3 + EOR $0x3f, R13, R13 + LSR R13, R14, R14 ADD R14, R8, R8 // Load ctx.ofTable - MOVD ctx+16(FP), R1 - MOVD 48(R1), R1 - ADD R8<<3, R1, R15 - MOVD (R15), R8 + MOVD 32(RSP), R1 + MOVD (R1)(R8<<3), R8 sequenceDecs_decode_56_amd64_skip_update: // Adjust offset @@ -628,10 +550,9 @@ sequenceDecs_decode_56_amd64_after_adjust: sequenceDecs_decode_56_amd64_match_len_ofs_ok: ADD $0x18, R9, R9 - MOVD ctx+16(FP), R0 - MOVD 96(R0), R16 + MOVD 40(RSP), R16 SUBS $1, R16, R16 - MOVD R16, 96(R0) + MOVD R16, 40(RSP) BPL sequenceDecs_decode_56_amd64_main_loop MOVD s+0(FP), R0 MOVD R10, 144(R0) @@ -643,35 +564,53 @@ sequenceDecs_decode_56_amd64_match_len_ofs_ok: MOVD R5, 32(R0) // Return success + MOVD 40(RSP), R0 + MOVD ctx+16(FP), R1 + MOVD R0, 96(R1) MOVD $0x00000000, R16 MOVD R16, ret+24(FP) RET // Return with match length error sequenceDecs_decode_56_amd64_error_match_len_ofs_mismatch: + MOVD 40(RSP), R0 + MOVD ctx+16(FP), R1 + MOVD R0, 96(R1) MOVD $0x00000001, R16 MOVD R16, ret+24(FP) RET // Return with match too long error sequenceDecs_decode_56_amd64_error_match_len_too_big: + MOVD 40(RSP), R0 + MOVD ctx+16(FP), R1 + MOVD R0, 96(R1) MOVD $0x00000002, R16 MOVD R16, ret+24(FP) RET // Return with match offset too long error + MOVD 40(RSP), R0 + MOVD ctx+16(FP), R1 + MOVD R0, 96(R1) MOVD $0x00000003, R16 MOVD R16, ret+24(FP) RET // Return with not enough literals error error_not_enough_literals: + MOVD 40(RSP), R0 + MOVD ctx+16(FP), R1 + MOVD R0, 96(R1) MOVD $0x00000004, R16 MOVD R16, ret+24(FP) RET // Return with overread error error_overread: + MOVD 40(RSP), R0 + MOVD ctx+16(FP), R1 + MOVD R0, 96(R1) MOVD $0x00000006, R16 MOVD R16, ret+24(FP) RET @@ -711,24 +650,27 @@ main_loop: MOVD 8(R0), R12 // Copy literals - TST R10, R10 - BEQ check_offset - MOVD $0, R13 + FMOVQ (R5), F0 + FMOVQ F0, (R3) + CMP $0x10, R10 + BLS copy_1_end + MOVD $0x00000010, R13 copy_1: - ADD R13, R5, R15 - VLD1 (R15), [V0.B16] - ADD R13, R3, R15 - VST1 [V0.B16], (R15) - ADD $0x10, R13, R13 - CMP R10, R13 - BLO copy_1 - ADD R10, R5, R5 - ADD R10, R3, R3 - ADD R10, R6, R6 + ADD R13, R5, R15 + FMOVQ (R15), F0 + ADD R13, R3, R15 + FMOVQ F0, (R15) + ADD $0x10, R13, R13 + CMP R10, R13 + BLO copy_1 + +copy_1_end: + ADD R10, R5, R5 + ADD R10, R3, R3 + ADD R10, R6, R6 // Malformed input if seq.mo > t+len(hist) || seq.mo > s.windowSize) -check_offset: ADD R9, R6, R10 CMP R10, R11 BGT error_match_off_too_big @@ -748,21 +690,19 @@ check_offset: BLO copy_4_small copy_4_loop: - VLD1 (R13), [V0.B16] - VST1 [V0.B16], (R3) - ADD $0x10, R13, R13 - ADD $0x10, R3, R3 - SUBS $0x10, R10, R10 - BHS copy_4_loop - ADD R10, R13, R13 - ADD $16, R13, R13 - ADD R10, R3, R3 - ADD $16, R3, R3 - ADD $-16, R13, R15 - VLD1 (R15), [V0.B16] - ADD $-16, R3, R15 - VST1 [V0.B16], (R15) - JMP copy_4_end + FMOVQ (R13), F0 + FMOVQ F0, (R3) + ADD $0x10, R13, R13 + ADD $0x10, R3, R3 + SUBS $0x10, R10, R10 + BHS copy_4_loop + ADD R10, R13, R13 + ADD $16, R13, R13 + ADD R10, R3, R3 + ADD $16, R3, R3 + FMOVQ -16(R13), F0 + FMOVQ F0, -16(R3) + JMP copy_4_end copy_4_small: CMP $0x03, R12 @@ -816,21 +756,19 @@ copy_all_from_history: BLO copy_5_small copy_5_loop: - VLD1 (R13), [V0.B16] - VST1 [V0.B16], (R3) - ADD $0x10, R13, R13 - ADD $0x10, R3, R3 - SUBS $0x10, R14, R14 - BHS copy_5_loop - ADD R14, R13, R13 - ADD $16, R13, R13 - ADD R14, R3, R3 - ADD $16, R3, R3 - ADD $-16, R13, R15 - VLD1 (R15), [V0.B16] - ADD $-16, R3, R15 - VST1 [V0.B16], (R15) - JMP copy_5_end + FMOVQ (R13), F0 + FMOVQ F0, (R3) + ADD $0x10, R13, R13 + ADD $0x10, R3, R3 + SUBS $0x10, R14, R14 + BHS copy_5_loop + ADD R14, R13, R13 + ADD $16, R13, R13 + ADD R14, R3, R3 + ADD $16, R3, R3 + FMOVQ -16(R13), F0 + FMOVQ F0, -16(R3) + JMP copy_5_end copy_5_small: CMP $0x03, R10 @@ -898,26 +836,35 @@ copy_match: BHI copy_overlapping_match // Copy non-overlapping match - ADD R12, R6, R6 - MOVD R3, R11 - ADD R12, R3, R3 + ADD R12, R6, R6 + FMOVQ (R10), F0 + FMOVQ F0, (R3) + CMP $0x10, R12 + BHI copy_2_long + ADD R12, R3, R3 + JMP handle_loop + +copy_2_long: + ADD $16, R3, R11 + ADD R12, R3, R3 + ADD $0x10, R10, R10 + SUB $0x10, R12, R12 copy_2: - VLD1 (R10), [V0.B16] - VST1 [V0.B16], (R11) - ADD $0x10, R10, R10 - ADD $0x10, R11, R11 - SUBS $0x10, R12, R12 - BHI copy_2 - JMP handle_loop + FMOVQ (R10), F0 + FMOVQ F0, (R11) + ADD $0x10, R10, R10 + ADD $0x10, R11, R11 + SUBS $0x10, R12, R12 + BHI copy_2 + JMP handle_loop // Copy overlapping match copy_overlapping_match: ADD R12, R6, R6 copy_slow_3: - MOVBU (R10), R16 - BFI $0, R16, $8, R11 + MOVBU (R10), R11 MOVB R11, (R3) ADD $1, R10, R10 ADD $1, R3, R3 @@ -1002,21 +949,19 @@ main_loop: BLO copy_1_small copy_1_loop: - VLD1 (R5), [V0.B16] - VST1 [V0.B16], (R3) - ADD $0x10, R5, R5 - ADD $0x10, R3, R3 - SUBS $0x10, R13, R13 - BHS copy_1_loop - ADD R13, R5, R5 - ADD $16, R5, R5 - ADD R13, R3, R3 - ADD $16, R3, R3 - ADD $-16, R5, R15 - VLD1 (R15), [V0.B16] - ADD $-16, R3, R15 - VST1 [V0.B16], (R15) - JMP copy_1_end + FMOVQ (R5), F0 + FMOVQ F0, (R3) + ADD $0x10, R5, R5 + ADD $0x10, R3, R3 + SUBS $0x10, R13, R13 + BHS copy_1_loop + ADD R13, R5, R5 + ADD $16, R5, R5 + ADD R13, R3, R3 + ADD $16, R3, R3 + FMOVQ -16(R5), F0 + FMOVQ F0, -16(R3) + JMP copy_1_end copy_1_small: CMP $0x03, R10 @@ -1094,21 +1039,19 @@ check_offset: BLO copy_4_small copy_4_loop: - VLD1 (R13), [V0.B16] - VST1 [V0.B16], (R3) - ADD $0x10, R13, R13 - ADD $0x10, R3, R3 - SUBS $0x10, R10, R10 - BHS copy_4_loop - ADD R10, R13, R13 - ADD $16, R13, R13 - ADD R10, R3, R3 - ADD $16, R3, R3 - ADD $-16, R13, R15 - VLD1 (R15), [V0.B16] - ADD $-16, R3, R15 - VST1 [V0.B16], (R15) - JMP copy_4_end + FMOVQ (R13), F0 + FMOVQ F0, (R3) + ADD $0x10, R13, R13 + ADD $0x10, R3, R3 + SUBS $0x10, R10, R10 + BHS copy_4_loop + ADD R10, R13, R13 + ADD $16, R13, R13 + ADD R10, R3, R3 + ADD $16, R3, R3 + FMOVQ -16(R13), F0 + FMOVQ F0, -16(R3) + JMP copy_4_end copy_4_small: CMP $0x03, R12 @@ -1162,21 +1105,19 @@ copy_all_from_history: BLO copy_5_small copy_5_loop: - VLD1 (R13), [V0.B16] - VST1 [V0.B16], (R3) - ADD $0x10, R13, R13 - ADD $0x10, R3, R3 - SUBS $0x10, R14, R14 - BHS copy_5_loop - ADD R14, R13, R13 - ADD $16, R13, R13 - ADD R14, R3, R3 - ADD $16, R3, R3 - ADD $-16, R13, R15 - VLD1 (R15), [V0.B16] - ADD $-16, R3, R15 - VST1 [V0.B16], (R15) - JMP copy_5_end + FMOVQ (R13), F0 + FMOVQ F0, (R3) + ADD $0x10, R13, R13 + ADD $0x10, R3, R3 + SUBS $0x10, R14, R14 + BHS copy_5_loop + ADD R14, R13, R13 + ADD $16, R13, R13 + ADD R14, R3, R3 + ADD $16, R3, R3 + FMOVQ -16(R13), F0 + FMOVQ F0, -16(R3) + JMP copy_5_end copy_5_small: CMP $0x03, R10 @@ -1250,21 +1191,19 @@ copy_match: BLO copy_2_small copy_2_loop: - VLD1 (R10), [V0.B16] - VST1 [V0.B16], (R3) - ADD $0x10, R10, R10 - ADD $0x10, R3, R3 - SUBS $0x10, R11, R11 - BHS copy_2_loop - ADD R11, R10, R10 - ADD $16, R10, R10 - ADD R11, R3, R3 - ADD $16, R3, R3 - ADD $-16, R10, R15 - VLD1 (R15), [V0.B16] - ADD $-16, R3, R15 - VST1 [V0.B16], (R15) - JMP copy_2_end + FMOVQ (R10), F0 + FMOVQ F0, (R3) + ADD $0x10, R10, R10 + ADD $0x10, R3, R3 + SUBS $0x10, R11, R11 + BHS copy_2_loop + ADD R11, R10, R10 + ADD $16, R10, R10 + ADD R11, R3, R3 + ADD $16, R3, R3 + FMOVQ -16(R10), F0 + FMOVQ F0, -16(R3) + JMP copy_2_end copy_2_small: CMP $0x03, R12 @@ -1326,8 +1265,7 @@ copy_overlapping_match: ADD R12, R6, R6 copy_slow_3: - MOVBU (R10), R16 - BFI $0, R16, $8, R11 + MOVBU (R10), R11 MOVB R11, (R3) ADD $1, R10, R10 ADD $1, R3, R3 @@ -1376,7 +1314,7 @@ empty_seqs: // func sequenceDecs_decodeSync_amd64(s *sequenceDecs, br *bitReader, ctx *decodeSyncAsmContext) int // Requires: CMOV, SSE -TEXT ·sequenceDecs_decodeSync_arm64(SB), $64-32 +TEXT ·sequenceDecs_decodeSync_arm64(SB), $96-32 MOVD br+8(FP), R1 MOVD 24(R1), R2 MOVBU 40(R1), R3 @@ -1388,30 +1326,38 @@ TEXT ·sequenceDecs_decodeSync_arm64(SB), $64-32 MOVD 72(R0), R6 MOVD 80(R0), R7 MOVD 88(R0), R8 - MOVD $0, R1 + MOVD (R0), R1 MOVD R1, 16(RSP) + MOVD 24(R0), R1 MOVD R1, 24(RSP) + MOVD 48(R0), R1 MOVD R1, 32(RSP) + MOVD 96(R0), R1 + MOVD R1, 40(RSP) + MOVD $0, R1 + MOVD R1, 48(RSP) + MOVD R1, 56(RSP) + MOVD R1, 64(RSP) MOVD 112(R0), R9 MOVD 128(R0), R1 - MOVD R1, 40(RSP) + MOVD R1, 72(RSP) MOVD 144(R0), R10 MOVD 136(R0), R11 MOVD 200(R0), R1 - MOVD R1, 64(RSP) + MOVD R1, 96(RSP) MOVD 176(R0), R1 - MOVD R1, 56(RSP) + MOVD R1, 88(RSP) MOVD 184(R0), R0 - MOVD R0, 48(RSP) - MOVD 48(RSP), R0 - MOVD 56(RSP), R16 + MOVD R0, 80(RSP) + MOVD 80(RSP), R0 + MOVD 88(RSP), R16 ADD R0, R16, R16 - MOVD R16, 56(RSP) + MOVD R16, 88(RSP) // Calculate pointer to s.out[cap(s.out)] (a past-end pointer) - MOVD 40(RSP), R16 + MOVD 72(RSP), R16 ADD R9, R16, R16 - MOVD R16, 40(RSP) + MOVD R16, 72(RSP) // outBase += outPosition ADD R11, R9, R9 @@ -1422,8 +1368,7 @@ sequenceDecs_decodeSync_amd64_main_loop: // Fill bitreader to have enough for the offset and match length. CMP $0x08, R5 BLT sequenceDecs_decodeSync_amd64_fill_byte_by_byte - MOVD R3, R0 - LSR $0x03, R0, R0 + LSR $0x03, R3, R0 SUB R0, R12, R12 MOVD (R12), R2 SUB R0, R5, R5 @@ -1450,53 +1395,32 @@ sequenceDecs_decodeSync_amd64_fill_check_overread: sequenceDecs_decodeSync_amd64_fill_end: // Update offset MOVD R8, R0 - MOVD R3, R1 - MOVD R2, R13 - LSL R1, R13, R13 - UBFX $8, R0, $8, R16 - BFI $0, R16, $8, R1 + LSL R3, R2, R13 + LSR $0x01, R13, R13 + UBFX $8, R0, $8, R1 LSR $0x20, R0, R0 - TST R1, R1 - BEQ sequenceDecs_decodeSync_amd64_of_update_zero ADD R1, R3, R3 - CMP $0x40, R3 - BHI sequenceDecs_decodeSync_amd64_of_update_zero - CMP $0x40, R1 - BHS sequenceDecs_decodeSync_amd64_of_update_zero - NEG R1, R1 + EOR $0x3f, R1, R1 LSR R1, R13, R13 ADD R13, R0, R0 - -sequenceDecs_decodeSync_amd64_of_update_zero: - MOVD R0, 16(RSP) + MOVD R0, 48(RSP) // Update match length MOVD R7, R0 - MOVD R3, R1 - MOVD R2, R13 - LSL R1, R13, R13 - UBFX $8, R0, $8, R16 - BFI $0, R16, $8, R1 + LSL R3, R2, R13 + LSR $0x01, R13, R13 + UBFX $8, R0, $8, R1 LSR $0x20, R0, R0 - TST R1, R1 - BEQ sequenceDecs_decodeSync_amd64_ml_update_zero ADD R1, R3, R3 - CMP $0x40, R3 - BHI sequenceDecs_decodeSync_amd64_ml_update_zero - CMP $0x40, R1 - BHS sequenceDecs_decodeSync_amd64_ml_update_zero - NEG R1, R1 + EOR $0x3f, R1, R1 LSR R1, R13, R13 ADD R13, R0, R0 - -sequenceDecs_decodeSync_amd64_ml_update_zero: - MOVD R0, 24(RSP) + MOVD R0, 56(RSP) // Fill bitreader to have enough for the remaining CMP $0x08, R5 BLT sequenceDecs_decodeSync_amd64_fill_2_byte_by_byte - MOVD R3, R0 - LSR $0x03, R0, R0 + LSR $0x03, R3, R0 SUB R0, R12, R12 MOVD (R12), R2 SUB R0, R5, R5 @@ -1523,114 +1447,78 @@ sequenceDecs_decodeSync_amd64_fill_2_check_overread: sequenceDecs_decodeSync_amd64_fill_2_end: // Update literal length MOVD R6, R0 - MOVD R3, R1 - MOVD R2, R13 - LSL R1, R13, R13 - UBFX $8, R0, $8, R16 - BFI $0, R16, $8, R1 + LSL R3, R2, R13 + LSR $0x01, R13, R13 + UBFX $8, R0, $8, R1 LSR $0x20, R0, R0 - TST R1, R1 - BEQ sequenceDecs_decodeSync_amd64_ll_update_zero ADD R1, R3, R3 - CMP $0x40, R3 - BHI sequenceDecs_decodeSync_amd64_ll_update_zero - CMP $0x40, R1 - BHS sequenceDecs_decodeSync_amd64_ll_update_zero - NEG R1, R1 + EOR $0x3f, R1, R1 LSR R1, R13, R13 ADD R13, R0, R0 - -sequenceDecs_decodeSync_amd64_ll_update_zero: - MOVD R0, 32(RSP) + MOVD R0, 64(RSP) // Fill bitreader for state updates - MOVD R12, 8(RSP) - MOVD R8, R0 - LSR $0x08, R0, R0 - MOVBU R0, R0 - MOVD ctx+16(FP), R1 - MOVD 96(R1), R16 - CMP $0x00, R16 - BEQ sequenceDecs_decodeSync_amd64_skip_update + MOVD R12, 8(RSP) + UBFX $8, R8, $8, R0 + MOVD 40(RSP), R16 + CMP $0x00, R16 + BEQ sequenceDecs_decodeSync_amd64_skip_update // Update Literal Length State MOVBU R6, R12 LSRW $0x10, R6, R6 - ADD R12, R3, R1 - MOVD R2, R13 - MOVD R1, R3 - NEG R1, R16 - ROR R16, R13, R13 - MOVD $0x00000001, R14 - BFI $0, R12, $8, R1 - LSLW R1, R14, R14 - SUBW $1, R14, R14 - AND R14, R13, R13 + LSL R3, R2, R13 + LSR $0x01, R13, R13 + ADD R12, R3, R3 + EOR $0x3f, R12, R12 + LSR R12, R13, R13 ADD R13, R6, R6 // Load ctx.llTable - MOVD ctx+16(FP), R1 - MOVD (R1), R1 - ADD R6<<3, R1, R15 - MOVD (R15), R6 + MOVD 16(RSP), R1 + MOVD (R1)(R6<<3), R6 // Update Match Length State MOVBU R7, R12 LSRW $0x10, R7, R7 - ADD R12, R3, R1 - MOVD R2, R13 - MOVD R1, R3 - NEG R1, R16 - ROR R16, R13, R13 - MOVD $0x00000001, R14 - BFI $0, R12, $8, R1 - LSLW R1, R14, R14 - SUBW $1, R14, R14 - AND R14, R13, R13 + LSL R3, R2, R13 + LSR $0x01, R13, R13 + ADD R12, R3, R3 + EOR $0x3f, R12, R12 + LSR R12, R13, R13 ADD R13, R7, R7 // Load ctx.mlTable - MOVD ctx+16(FP), R1 - MOVD 24(R1), R1 - ADD R7<<3, R1, R15 - MOVD (R15), R7 + MOVD 24(RSP), R1 + MOVD (R1)(R7<<3), R7 // Update Offset State MOVBU R8, R12 LSRW $0x10, R8, R8 - ADD R12, R3, R1 - MOVD R2, R13 - MOVD R1, R3 - NEG R1, R16 - ROR R16, R13, R13 - MOVD $0x00000001, R14 - BFI $0, R12, $8, R1 - LSLW R1, R14, R14 - SUBW $1, R14, R14 - AND R14, R13, R13 + LSL R3, R2, R13 + LSR $0x01, R13, R13 + ADD R12, R3, R3 + EOR $0x3f, R12, R12 + LSR R12, R13, R13 ADD R13, R8, R8 // Load ctx.ofTable - MOVD ctx+16(FP), R1 - MOVD 48(R1), R1 - ADD R8<<3, R1, R15 - MOVD (R15), R8 + MOVD 32(RSP), R1 + MOVD (R1)(R8<<3), R8 sequenceDecs_decodeSync_amd64_skip_update: // Adjust offset - MOVD s+0(FP), R1 - MOVD 16(RSP), R12 - CMP $0x01, R0 - BLS sequenceDecs_decodeSync_amd64_adjust_offsetB_1_or_0 - ADD $144, R1, R15 - VLD1 (R15), [V0.B16] - MOVD R12, 144(R1) - ADD $152, R1, R15 - VST1 [V0.B16], (R15) - JMP sequenceDecs_decodeSync_amd64_after_adjust + MOVD s+0(FP), R1 + MOVD 48(RSP), R12 + CMP $0x01, R0 + BLS sequenceDecs_decodeSync_amd64_adjust_offsetB_1_or_0 + FMOVQ 144(R1), F0 + MOVD R12, 144(R1) + FMOVQ F0, 152(R1) + JMP sequenceDecs_decodeSync_amd64_after_adjust sequenceDecs_decodeSync_amd64_adjust_offsetB_1_or_0: - MOVD 32(RSP), R16 + MOVD 64(RSP), R16 CMP $0x00000000, R16 BNE sequenceDecs_decodeSync_amd64_adjust_offset_maybezero ADD $1, R12, R12 @@ -1668,16 +1556,11 @@ sequenceDecs_decodeSync_amd64_adjust_skip: MOVD R13, R12 sequenceDecs_decodeSync_amd64_after_adjust: - MOVD R12, 16(RSP) + MOVD R12, 48(RSP) // Check values - MOVD 24(RSP), R0 - MOVD 32(RSP), R1 - ADD R1, R0, R13 - MOVD s+0(FP), R14 - MOVD 256(R14), R16 - ADD R13, R16, R16 - MOVD R16, 256(R14) + MOVD 56(RSP), R0 + MOVD 64(RSP), R1 MOVD ctx+16(FP), R13 MOVD 104(R13), R16 SUBS R1, R16, R16 @@ -1691,159 +1574,154 @@ sequenceDecs_decodeSync_amd64_after_adjust: BNE sequenceDecs_decodeSync_amd64_error_match_len_ofs_mismatch sequenceDecs_decodeSync_amd64_match_len_ofs_ok: - MOVD 32(RSP), R0 - MOVD 16(RSP), R1 - MOVD 24(RSP), R12 - // Check if we have enough space in s.out - ADD R12, R0, R13 + ADD R0, R1, R13 ADD $16, R13, R13 ADD R9, R13, R13 - MOVD 40(RSP), R16 + MOVD 72(RSP), R16 CMP R16, R13 BHI error_not_enough_space // Copy literals - TST R0, R0 - BEQ check_offset - MOVD $0, R13 + FMOVQ (R10), F0 + FMOVQ F0, (R9) + CMP $0x10, R1 + BLS copy_1_end + MOVD $0x00000010, R13 copy_1: - ADD R13, R10, R15 - VLD1 (R15), [V0.B16] - ADD R13, R9, R15 - VST1 [V0.B16], (R15) - ADD $0x10, R13, R13 - CMP R0, R13 - BLO copy_1 - ADD R0, R10, R10 - ADD R0, R9, R9 - ADD R0, R11, R11 + ADD R13, R10, R15 + FMOVQ (R15), F0 + ADD R13, R9, R15 + FMOVQ F0, (R15) + ADD $0x10, R13, R13 + CMP R1, R13 + BLO copy_1 + +copy_1_end: + ADD R1, R10, R10 + ADD R1, R9, R9 + ADD R1, R11, R11 // Malformed input if seq.mo > t+len(hist) || seq.mo > s.windowSize) -check_offset: - MOVD R11, R0 - MOVD 48(RSP), R16 - ADD R16, R0, R0 - CMP R0, R1 + MOVD R11, R1 + MOVD 80(RSP), R16 + ADD R16, R1, R1 + CMP R1, R12 BGT error_match_off_too_big - MOVD 64(RSP), R16 - CMP R16, R1 + MOVD 96(RSP), R16 + CMP R16, R12 BGT error_match_off_too_big // Copy match from history - MOVD R1, R0 - SUBS R11, R0, R0 + MOVD R12, R1 + SUBS R11, R1, R1 BLS copy_match - MOVD 56(RSP), R13 - SUB R0, R13, R13 - CMP R0, R12 + MOVD 88(RSP), R13 + SUB R1, R13, R13 + CMP R1, R0 BGT copy_all_from_history - MOVD R12, R0 - SUBS $0x10, R0, R0 + MOVD R0, R1 + SUBS $0x10, R1, R1 BLO copy_4_small copy_4_loop: - VLD1 (R13), [V0.B16] - VST1 [V0.B16], (R9) - ADD $0x10, R13, R13 - ADD $0x10, R9, R9 - SUBS $0x10, R0, R0 - BHS copy_4_loop - ADD R0, R13, R13 - ADD $16, R13, R13 - ADD R0, R9, R9 - ADD $16, R9, R9 - ADD $-16, R13, R15 - VLD1 (R15), [V0.B16] - ADD $-16, R9, R15 - VST1 [V0.B16], (R15) - JMP copy_4_end + FMOVQ (R13), F0 + FMOVQ F0, (R9) + ADD $0x10, R13, R13 + ADD $0x10, R9, R9 + SUBS $0x10, R1, R1 + BHS copy_4_loop + ADD R1, R13, R13 + ADD $16, R13, R13 + ADD R1, R9, R9 + ADD $16, R9, R9 + FMOVQ -16(R13), F0 + FMOVQ F0, -16(R9) + JMP copy_4_end copy_4_small: - CMP $0x03, R12 + CMP $0x03, R0 BEQ copy_4_move_3 - CMP $0x08, R12 + CMP $0x08, R0 BLO copy_4_move_4through7 JMP copy_4_move_8through16 copy_4_move_3: - MOVHU (R13), R0 + MOVHU (R13), R1 MOVBU 2(R13), R16 - BFI $0, R16, $8, R1 - MOVH R0, (R9) - MOVB R1, 2(R9) - ADD R12, R13, R13 - ADD R12, R9, R9 + BFI $0, R16, $8, R12 + MOVH R1, (R9) + MOVB R12, 2(R9) + ADD R0, R13, R13 + ADD R0, R9, R9 JMP copy_4_end copy_4_move_4through7: - MOVWU (R13), R0 - ADD R12, R13, R15 - MOVWU -4(R15), R1 - MOVW R0, (R9) - ADD R12, R9, R15 - MOVW R1, -4(R15) - ADD R12, R13, R13 - ADD R12, R9, R9 + MOVWU (R13), R1 + ADD R0, R13, R15 + MOVWU -4(R15), R12 + MOVW R1, (R9) + ADD R0, R9, R15 + MOVW R12, -4(R15) + ADD R0, R13, R13 + ADD R0, R9, R9 JMP copy_4_end copy_4_move_8through16: - MOVD (R13), R0 - ADD R12, R13, R15 - MOVD -8(R15), R1 - MOVD R0, (R9) - ADD R12, R9, R15 - MOVD R1, -8(R15) - ADD R12, R13, R13 - ADD R12, R9, R9 + MOVD (R13), R1 + ADD R0, R13, R15 + MOVD -8(R15), R12 + MOVD R1, (R9) + ADD R0, R9, R15 + MOVD R12, -8(R15) + ADD R0, R13, R13 + ADD R0, R9, R9 copy_4_end: - ADD R12, R11, R11 + ADD R0, R11, R11 JMP handle_loop JMP loop_finished copy_all_from_history: - MOVD R0, R14 + MOVD R1, R14 SUBS $0x10, R14, R14 BLO copy_5_small copy_5_loop: - VLD1 (R13), [V0.B16] - VST1 [V0.B16], (R9) - ADD $0x10, R13, R13 - ADD $0x10, R9, R9 - SUBS $0x10, R14, R14 - BHS copy_5_loop - ADD R14, R13, R13 - ADD $16, R13, R13 - ADD R14, R9, R9 - ADD $16, R9, R9 - ADD $-16, R13, R15 - VLD1 (R15), [V0.B16] - ADD $-16, R9, R15 - VST1 [V0.B16], (R15) - JMP copy_5_end + FMOVQ (R13), F0 + FMOVQ F0, (R9) + ADD $0x10, R13, R13 + ADD $0x10, R9, R9 + SUBS $0x10, R14, R14 + BHS copy_5_loop + ADD R14, R13, R13 + ADD $16, R13, R13 + ADD R14, R9, R9 + ADD $16, R9, R9 + FMOVQ -16(R13), F0 + FMOVQ F0, -16(R9) + JMP copy_5_end copy_5_small: - CMP $0x03, R0 + CMP $0x03, R1 BEQ copy_5_move_3 BLO copy_5_move_1or2 - CMP $0x08, R0 + CMP $0x08, R1 BLO copy_5_move_4through7 JMP copy_5_move_8through16 copy_5_move_1or2: MOVBU (R13), R16 BFI $0, R16, $8, R14 - ADD R0, R13, R15 + ADD R1, R13, R15 MOVBU -1(R15), R16 BFI $0, R16, $8, R4 MOVB R14, (R9) - ADD R0, R9, R15 + ADD R1, R9, R15 MOVB R4, -1(R15) - ADD R0, R13, R13 - ADD R0, R9, R9 + ADD R1, R13, R13 + ADD R1, R9, R9 JMP copy_5_end copy_5_move_3: @@ -1852,76 +1730,84 @@ copy_5_move_3: BFI $0, R16, $8, R4 MOVH R14, (R9) MOVB R4, 2(R9) - ADD R0, R13, R13 - ADD R0, R9, R9 + ADD R1, R13, R13 + ADD R1, R9, R9 JMP copy_5_end copy_5_move_4through7: MOVWU (R13), R14 - ADD R0, R13, R15 + ADD R1, R13, R15 MOVWU -4(R15), R4 MOVW R14, (R9) - ADD R0, R9, R15 + ADD R1, R9, R15 MOVW R4, -4(R15) - ADD R0, R13, R13 - ADD R0, R9, R9 + ADD R1, R13, R13 + ADD R1, R9, R9 JMP copy_5_end copy_5_move_8through16: MOVD (R13), R14 - ADD R0, R13, R15 + ADD R1, R13, R15 MOVD -8(R15), R4 MOVD R14, (R9) - ADD R0, R9, R15 + ADD R1, R9, R15 MOVD R4, -8(R15) - ADD R0, R13, R13 - ADD R0, R9, R9 + ADD R1, R13, R13 + ADD R1, R9, R9 copy_5_end: - ADD R0, R11, R11 - SUB R0, R12, R12 + ADD R1, R11, R11 + SUB R1, R0, R0 // Copy match from the current buffer copy_match: - MOVD R9, R0 - SUB R1, R0, R0 + MOVD R9, R1 + SUB R12, R1, R1 // ml <= mo - CMP R1, R12 + CMP R12, R0 BHI copy_overlapping_match // Copy non-overlapping match - ADD R12, R11, R11 - MOVD R9, R1 - ADD R12, R9, R9 + ADD R0, R11, R11 + FMOVQ (R1), F0 + FMOVQ F0, (R9) + CMP $0x10, R0 + BHI copy_2_long + ADD R0, R9, R9 + JMP handle_loop + +copy_2_long: + ADD $16, R9, R12 + ADD R0, R9, R9 + ADD $0x10, R1, R1 + SUB $0x10, R0, R0 copy_2: - VLD1 (R0), [V0.B16] - VST1 [V0.B16], (R1) - ADD $0x10, R0, R0 - ADD $0x10, R1, R1 - SUBS $0x10, R12, R12 - BHI copy_2 - JMP handle_loop + FMOVQ (R1), F0 + FMOVQ F0, (R12) + ADD $0x10, R1, R1 + ADD $0x10, R12, R12 + SUBS $0x10, R0, R0 + BHI copy_2 + JMP handle_loop // Copy overlapping match copy_overlapping_match: - ADD R12, R11, R11 + ADD R0, R11, R11 copy_slow_3: - MOVBU (R0), R16 - BFI $0, R16, $8, R1 - MOVB R1, (R9) - ADD $1, R0, R0 + MOVBU (R1), R12 + MOVB R12, (R9) + ADD $1, R1, R1 ADD $1, R9, R9 - SUBS $1, R12, R12 + SUBS $1, R0, R0 BNE copy_slow_3 handle_loop: - MOVD ctx+16(FP), R0 - MOVD 96(R0), R16 + MOVD 40(RSP), R16 SUBS $1, R16, R16 - MOVD R16, 96(R0) + MOVD R16, 40(RSP) BPL sequenceDecs_decodeSync_amd64_main_loop loop_finished: @@ -1930,6 +1816,16 @@ loop_finished: MOVB R3, 40(R0) MOVD R5, 32(R0) + // s.seqSize += outPosition - ctx.outPosition + MOVD ctx+16(FP), R0 + MOVD s+0(FP), R1 + MOVD R11, R2 + MOVD 136(R0), R16 + SUB R16, R2, R2 + MOVD 256(R1), R16 + ADD R2, R16, R16 + MOVD R16, 256(R1) + // Update the context MOVD ctx+16(FP), R0 MOVD R11, 136(R0) @@ -1944,7 +1840,7 @@ loop_finished: // Return with match length error sequenceDecs_decodeSync_amd64_error_match_len_ofs_mismatch: - MOVD 24(RSP), R0 + MOVD 56(RSP), R0 MOVD ctx+16(FP), R1 MOVD R0, 216(R1) MOVD $0x00000001, R16 @@ -1954,7 +1850,7 @@ sequenceDecs_decodeSync_amd64_error_match_len_ofs_mismatch: // Return with match too long error sequenceDecs_decodeSync_amd64_error_match_len_too_big: MOVD ctx+16(FP), R0 - MOVD 24(RSP), R1 + MOVD 56(RSP), R1 MOVD R1, 216(R0) MOVD $0x00000002, R16 MOVD R16, ret+24(FP) @@ -1963,7 +1859,7 @@ sequenceDecs_decodeSync_amd64_error_match_len_too_big: // Return with match offset too long error error_match_off_too_big: MOVD ctx+16(FP), R0 - MOVD 16(RSP), R1 + MOVD 48(RSP), R1 MOVD R1, 224(R0) MOVD R11, 136(R0) MOVD $0x00000003, R16 @@ -1973,7 +1869,7 @@ error_match_off_too_big: // Return with not enough literals error error_not_enough_literals: MOVD ctx+16(FP), R0 - MOVD 32(RSP), R1 + MOVD 64(RSP), R1 MOVD R1, 208(R0) MOVD $0x00000004, R16 MOVD R16, ret+24(FP) @@ -1988,9 +1884,9 @@ error_overread: // Return with not enough output space error error_not_enough_space: MOVD ctx+16(FP), R0 - MOVD 32(RSP), R1 + MOVD 64(RSP), R1 MOVD R1, 208(R0) - MOVD 24(RSP), R1 + MOVD 56(RSP), R1 MOVD R1, 216(R0) MOVD R11, 136(R0) MOVD $0x00000005, R16 @@ -2001,7 +1897,7 @@ error_not_enough_space: // func sequenceDecs_decodeSync_safe_amd64(s *sequenceDecs, br *bitReader, ctx *decodeSyncAsmContext) int // Requires: CMOV, SSE -TEXT ·sequenceDecs_decodeSync_safe_arm64(SB), $64-32 +TEXT ·sequenceDecs_decodeSync_safe_arm64(SB), $96-32 MOVD br+8(FP), R1 MOVD 24(R1), R2 MOVBU 40(R1), R3 @@ -2013,30 +1909,38 @@ TEXT ·sequenceDecs_decodeSync_safe_arm64(SB), $64-32 MOVD 72(R0), R6 MOVD 80(R0), R7 MOVD 88(R0), R8 - MOVD $0, R1 + MOVD (R0), R1 MOVD R1, 16(RSP) + MOVD 24(R0), R1 MOVD R1, 24(RSP) + MOVD 48(R0), R1 MOVD R1, 32(RSP) + MOVD 96(R0), R1 + MOVD R1, 40(RSP) + MOVD $0, R1 + MOVD R1, 48(RSP) + MOVD R1, 56(RSP) + MOVD R1, 64(RSP) MOVD 112(R0), R9 MOVD 128(R0), R1 - MOVD R1, 40(RSP) + MOVD R1, 72(RSP) MOVD 144(R0), R10 MOVD 136(R0), R11 MOVD 200(R0), R1 - MOVD R1, 64(RSP) + MOVD R1, 96(RSP) MOVD 176(R0), R1 - MOVD R1, 56(RSP) + MOVD R1, 88(RSP) MOVD 184(R0), R0 - MOVD R0, 48(RSP) - MOVD 48(RSP), R0 - MOVD 56(RSP), R16 + MOVD R0, 80(RSP) + MOVD 80(RSP), R0 + MOVD 88(RSP), R16 ADD R0, R16, R16 - MOVD R16, 56(RSP) + MOVD R16, 88(RSP) // Calculate pointer to s.out[cap(s.out)] (a past-end pointer) - MOVD 40(RSP), R16 + MOVD 72(RSP), R16 ADD R9, R16, R16 - MOVD R16, 40(RSP) + MOVD R16, 72(RSP) // outBase += outPosition ADD R11, R9, R9 @@ -2047,8 +1951,7 @@ sequenceDecs_decodeSync_safe_amd64_main_loop: // Fill bitreader to have enough for the offset and match length. CMP $0x08, R5 BLT sequenceDecs_decodeSync_safe_amd64_fill_byte_by_byte - MOVD R3, R0 - LSR $0x03, R0, R0 + LSR $0x03, R3, R0 SUB R0, R12, R12 MOVD (R12), R2 SUB R0, R5, R5 @@ -2075,53 +1978,32 @@ sequenceDecs_decodeSync_safe_amd64_fill_check_overread: sequenceDecs_decodeSync_safe_amd64_fill_end: // Update offset MOVD R8, R0 - MOVD R3, R1 - MOVD R2, R13 - LSL R1, R13, R13 - UBFX $8, R0, $8, R16 - BFI $0, R16, $8, R1 + LSL R3, R2, R13 + LSR $0x01, R13, R13 + UBFX $8, R0, $8, R1 LSR $0x20, R0, R0 - TST R1, R1 - BEQ sequenceDecs_decodeSync_safe_amd64_of_update_zero ADD R1, R3, R3 - CMP $0x40, R3 - BHI sequenceDecs_decodeSync_safe_amd64_of_update_zero - CMP $0x40, R1 - BHS sequenceDecs_decodeSync_safe_amd64_of_update_zero - NEG R1, R1 + EOR $0x3f, R1, R1 LSR R1, R13, R13 ADD R13, R0, R0 - -sequenceDecs_decodeSync_safe_amd64_of_update_zero: - MOVD R0, 16(RSP) + MOVD R0, 48(RSP) // Update match length MOVD R7, R0 - MOVD R3, R1 - MOVD R2, R13 - LSL R1, R13, R13 - UBFX $8, R0, $8, R16 - BFI $0, R16, $8, R1 + LSL R3, R2, R13 + LSR $0x01, R13, R13 + UBFX $8, R0, $8, R1 LSR $0x20, R0, R0 - TST R1, R1 - BEQ sequenceDecs_decodeSync_safe_amd64_ml_update_zero ADD R1, R3, R3 - CMP $0x40, R3 - BHI sequenceDecs_decodeSync_safe_amd64_ml_update_zero - CMP $0x40, R1 - BHS sequenceDecs_decodeSync_safe_amd64_ml_update_zero - NEG R1, R1 + EOR $0x3f, R1, R1 LSR R1, R13, R13 ADD R13, R0, R0 - -sequenceDecs_decodeSync_safe_amd64_ml_update_zero: - MOVD R0, 24(RSP) + MOVD R0, 56(RSP) // Fill bitreader to have enough for the remaining CMP $0x08, R5 BLT sequenceDecs_decodeSync_safe_amd64_fill_2_byte_by_byte - MOVD R3, R0 - LSR $0x03, R0, R0 + LSR $0x03, R3, R0 SUB R0, R12, R12 MOVD (R12), R2 SUB R0, R5, R5 @@ -2148,114 +2030,78 @@ sequenceDecs_decodeSync_safe_amd64_fill_2_check_overread: sequenceDecs_decodeSync_safe_amd64_fill_2_end: // Update literal length MOVD R6, R0 - MOVD R3, R1 - MOVD R2, R13 - LSL R1, R13, R13 - UBFX $8, R0, $8, R16 - BFI $0, R16, $8, R1 + LSL R3, R2, R13 + LSR $0x01, R13, R13 + UBFX $8, R0, $8, R1 LSR $0x20, R0, R0 - TST R1, R1 - BEQ sequenceDecs_decodeSync_safe_amd64_ll_update_zero ADD R1, R3, R3 - CMP $0x40, R3 - BHI sequenceDecs_decodeSync_safe_amd64_ll_update_zero - CMP $0x40, R1 - BHS sequenceDecs_decodeSync_safe_amd64_ll_update_zero - NEG R1, R1 + EOR $0x3f, R1, R1 LSR R1, R13, R13 ADD R13, R0, R0 - -sequenceDecs_decodeSync_safe_amd64_ll_update_zero: - MOVD R0, 32(RSP) + MOVD R0, 64(RSP) // Fill bitreader for state updates - MOVD R12, 8(RSP) - MOVD R8, R0 - LSR $0x08, R0, R0 - MOVBU R0, R0 - MOVD ctx+16(FP), R1 - MOVD 96(R1), R16 - CMP $0x00, R16 - BEQ sequenceDecs_decodeSync_safe_amd64_skip_update + MOVD R12, 8(RSP) + UBFX $8, R8, $8, R0 + MOVD 40(RSP), R16 + CMP $0x00, R16 + BEQ sequenceDecs_decodeSync_safe_amd64_skip_update // Update Literal Length State MOVBU R6, R12 LSRW $0x10, R6, R6 - ADD R12, R3, R1 - MOVD R2, R13 - MOVD R1, R3 - NEG R1, R16 - ROR R16, R13, R13 - MOVD $0x00000001, R14 - BFI $0, R12, $8, R1 - LSLW R1, R14, R14 - SUBW $1, R14, R14 - AND R14, R13, R13 + LSL R3, R2, R13 + LSR $0x01, R13, R13 + ADD R12, R3, R3 + EOR $0x3f, R12, R12 + LSR R12, R13, R13 ADD R13, R6, R6 // Load ctx.llTable - MOVD ctx+16(FP), R1 - MOVD (R1), R1 - ADD R6<<3, R1, R15 - MOVD (R15), R6 + MOVD 16(RSP), R1 + MOVD (R1)(R6<<3), R6 // Update Match Length State MOVBU R7, R12 LSRW $0x10, R7, R7 - ADD R12, R3, R1 - MOVD R2, R13 - MOVD R1, R3 - NEG R1, R16 - ROR R16, R13, R13 - MOVD $0x00000001, R14 - BFI $0, R12, $8, R1 - LSLW R1, R14, R14 - SUBW $1, R14, R14 - AND R14, R13, R13 + LSL R3, R2, R13 + LSR $0x01, R13, R13 + ADD R12, R3, R3 + EOR $0x3f, R12, R12 + LSR R12, R13, R13 ADD R13, R7, R7 // Load ctx.mlTable - MOVD ctx+16(FP), R1 - MOVD 24(R1), R1 - ADD R7<<3, R1, R15 - MOVD (R15), R7 + MOVD 24(RSP), R1 + MOVD (R1)(R7<<3), R7 // Update Offset State MOVBU R8, R12 LSRW $0x10, R8, R8 - ADD R12, R3, R1 - MOVD R2, R13 - MOVD R1, R3 - NEG R1, R16 - ROR R16, R13, R13 - MOVD $0x00000001, R14 - BFI $0, R12, $8, R1 - LSLW R1, R14, R14 - SUBW $1, R14, R14 - AND R14, R13, R13 + LSL R3, R2, R13 + LSR $0x01, R13, R13 + ADD R12, R3, R3 + EOR $0x3f, R12, R12 + LSR R12, R13, R13 ADD R13, R8, R8 // Load ctx.ofTable - MOVD ctx+16(FP), R1 - MOVD 48(R1), R1 - ADD R8<<3, R1, R15 - MOVD (R15), R8 + MOVD 32(RSP), R1 + MOVD (R1)(R8<<3), R8 sequenceDecs_decodeSync_safe_amd64_skip_update: // Adjust offset - MOVD s+0(FP), R1 - MOVD 16(RSP), R12 - CMP $0x01, R0 - BLS sequenceDecs_decodeSync_safe_amd64_adjust_offsetB_1_or_0 - ADD $144, R1, R15 - VLD1 (R15), [V0.B16] - MOVD R12, 144(R1) - ADD $152, R1, R15 - VST1 [V0.B16], (R15) - JMP sequenceDecs_decodeSync_safe_amd64_after_adjust + MOVD s+0(FP), R1 + MOVD 48(RSP), R12 + CMP $0x01, R0 + BLS sequenceDecs_decodeSync_safe_amd64_adjust_offsetB_1_or_0 + FMOVQ 144(R1), F0 + MOVD R12, 144(R1) + FMOVQ F0, 152(R1) + JMP sequenceDecs_decodeSync_safe_amd64_after_adjust sequenceDecs_decodeSync_safe_amd64_adjust_offsetB_1_or_0: - MOVD 32(RSP), R16 + MOVD 64(RSP), R16 CMP $0x00000000, R16 BNE sequenceDecs_decodeSync_safe_amd64_adjust_offset_maybezero ADD $1, R12, R12 @@ -2293,16 +2139,11 @@ sequenceDecs_decodeSync_safe_amd64_adjust_skip: MOVD R13, R12 sequenceDecs_decodeSync_safe_amd64_after_adjust: - MOVD R12, 16(RSP) + MOVD R12, 48(RSP) // Check values - MOVD 24(RSP), R0 - MOVD 32(RSP), R1 - ADD R1, R0, R13 - MOVD s+0(FP), R14 - MOVD 256(R14), R16 - ADD R13, R16, R16 - MOVD R16, 256(R14) + MOVD 56(RSP), R0 + MOVD 64(RSP), R1 MOVD ctx+16(FP), R13 MOVD 104(R13), R16 SUBS R1, R16, R16 @@ -2316,60 +2157,54 @@ sequenceDecs_decodeSync_safe_amd64_after_adjust: BNE sequenceDecs_decodeSync_safe_amd64_error_match_len_ofs_mismatch sequenceDecs_decodeSync_safe_amd64_match_len_ofs_ok: - MOVD 32(RSP), R0 - MOVD 16(RSP), R1 - MOVD 24(RSP), R12 - // Check if we have enough space in s.out - ADD R12, R0, R13 + ADD R0, R1, R13 ADD R9, R13, R13 - MOVD 40(RSP), R16 + MOVD 72(RSP), R16 CMP R16, R13 BHI error_not_enough_space // Copy literals - TST R0, R0 + TST R1, R1 BEQ check_offset - MOVD R0, R13 + MOVD R1, R13 SUBS $0x10, R13, R13 BLO copy_1_small copy_1_loop: - VLD1 (R10), [V0.B16] - VST1 [V0.B16], (R9) - ADD $0x10, R10, R10 - ADD $0x10, R9, R9 - SUBS $0x10, R13, R13 - BHS copy_1_loop - ADD R13, R10, R10 - ADD $16, R10, R10 - ADD R13, R9, R9 - ADD $16, R9, R9 - ADD $-16, R10, R15 - VLD1 (R15), [V0.B16] - ADD $-16, R9, R15 - VST1 [V0.B16], (R15) - JMP copy_1_end + FMOVQ (R10), F0 + FMOVQ F0, (R9) + ADD $0x10, R10, R10 + ADD $0x10, R9, R9 + SUBS $0x10, R13, R13 + BHS copy_1_loop + ADD R13, R10, R10 + ADD $16, R10, R10 + ADD R13, R9, R9 + ADD $16, R9, R9 + FMOVQ -16(R10), F0 + FMOVQ F0, -16(R9) + JMP copy_1_end copy_1_small: - CMP $0x03, R0 + CMP $0x03, R1 BEQ copy_1_move_3 BLO copy_1_move_1or2 - CMP $0x08, R0 + CMP $0x08, R1 BLO copy_1_move_4through7 JMP copy_1_move_8through16 copy_1_move_1or2: MOVBU (R10), R16 BFI $0, R16, $8, R13 - ADD R0, R10, R15 + ADD R1, R10, R15 MOVBU -1(R15), R16 BFI $0, R16, $8, R14 MOVB R13, (R9) - ADD R0, R9, R15 + ADD R1, R9, R15 MOVB R14, -1(R15) - ADD R0, R10, R10 - ADD R0, R9, R9 + ADD R1, R10, R10 + ADD R1, R9, R9 JMP copy_1_end copy_1_move_3: @@ -2378,158 +2213,154 @@ copy_1_move_3: BFI $0, R16, $8, R14 MOVH R13, (R9) MOVB R14, 2(R9) - ADD R0, R10, R10 - ADD R0, R9, R9 + ADD R1, R10, R10 + ADD R1, R9, R9 JMP copy_1_end copy_1_move_4through7: MOVWU (R10), R13 - ADD R0, R10, R15 + ADD R1, R10, R15 MOVWU -4(R15), R14 MOVW R13, (R9) - ADD R0, R9, R15 + ADD R1, R9, R15 MOVW R14, -4(R15) - ADD R0, R10, R10 - ADD R0, R9, R9 + ADD R1, R10, R10 + ADD R1, R9, R9 JMP copy_1_end copy_1_move_8through16: MOVD (R10), R13 - ADD R0, R10, R15 + ADD R1, R10, R15 MOVD -8(R15), R14 MOVD R13, (R9) - ADD R0, R9, R15 + ADD R1, R9, R15 MOVD R14, -8(R15) - ADD R0, R10, R10 - ADD R0, R9, R9 + ADD R1, R10, R10 + ADD R1, R9, R9 copy_1_end: - ADD R0, R11, R11 + ADD R1, R11, R11 // Malformed input if seq.mo > t+len(hist) || seq.mo > s.windowSize) check_offset: - MOVD R11, R0 - MOVD 48(RSP), R16 - ADD R16, R0, R0 - CMP R0, R1 + MOVD R11, R1 + MOVD 80(RSP), R16 + ADD R16, R1, R1 + CMP R1, R12 BGT error_match_off_too_big - MOVD 64(RSP), R16 - CMP R16, R1 + MOVD 96(RSP), R16 + CMP R16, R12 BGT error_match_off_too_big // Copy match from history - MOVD R1, R0 - SUBS R11, R0, R0 + MOVD R12, R1 + SUBS R11, R1, R1 BLS copy_match - MOVD 56(RSP), R13 - SUB R0, R13, R13 - CMP R0, R12 + MOVD 88(RSP), R13 + SUB R1, R13, R13 + CMP R1, R0 BGT copy_all_from_history - MOVD R12, R0 - SUBS $0x10, R0, R0 + MOVD R0, R1 + SUBS $0x10, R1, R1 BLO copy_4_small copy_4_loop: - VLD1 (R13), [V0.B16] - VST1 [V0.B16], (R9) - ADD $0x10, R13, R13 - ADD $0x10, R9, R9 - SUBS $0x10, R0, R0 - BHS copy_4_loop - ADD R0, R13, R13 - ADD $16, R13, R13 - ADD R0, R9, R9 - ADD $16, R9, R9 - ADD $-16, R13, R15 - VLD1 (R15), [V0.B16] - ADD $-16, R9, R15 - VST1 [V0.B16], (R15) - JMP copy_4_end + FMOVQ (R13), F0 + FMOVQ F0, (R9) + ADD $0x10, R13, R13 + ADD $0x10, R9, R9 + SUBS $0x10, R1, R1 + BHS copy_4_loop + ADD R1, R13, R13 + ADD $16, R13, R13 + ADD R1, R9, R9 + ADD $16, R9, R9 + FMOVQ -16(R13), F0 + FMOVQ F0, -16(R9) + JMP copy_4_end copy_4_small: - CMP $0x03, R12 + CMP $0x03, R0 BEQ copy_4_move_3 - CMP $0x08, R12 + CMP $0x08, R0 BLO copy_4_move_4through7 JMP copy_4_move_8through16 copy_4_move_3: - MOVHU (R13), R0 + MOVHU (R13), R1 MOVBU 2(R13), R16 - BFI $0, R16, $8, R1 - MOVH R0, (R9) - MOVB R1, 2(R9) - ADD R12, R13, R13 - ADD R12, R9, R9 + BFI $0, R16, $8, R12 + MOVH R1, (R9) + MOVB R12, 2(R9) + ADD R0, R13, R13 + ADD R0, R9, R9 JMP copy_4_end copy_4_move_4through7: - MOVWU (R13), R0 - ADD R12, R13, R15 - MOVWU -4(R15), R1 - MOVW R0, (R9) - ADD R12, R9, R15 - MOVW R1, -4(R15) - ADD R12, R13, R13 - ADD R12, R9, R9 + MOVWU (R13), R1 + ADD R0, R13, R15 + MOVWU -4(R15), R12 + MOVW R1, (R9) + ADD R0, R9, R15 + MOVW R12, -4(R15) + ADD R0, R13, R13 + ADD R0, R9, R9 JMP copy_4_end copy_4_move_8through16: - MOVD (R13), R0 - ADD R12, R13, R15 - MOVD -8(R15), R1 - MOVD R0, (R9) - ADD R12, R9, R15 - MOVD R1, -8(R15) - ADD R12, R13, R13 - ADD R12, R9, R9 + MOVD (R13), R1 + ADD R0, R13, R15 + MOVD -8(R15), R12 + MOVD R1, (R9) + ADD R0, R9, R15 + MOVD R12, -8(R15) + ADD R0, R13, R13 + ADD R0, R9, R9 copy_4_end: - ADD R12, R11, R11 + ADD R0, R11, R11 JMP handle_loop JMP loop_finished copy_all_from_history: - MOVD R0, R14 + MOVD R1, R14 SUBS $0x10, R14, R14 BLO copy_5_small copy_5_loop: - VLD1 (R13), [V0.B16] - VST1 [V0.B16], (R9) - ADD $0x10, R13, R13 - ADD $0x10, R9, R9 - SUBS $0x10, R14, R14 - BHS copy_5_loop - ADD R14, R13, R13 - ADD $16, R13, R13 - ADD R14, R9, R9 - ADD $16, R9, R9 - ADD $-16, R13, R15 - VLD1 (R15), [V0.B16] - ADD $-16, R9, R15 - VST1 [V0.B16], (R15) - JMP copy_5_end + FMOVQ (R13), F0 + FMOVQ F0, (R9) + ADD $0x10, R13, R13 + ADD $0x10, R9, R9 + SUBS $0x10, R14, R14 + BHS copy_5_loop + ADD R14, R13, R13 + ADD $16, R13, R13 + ADD R14, R9, R9 + ADD $16, R9, R9 + FMOVQ -16(R13), F0 + FMOVQ F0, -16(R9) + JMP copy_5_end copy_5_small: - CMP $0x03, R0 + CMP $0x03, R1 BEQ copy_5_move_3 BLO copy_5_move_1or2 - CMP $0x08, R0 + CMP $0x08, R1 BLO copy_5_move_4through7 JMP copy_5_move_8through16 copy_5_move_1or2: MOVBU (R13), R16 BFI $0, R16, $8, R14 - ADD R0, R13, R15 + ADD R1, R13, R15 MOVBU -1(R15), R16 BFI $0, R16, $8, R4 MOVB R14, (R9) - ADD R0, R9, R15 + ADD R1, R9, R15 MOVB R4, -1(R15) - ADD R0, R13, R13 - ADD R0, R9, R9 + ADD R1, R13, R13 + ADD R1, R9, R9 JMP copy_5_end copy_5_move_3: @@ -2538,140 +2369,136 @@ copy_5_move_3: BFI $0, R16, $8, R4 MOVH R14, (R9) MOVB R4, 2(R9) - ADD R0, R13, R13 - ADD R0, R9, R9 + ADD R1, R13, R13 + ADD R1, R9, R9 JMP copy_5_end copy_5_move_4through7: MOVWU (R13), R14 - ADD R0, R13, R15 + ADD R1, R13, R15 MOVWU -4(R15), R4 MOVW R14, (R9) - ADD R0, R9, R15 + ADD R1, R9, R15 MOVW R4, -4(R15) - ADD R0, R13, R13 - ADD R0, R9, R9 + ADD R1, R13, R13 + ADD R1, R9, R9 JMP copy_5_end copy_5_move_8through16: MOVD (R13), R14 - ADD R0, R13, R15 + ADD R1, R13, R15 MOVD -8(R15), R4 MOVD R14, (R9) - ADD R0, R9, R15 + ADD R1, R9, R15 MOVD R4, -8(R15) - ADD R0, R13, R13 - ADD R0, R9, R9 + ADD R1, R13, R13 + ADD R1, R9, R9 copy_5_end: - ADD R0, R11, R11 - SUB R0, R12, R12 + ADD R1, R11, R11 + SUB R1, R0, R0 // Copy match from the current buffer copy_match: - MOVD R9, R0 - SUB R1, R0, R0 + MOVD R9, R1 + SUB R12, R1, R1 // ml <= mo - CMP R1, R12 + CMP R12, R0 BHI copy_overlapping_match // Copy non-overlapping match - ADD R12, R11, R11 - MOVD R12, R1 - SUBS $0x10, R1, R1 + ADD R0, R11, R11 + MOVD R0, R12 + SUBS $0x10, R12, R12 BLO copy_2_small copy_2_loop: - VLD1 (R0), [V0.B16] - VST1 [V0.B16], (R9) - ADD $0x10, R0, R0 - ADD $0x10, R9, R9 - SUBS $0x10, R1, R1 - BHS copy_2_loop - ADD R1, R0, R0 - ADD $16, R0, R0 - ADD R1, R9, R9 - ADD $16, R9, R9 - ADD $-16, R0, R15 - VLD1 (R15), [V0.B16] - ADD $-16, R9, R15 - VST1 [V0.B16], (R15) - JMP copy_2_end + FMOVQ (R1), F0 + FMOVQ F0, (R9) + ADD $0x10, R1, R1 + ADD $0x10, R9, R9 + SUBS $0x10, R12, R12 + BHS copy_2_loop + ADD R12, R1, R1 + ADD $16, R1, R1 + ADD R12, R9, R9 + ADD $16, R9, R9 + FMOVQ -16(R1), F0 + FMOVQ F0, -16(R9) + JMP copy_2_end copy_2_small: - CMP $0x03, R12 + CMP $0x03, R0 BEQ copy_2_move_3 BLO copy_2_move_1or2 - CMP $0x08, R12 + CMP $0x08, R0 BLO copy_2_move_4through7 JMP copy_2_move_8through16 copy_2_move_1or2: - MOVBU (R0), R16 - BFI $0, R16, $8, R1 - ADD R12, R0, R15 + MOVBU (R1), R16 + BFI $0, R16, $8, R12 + ADD R0, R1, R15 MOVBU -1(R15), R16 BFI $0, R16, $8, R13 - MOVB R1, (R9) - ADD R12, R9, R15 + MOVB R12, (R9) + ADD R0, R9, R15 MOVB R13, -1(R15) - ADD R12, R0, R0 - ADD R12, R9, R9 + ADD R0, R1, R1 + ADD R0, R9, R9 JMP copy_2_end copy_2_move_3: - MOVHU (R0), R1 - MOVBU 2(R0), R16 + MOVHU (R1), R12 + MOVBU 2(R1), R16 BFI $0, R16, $8, R13 - MOVH R1, (R9) + MOVH R12, (R9) MOVB R13, 2(R9) - ADD R12, R0, R0 - ADD R12, R9, R9 + ADD R0, R1, R1 + ADD R0, R9, R9 JMP copy_2_end copy_2_move_4through7: - MOVWU (R0), R1 - ADD R12, R0, R15 + MOVWU (R1), R12 + ADD R0, R1, R15 MOVWU -4(R15), R13 - MOVW R1, (R9) - ADD R12, R9, R15 + MOVW R12, (R9) + ADD R0, R9, R15 MOVW R13, -4(R15) - ADD R12, R0, R0 - ADD R12, R9, R9 + ADD R0, R1, R1 + ADD R0, R9, R9 JMP copy_2_end copy_2_move_8through16: - MOVD (R0), R1 - ADD R12, R0, R15 + MOVD (R1), R12 + ADD R0, R1, R15 MOVD -8(R15), R13 - MOVD R1, (R9) - ADD R12, R9, R15 + MOVD R12, (R9) + ADD R0, R9, R15 MOVD R13, -8(R15) - ADD R12, R0, R0 - ADD R12, R9, R9 + ADD R0, R1, R1 + ADD R0, R9, R9 copy_2_end: JMP handle_loop // Copy overlapping match copy_overlapping_match: - ADD R12, R11, R11 + ADD R0, R11, R11 copy_slow_3: - MOVBU (R0), R16 - BFI $0, R16, $8, R1 - MOVB R1, (R9) - ADD $1, R0, R0 + MOVBU (R1), R12 + MOVB R12, (R9) + ADD $1, R1, R1 ADD $1, R9, R9 - SUBS $1, R12, R12 + SUBS $1, R0, R0 BNE copy_slow_3 handle_loop: - MOVD ctx+16(FP), R0 - MOVD 96(R0), R16 + MOVD 40(RSP), R16 SUBS $1, R16, R16 - MOVD R16, 96(R0) + MOVD R16, 40(RSP) BPL sequenceDecs_decodeSync_safe_amd64_main_loop loop_finished: @@ -2680,6 +2507,16 @@ loop_finished: MOVB R3, 40(R0) MOVD R5, 32(R0) + // s.seqSize += outPosition - ctx.outPosition + MOVD ctx+16(FP), R0 + MOVD s+0(FP), R1 + MOVD R11, R2 + MOVD 136(R0), R16 + SUB R16, R2, R2 + MOVD 256(R1), R16 + ADD R2, R16, R16 + MOVD R16, 256(R1) + // Update the context MOVD ctx+16(FP), R0 MOVD R11, 136(R0) @@ -2694,7 +2531,7 @@ loop_finished: // Return with match length error sequenceDecs_decodeSync_safe_amd64_error_match_len_ofs_mismatch: - MOVD 24(RSP), R0 + MOVD 56(RSP), R0 MOVD ctx+16(FP), R1 MOVD R0, 216(R1) MOVD $0x00000001, R16 @@ -2704,7 +2541,7 @@ sequenceDecs_decodeSync_safe_amd64_error_match_len_ofs_mismatch: // Return with match too long error sequenceDecs_decodeSync_safe_amd64_error_match_len_too_big: MOVD ctx+16(FP), R0 - MOVD 24(RSP), R1 + MOVD 56(RSP), R1 MOVD R1, 216(R0) MOVD $0x00000002, R16 MOVD R16, ret+24(FP) @@ -2713,7 +2550,7 @@ sequenceDecs_decodeSync_safe_amd64_error_match_len_too_big: // Return with match offset too long error error_match_off_too_big: MOVD ctx+16(FP), R0 - MOVD 16(RSP), R1 + MOVD 48(RSP), R1 MOVD R1, 224(R0) MOVD R11, 136(R0) MOVD $0x00000003, R16 @@ -2723,7 +2560,7 @@ error_match_off_too_big: // Return with not enough literals error error_not_enough_literals: MOVD ctx+16(FP), R0 - MOVD 32(RSP), R1 + MOVD 64(RSP), R1 MOVD R1, 208(R0) MOVD $0x00000004, R16 MOVD R16, ret+24(FP) @@ -2738,9 +2575,9 @@ error_overread: // Return with not enough output space error error_not_enough_space: MOVD ctx+16(FP), R0 - MOVD 32(RSP), R1 + MOVD 64(RSP), R1 MOVD R1, 208(R0) - MOVD 24(RSP), R1 + MOVD 56(RSP), R1 MOVD R1, 216(R0) MOVD R11, 136(R0) MOVD $0x00000005, R16 diff --git a/vendor/github.com/klauspost/compress/zstd/seqdec_asm.go b/vendor/github.com/klauspost/compress/zstd/seqdec_asm.go index 42ce5de71636..94559c459a8f 100644 --- a/vendor/github.com/klauspost/compress/zstd/seqdec_asm.go +++ b/vendor/github.com/klauspost/compress/zstd/seqdec_asm.go @@ -83,9 +83,10 @@ const errorOverread = 6 // When the output and literal buffers have compressedBlockOverAlloc (16) // bytes of slack past their logical use, the assembly may use extended // memory copies that read and write in 16-byte blocks, overrunning the end -// of a literal run or match by up to 15 bytes. Otherwise it must use the -// bounds-exact ("safe") copies. This mirrors the analogous, always-dynamic -// selection in executeSimple below. +// of a literal run or match by up to 15 bytes; an empty literal run still +// copies one block, reading and writing the full 16 bytes of slack past +// its position. Otherwise it must use the bounds-exact ("safe") copies. +// This mirrors the analogous, always-dynamic selection in executeSimple below. // // The unsafe copies were disabled in #644 (2022) as a mitigation for a // crash, but that crash's root cause — an unguarded bitReader overread that diff --git a/vendor/modules.txt b/vendor/modules.txt index 501237c5a1e8..77d89fad8ef0 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -145,8 +145,8 @@ github.com/grpc-ecosystem/grpc-gateway/v2/utilities # github.com/inconshreveable/mousetrap v1.1.0 ## explicit; go 1.18 github.com/inconshreveable/mousetrap -# github.com/klauspost/compress v1.19.2 -## explicit; go 1.24 +# github.com/klauspost/compress v1.20.1 +## explicit; go 1.25 github.com/klauspost/compress github.com/klauspost/compress/fse github.com/klauspost/compress/huff0