Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Fix incorrectly rejected blocks #569

Merged
merged 4 commits into from Apr 25, 2022
Merged

Fix incorrectly rejected blocks #569

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
d1497b0
Fix incorrectly rejected blocks
klauspost Apr 25, 2022
1199343
Use safe more strictly.
klauspost Apr 25, 2022
e9efc87
Generate directly to output
klauspost Apr 25, 2022
c11aad3
Update benchmarks. Remove cgo.
klauspost Apr 25, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
4 changes: 2 additions & 2 deletions zstd/_generate/gen.go
View file
Open in desktop
Expand Up @@ -1044,12 +1044,12 @@ func (e executeSimple) executeSingleTriple(c *executeSingleTripleContext, handle
if !e.useSeqs {
Comment("Check if we have enough space in s.out")
{
// baseAfterCopy = ll + ml + c.outBese
// baseAfterCopy = ll + ml + c.outBase
baseAfterCopy := GP64()
LEAQ(Mem{Base: ll, Index: ml, Scale: 1}, baseAfterCopy)
ADDQ(c.outBase, baseAfterCopy)
CMPQ(baseAfterCopy, c.outEndPtr)
JAE(LabelRef("error_not_enough_space"))
JA(LabelRef("error_not_enough_space"))
}
}

Expand Down
6 changes: 5 additions & 1 deletion zstd/blockdec.go
View file
Open in desktop
Expand Up @@ -494,11 +494,15 @@ func (b *blockDec) decodeCompressed(hist *history) error {
b.dst = append(b.dst, hist.decoders.literals...)
return nil
}

before := len(hist.decoders.out)
err = hist.decoders.decodeSync(hist.b[hist.ignoreBuffer:])
if err != nil {
return err
}
if hist.decoders.maxSyncLen > 0 {
hist.decoders.maxSyncLen += uint64(before)
hist.decoders.maxSyncLen -= uint64(len(hist.decoders.out))
}
b.dst = hist.decoders.out
hist.recentOffsets = hist.decoders.prevOffset
return nil
Expand Down
6 changes: 3 additions & 3 deletions zstd/decoder_test.go
View file
Open in desktop
Expand Up @@ -703,7 +703,7 @@ func TestDecoderRegression(t *testing.T) {
if err != nil {
t.Error(err)
}
got, gotErr := dec.DecodeAll(in, nil)
got, gotErr := dec.DecodeAll(in, make([]byte, 0, len(in)))
t.Log("Received:", len(got), gotErr)

// Check if we got the same:
Expand All @@ -713,7 +713,7 @@ func TestDecoderRegression(t *testing.T) {
return
}
defer decL.Close()
got2, gotErr2 := decL.DecodeAll(in, nil)
got2, gotErr2 := decL.DecodeAll(in, make([]byte, 0, len(in)/2))
t.Log("Fresh Reader received:", len(got2), gotErr2)
if gotErr != gotErr2 {
if gotErr != nil && gotErr2 != nil && gotErr.Error() != gotErr2.Error() {
Expand Down Expand Up @@ -741,7 +741,7 @@ func TestDecoderRegression(t *testing.T) {
if err != nil {
t.Error(err)
}
got, gotErr := dec.DecodeAll(in, nil)
got, gotErr := dec.DecodeAll(in, make([]byte, 0, len(in)))
t.Log("Received:", len(got), gotErr)

// Check a fresh instance
Expand Down
6 changes: 3 additions & 3 deletions zstd/encoder_test.go
View file
Open in desktop
Expand Up @@ -251,12 +251,12 @@ func TestEncoderRegression(t *testing.T) {
t.Error(err)
}
encoded := enc.EncodeAll(in, nil)
got, err := dec.DecodeAll(encoded, nil)
// Usually too small...
got, err := dec.DecodeAll(encoded, make([]byte, 0, len(in)))
if err != nil {
t.Logf("error: %v\nwant: %v\ngot: %v", err, len(in), len(got))
t.Fatal(err)
}

// Use the Writer
var dst bytes.Buffer
enc.ResetContentSize(&dst, int64(len(in)))
Expand All @@ -269,7 +269,7 @@ func TestEncoderRegression(t *testing.T) {
t.Error(err)
}
encoded = dst.Bytes()
got, err = dec.DecodeAll(encoded, nil)
got, err = dec.DecodeAll(encoded, make([]byte, 0, len(in)/2))
if err != nil {
t.Logf("error: %v\nwant: %v\ngot: %v", err, in, got)
t.Error(err)
Expand Down
4 changes: 3 additions & 1 deletion zstd/seqdec_amd64.go
View file
Open in desktop
Expand Up @@ -62,7 +62,6 @@ func (s *sequenceDecs) decodeSyncSimple(hist []byte) (bool, error) {
if s.maxSyncLen > 0 && uint64(cap(s.out))-compressedBlockOverAlloc < s.maxSyncLen {
useSafe = true
}

br := s.br

maxBlockSize := maxCompressedBlockSize
Expand Down Expand Up @@ -123,6 +122,9 @@ func (s *sequenceDecs) decodeSyncSimple(hist []byte) (bool, error) {

case errorNotEnoughSpace:
size := ctx.outPosition + ctx.ll + ctx.ml
if debugDecoder {
println("msl:", s.maxSyncLen, "cap", cap(s.out), "bef:", startSize, "sz:", size-startSize, "mbs:", maxBlockSize, "outsz:", cap(s.out)-startSize)
}
return true, fmt.Errorf("output (%d) bigger than max block size (%d)", size-startSize, maxBlockSize)

default:
Expand Down
12 changes: 8 additions & 4 deletions zstd/seqdec_amd64.s
View file
Open in desktop
Expand Up @@ -1667,7 +1667,7 @@ sequenceDecs_decodeSync_amd64_match_len_ofs_ok:
LEAQ (AX)(R13*1), R14
ADDQ R10, R14
CMPQ R14, 32(SP)
JAE error_not_enough_space
JA error_not_enough_space

// Copy literals
TESTQ AX, AX
Expand Down Expand Up @@ -1913,6 +1913,7 @@ error_not_enough_space:
MOVQ CX, 208(AX)
MOVQ 16(SP), CX
MOVQ CX, 216(AX)
MOVQ R12, 136(AX)
Copy link
Owner Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

These showed up after re-regenerating.

MOVQ $0x00000005, ret+24(FP)
RET

Expand Down Expand Up @@ -2173,7 +2174,7 @@ sequenceDecs_decodeSync_bmi2_match_len_ofs_ok:
LEAQ (CX)(R13*1), R14
ADDQ R9, R14
CMPQ R14, 32(SP)
JAE error_not_enough_space
JA error_not_enough_space

// Copy literals
TESTQ CX, CX
Expand Down Expand Up @@ -2419,6 +2420,7 @@ error_not_enough_space:
MOVQ CX, 208(AX)
MOVQ 16(SP), CX
MOVQ CX, 216(AX)
MOVQ R11, 136(AX)
MOVQ $0x00000005, ret+24(FP)
RET

Expand Down Expand Up @@ -2701,7 +2703,7 @@ sequenceDecs_decodeSync_safe_amd64_match_len_ofs_ok:
LEAQ (AX)(R13*1), R14
ADDQ R10, R14
CMPQ R14, 32(SP)
JAE error_not_enough_space
JA error_not_enough_space

// Copy literals
TESTQ AX, AX
Expand Down Expand Up @@ -2976,6 +2978,7 @@ error_not_enough_space:
MOVQ CX, 208(AX)
MOVQ 16(SP), CX
MOVQ CX, 216(AX)
MOVQ R12, 136(AX)
MOVQ $0x00000005, ret+24(FP)
RET

Expand Down Expand Up @@ -3236,7 +3239,7 @@ sequenceDecs_decodeSync_safe_bmi2_match_len_ofs_ok:
LEAQ (CX)(R13*1), R14
ADDQ R9, R14
CMPQ R14, 32(SP)
JAE error_not_enough_space
JA error_not_enough_space

// Copy literals
TESTQ CX, CX
Expand Down Expand Up @@ -3511,5 +3514,6 @@ error_not_enough_space:
MOVQ CX, 208(AX)
MOVQ 16(SP), CX
MOVQ CX, 216(AX)
MOVQ R11, 136(AX)
MOVQ $0x00000005, ret+24(FP)
RET
Binary file modified zstd/testdata/comp-crashers.zip
View file
Open in desktop
Binary file not shown.