diff --git a/flate/fuzz_test.go b/flate/fuzz_test.go new file mode 100644 index 0000000000..9f20a8775d --- /dev/null +++ b/flate/fuzz_test.go @@ -0,0 +1,113 @@ +//go:build go1.18 +// +build go1.18 + +package flate + +import ( + "bytes" + "io" + "strconv" + "testing" + + "github.com/klauspost/compress/internal/fuzz" +) + +func FuzzEncoding(f *testing.F) { + fuzz.AddFromZip(f, "testdata/regression.zip", true, false) + fuzz.AddFromZip(f, "testdata/fuzz/encode-raw-corpus.zip", true, testing.Short()) + fuzz.AddFromZip(f, "testdata/fuzz/FuzzEncoding.zip", false, testing.Short()) + // Fuzzing tweaks: + const ( + // Test a subset of encoders. + startFuzz = HuffmanOnly + endFuzz = BestCompression + + // Also tests with dictionaries... + testDicts = true + + // Max input size: + maxSize = 1 << 20 + ) + decoder := NewReader(nil) + buf := new(bytes.Buffer) + + f.Fuzz(func(t *testing.T, data []byte) { + if len(data) > maxSize { + return + } + for level := startFuzz; level <= endFuzz; level++ { + msg := "level " + strconv.Itoa(level) + ":" + buf.Reset() + fw, err := NewWriter(buf, level) + if err != nil { + t.Fatal(msg + err.Error()) + } + n, err := fw.Write(data) + if n != len(data) { + t.Fatal(msg + "short write") + } + if err != nil { + t.Fatal(msg + err.Error()) + } + err = fw.Close() + if err != nil { + t.Fatal(msg + err.Error()) + } + decoder.(Resetter).Reset(buf, nil) + data2, err := io.ReadAll(decoder) + if err != nil { + t.Fatal(msg + err.Error()) + } + if !bytes.Equal(data, data2) { + t.Fatal(msg + "not equal") + } + // Do it again... + msg = "level " + strconv.Itoa(level) + " (reset):" + buf.Reset() + fw.Reset(buf) + n, err = fw.Write(data) + if n != len(data) { + t.Fatal(msg + "short write") + } + if err != nil { + t.Fatal(msg + err.Error()) + } + err = fw.Close() + if err != nil { + t.Fatal(msg + err.Error()) + } + decoder.(Resetter).Reset(buf, nil) + data2, err = io.ReadAll(decoder) + if err != nil { + t.Fatal(msg + err.Error()) + } + if !bytes.Equal(data, data2) { + t.Fatal(msg + "not equal") + } + } + + // Split into two and use history... + buf.Reset() + err := StatelessDeflate(buf, data[:len(data)/2], false, nil) + if err != nil { + t.Error(err) + } + + // Use top half as dictionary... + dict := data[:len(data)/2] + err = StatelessDeflate(buf, data[len(data)/2:], true, dict) + if err != nil { + t.Error(err) + } + + decoder.(Resetter).Reset(buf, nil) + data2, err := io.ReadAll(decoder) + if err != nil { + t.Error(err) + } + if !bytes.Equal(data, data2) { + //fmt.Printf("want:%x\ngot: %x\n", data1, data2) + t.Error("not equal") + } + }) +} diff --git a/flate/testdata/fuzz/FuzzEncoding.zip b/flate/testdata/fuzz/FuzzEncoding.zip new file mode 100644 index 0000000000..5af4b5bb64 Binary files /dev/null and b/flate/testdata/fuzz/FuzzEncoding.zip differ diff --git a/flate/testdata/fuzz/encode-raw-corpus.zip b/flate/testdata/fuzz/encode-raw-corpus.zip new file mode 100644 index 0000000000..7b33f54fc0 Binary files /dev/null and b/flate/testdata/fuzz/encode-raw-corpus.zip differ diff --git a/internal/fuzz/helpers.go b/internal/fuzz/helpers.go new file mode 100644 index 0000000000..3efadad39a --- /dev/null +++ b/internal/fuzz/helpers.go @@ -0,0 +1,124 @@ +//go:build go1.18 +// +build go1.18 + +package fuzz + +import ( + "archive/zip" + "bytes" + "fmt" + "go/ast" + "go/parser" + "go/token" + "io" + "os" + "strconv" + "testing" +) + +// AddFromZip will read the supplied zip and add all as corpus for f. +func AddFromZip(f *testing.F, filename string, raw, short bool) { + file, err := os.Open(filename) + if err != nil { + f.Fatal(err) + } + fi, err := file.Stat() + if err != nil { + f.Fatal(err) + } + zr, err := zip.NewReader(file, fi.Size()) + if err != nil { + f.Fatal(err) + } + for i, file := range zr.File { + if short && i%10 != 0 { + continue + } + rc, err := file.Open() + if err != nil { + f.Fatal(err) + } + + b, err := io.ReadAll(rc) + if err != nil { + f.Fatal(err) + } + rc.Close() + raw := raw + if bytes.HasPrefix(b, []byte("go test fuzz")) { + raw = false + } + if raw { + f.Add(b) + continue + } + vals, err := unmarshalCorpusFile(b) + if err != nil { + f.Fatal(err) + } + for _, v := range vals { + f.Add(v) + } + } +} + +// unmarshalCorpusFile decodes corpus bytes into their respective values. +func unmarshalCorpusFile(b []byte) ([][]byte, error) { + if len(b) == 0 { + return nil, fmt.Errorf("cannot unmarshal empty string") + } + lines := bytes.Split(b, []byte("\n")) + if len(lines) < 2 { + return nil, fmt.Errorf("must include version and at least one value") + } + var vals = make([][]byte, 0, len(lines)-1) + for _, line := range lines[1:] { + line = bytes.TrimSpace(line) + if len(line) == 0 { + continue + } + v, err := parseCorpusValue(line) + if err != nil { + return nil, fmt.Errorf("malformed line %q: %v", line, err) + } + vals = append(vals, v) + } + return vals, nil +} + +// parseCorpusValue +func parseCorpusValue(line []byte) ([]byte, error) { + fs := token.NewFileSet() + expr, err := parser.ParseExprFrom(fs, "(test)", line, 0) + if err != nil { + return nil, err + } + call, ok := expr.(*ast.CallExpr) + if !ok { + return nil, fmt.Errorf("expected call expression") + } + if len(call.Args) != 1 { + return nil, fmt.Errorf("expected call expression with 1 argument; got %d", len(call.Args)) + } + arg := call.Args[0] + + if arrayType, ok := call.Fun.(*ast.ArrayType); ok { + if arrayType.Len != nil { + return nil, fmt.Errorf("expected []byte or primitive type") + } + elt, ok := arrayType.Elt.(*ast.Ident) + if !ok || elt.Name != "byte" { + return nil, fmt.Errorf("expected []byte") + } + lit, ok := arg.(*ast.BasicLit) + if !ok || lit.Kind != token.STRING { + return nil, fmt.Errorf("string literal required for type []byte") + } + s, err := strconv.Unquote(lit.Value) + if err != nil { + return nil, err + } + return []byte(s), nil + } + return nil, fmt.Errorf("expected []byte") +} diff --git a/zstd/fuzz_test.go b/zstd/fuzz_test.go index fbe1104997..22257e2fc7 100644 --- a/zstd/fuzz_test.go +++ b/zstd/fuzz_test.go @@ -6,22 +6,17 @@ package zstd import ( "bytes" "fmt" - "go/ast" - "go/parser" - "go/token" "io" - "os" rdebug "runtime/debug" - "strconv" "testing" "github.com/klauspost/compress/internal/cpuinfo" - "github.com/klauspost/compress/zip" + "github.com/klauspost/compress/internal/fuzz" ) func FuzzDecodeAll(f *testing.F) { - addBytesFromZip(f, "testdata/fuzz/decode-corpus-raw.zip", true) - addBytesFromZip(f, "testdata/fuzz/decode-corpus-encoded.zip", false) + fuzz.AddFromZip(f, "testdata/fuzz/decode-corpus-raw.zip", true, testing.Short()) + fuzz.AddFromZip(f, "testdata/fuzz/decode-corpus-encoded.zip", false, testing.Short()) decLow, err := NewReader(nil, WithDecoderLowmem(true), WithDecoderConcurrency(2), WithDecoderMaxMemory(20<<20), WithDecoderMaxWindow(1<<20), IgnoreChecksum(true)) if err != nil { f.Fatal(err) @@ -65,8 +60,8 @@ func FuzzDecodeAllNoBMI2(f *testing.F) { } func FuzzDecoder(f *testing.F) { - addBytesFromZip(f, "testdata/fuzz/decode-corpus-raw.zip", true) - addBytesFromZip(f, "testdata/fuzz/decode-corpus-encoded.zip", false) + fuzz.AddFromZip(f, "testdata/fuzz/decode-corpus-raw.zip", true, testing.Short()) + fuzz.AddFromZip(f, "testdata/fuzz/decode-corpus-encoded.zip", false, testing.Short()) decLow, err := NewReader(nil, WithDecoderLowmem(true), WithDecoderConcurrency(2), WithDecoderMaxMemory(20<<20), WithDecoderMaxWindow(1<<20), IgnoreChecksum(true)) if err != nil { f.Fatal(err) @@ -110,14 +105,14 @@ func FuzzDecoder(f *testing.F) { } func FuzzEncoding(f *testing.F) { - addBytesFromZip(f, "testdata/fuzz/encode-corpus-raw.zip", true) - addBytesFromZip(f, "testdata/comp-crashers.zip", true) - addBytesFromZip(f, "testdata/fuzz/encode-corpus-encoded.zip", false) + fuzz.AddFromZip(f, "testdata/fuzz/encode-corpus-raw.zip", true, testing.Short()) + fuzz.AddFromZip(f, "testdata/comp-crashers.zip", true, false) + fuzz.AddFromZip(f, "testdata/fuzz/encode-corpus-encoded.zip", false, testing.Short()) // Fuzzing tweaks: const ( // Test a subset of encoders. startFuzz = SpeedFastest - endFuzz = SpeedBestCompression + endFuzz = SpeedBetterCompression // Also tests with dictionaries... testDicts = true @@ -264,109 +259,3 @@ func FuzzEncoding(f *testing.F) { } }) } - -func addBytesFromZip(f *testing.F, filename string, raw bool) { - file, err := os.Open(filename) - if err != nil { - f.Fatal(err) - } - fi, err := file.Stat() - if err != nil { - f.Fatal(err) - } - zr, err := zip.NewReader(file, fi.Size()) - if err != nil { - f.Fatal(err) - } - for i, file := range zr.File { - if testing.Short() && i%10 != 0 { - continue - } - rc, err := file.Open() - if err != nil { - f.Fatal(err) - } - - b, err := io.ReadAll(rc) - if err != nil { - f.Fatal(err) - } - rc.Close() - raw := raw - if bytes.HasPrefix(b, []byte("go test fuzz")) { - raw = false - } - if raw { - f.Add(b) - continue - } - vals, err := unmarshalCorpusFile(b) - if err != nil { - f.Fatal(err) - } - for _, v := range vals { - f.Add(v) - } - } -} - -// unmarshalCorpusFile decodes corpus bytes into their respective values. -func unmarshalCorpusFile(b []byte) ([][]byte, error) { - if len(b) == 0 { - return nil, fmt.Errorf("cannot unmarshal empty string") - } - lines := bytes.Split(b, []byte("\n")) - if len(lines) < 2 { - return nil, fmt.Errorf("must include version and at least one value") - } - var vals = make([][]byte, 0, len(lines)-1) - for _, line := range lines[1:] { - line = bytes.TrimSpace(line) - if len(line) == 0 { - continue - } - v, err := parseCorpusValue(line) - if err != nil { - return nil, fmt.Errorf("malformed line %q: %v", line, err) - } - vals = append(vals, v) - } - return vals, nil -} - -// parseCorpusValue -func parseCorpusValue(line []byte) ([]byte, error) { - fs := token.NewFileSet() - expr, err := parser.ParseExprFrom(fs, "(test)", line, 0) - if err != nil { - return nil, err - } - call, ok := expr.(*ast.CallExpr) - if !ok { - return nil, fmt.Errorf("expected call expression") - } - if len(call.Args) != 1 { - return nil, fmt.Errorf("expected call expression with 1 argument; got %d", len(call.Args)) - } - arg := call.Args[0] - - if arrayType, ok := call.Fun.(*ast.ArrayType); ok { - if arrayType.Len != nil { - return nil, fmt.Errorf("expected []byte or primitive type") - } - elt, ok := arrayType.Elt.(*ast.Ident) - if !ok || elt.Name != "byte" { - return nil, fmt.Errorf("expected []byte") - } - lit, ok := arg.(*ast.BasicLit) - if !ok || lit.Kind != token.STRING { - return nil, fmt.Errorf("string literal required for type []byte") - } - s, err := strconv.Unquote(lit.Value) - if err != nil { - return nil, err - } - return []byte(s), nil - } - return nil, fmt.Errorf("expected []byte") -}