Skip to content

Commit

Permalink
s2: Reduce ReadFrom temporary allocations (#949)
Browse files Browse the repository at this point in the history 
Only functional change is to add minimum 1MB between entries (which was enforced when saving anyway).

Probably close to a worst case (and probably noisy), but overall looks good:

```
benchmark                                                old ns/op     new ns/op     delta
BenchmarkReadFromRandom/default-c1-4k-win-32             4157177       2604339       -37.35%
BenchmarkReadFromRandom/default-c2-32                    788147        679660        -13.76%
BenchmarkReadFromRandom/best-c2-4k-win-32                215513740     207505150     -3.72%
BenchmarkReadFromRandom/default-c2-4k-win-32             8712437       8354241       -4.11%
BenchmarkReadFromRandom/better-c1-32                     2861139       1862028       -34.92%
BenchmarkReadFromRandom/none-c1-32                       2331903       1430642       -38.65%
BenchmarkReadFromRandom/best-c1-4k-win-32                278422200     267546000     -3.91%
BenchmarkReadFromRandom/none-c2-pad-min-32               772803        672716        -12.95%
BenchmarkReadFromRandom/best-c2-pad-min-32               5460325       5095500       -6.68%
BenchmarkReadFromRandom/none-c2-4M-win-pad-min-32        1121548       651889        -41.88%
BenchmarkReadFromRandom/better-c1-4M-win-32              3135030       1813419       -42.16%
BenchmarkReadFromRandom/none-c2-4k-win-32                4854943       4876356       +0.44%
BenchmarkReadFromRandom/default-c1-4M-win-pad-min-32     2526561       1457227       -42.32%
BenchmarkReadFromRandom/better-c2-4k-win-32              11868536      11714897      -1.29%
BenchmarkReadFromRandom/best-c1-4M-win-pad-min-32        15489805      11971380      -22.71%
BenchmarkReadFromRandom/none-c1-pad-min-32               2416111       1421856       -41.15%
BenchmarkReadFromRandom/none-c2-4M-win-32                1101574       668233        -39.34%
BenchmarkReadFromRandom/default-c2-4M-win-32             1125451       669567        -40.51%
BenchmarkReadFromRandom/better-c1-pad-min-32             3601453       1885376       -47.65%
BenchmarkReadFromRandom/best-c1-4M-win-32                15391320      13574288      -11.81%
BenchmarkReadFromRandom/default-c1-32                    2910080       1468122       -49.55%
BenchmarkReadFromRandom/better-c1-4k-win-pad-min-32      8912611       6817072       -23.51%
BenchmarkReadFromRandom/none-c2-32                       772356        670370        -13.20%
BenchmarkReadFromRandom/default-c1-4M-win-32             2588579       1450875       -43.95%
BenchmarkReadFromRandom/default-c2-4k-win-pad-min-32     8659309       8312865       -4.00%
BenchmarkReadFromRandom/better-c2-4k-win-pad-min-32      11869082      11664866      -1.72%
BenchmarkReadFromRandom/better-c1-4k-win-32              8838607       6716940       -24.00%
BenchmarkReadFromRandom/better-c2-4M-win-32              1191929       835844        -29.87%
BenchmarkReadFromRandom/best-c1-32                       14526139      12391446      -14.70%
BenchmarkReadFromRandom/best-c2-4k-win-pad-min-32        195756200     202187120     +3.29%
BenchmarkReadFromRandom/none-c1-4k-win-pad-min-32        2738898       833709        -69.56%
BenchmarkReadFromRandom/better-c2-pad-min-32             1004892       884365        -11.99%
BenchmarkReadFromRandom/none-c2-4k-win-pad-min-32        4823234       4856289       +0.69%
BenchmarkReadFromRandom/best-c2-4M-win-32                5898056       5408690       -8.30%
BenchmarkReadFromRandom/best-c2-4M-win-pad-min-32        5669392       5523631       -2.57%
BenchmarkReadFromRandom/default-c1-4k-win-pad-min-32     4570917       2567954       -43.82%
BenchmarkReadFromRandom/better-c2-4M-win-pad-min-32      1231687       847167        -31.22%
BenchmarkReadFromRandom/default-c2-4M-win-pad-min-32     1096798       678716        -38.12%
BenchmarkReadFromRandom/none-c1-4k-win-32                3316746       843895        -74.56%
BenchmarkReadFromRandom/none-c1-4M-win-pad-min-32        2640620       1429627       -45.86%
BenchmarkReadFromRandom/best-c1-4k-win-pad-min-32        278722100     273288575     -1.95%
BenchmarkReadFromRandom/best-c1-pad-min-32               14499535      12347208      -14.84%
BenchmarkReadFromRandom/none-c1-4M-win-32                2652251       1453083       -45.21%
BenchmarkReadFromRandom/default-c1-pad-min-32            2791364       1474998       -47.16%
BenchmarkReadFromRandom/default-c2-pad-min-32            824566        707436        -14.21%
BenchmarkReadFromRandom/best-c2-32                       5362292       5104630       -4.81%
BenchmarkReadFromRandom/better-c1-4M-win-pad-min-32      3919478       1822777       -53.49%
BenchmarkReadFromRandom/better-c2-32                     969154        880528        -9.14%

benchmark                                                old MB/s     new MB/s     speedup
BenchmarkReadFromRandom/default-c1-4k-win-32             2017.86      3221.01      1.60x
BenchmarkReadFromRandom/default-c2-32                    10643.46     12342.35     1.16x
BenchmarkReadFromRandom/best-c2-4k-win-32                38.92        40.43        1.04x
BenchmarkReadFromRandom/default-c2-4k-win-32             962.83       1004.11      1.04x
BenchmarkReadFromRandom/better-c1-32                     2931.91      4505.09      1.54x
BenchmarkReadFromRandom/none-c1-32                       3597.32      5863.53      1.63x
BenchmarkReadFromRandom/best-c1-4k-win-32                30.13        31.35        1.04x
BenchmarkReadFromRandom/none-c2-pad-min-32               10854.78     12469.75     1.15x
BenchmarkReadFromRandom/best-c2-pad-min-32               1536.28      1646.28      1.07x
BenchmarkReadFromRandom/none-c2-4M-win-pad-min-32        7479.49      12868.15     1.72x
BenchmarkReadFromRandom/better-c1-4M-win-32              2675.77      4625.85      1.73x
BenchmarkReadFromRandom/none-c2-4k-win-32                1727.85      1720.26      1.00x
BenchmarkReadFromRandom/default-c1-4M-win-pad-min-32     3320.17      5756.55      1.73x
BenchmarkReadFromRandom/better-c2-4k-win-32              706.79       716.06       1.01x
BenchmarkReadFromRandom/best-c1-4M-win-pad-min-32        541.56       700.72       1.29x
BenchmarkReadFromRandom/none-c1-pad-min-32               3471.95      5899.76      1.70x
BenchmarkReadFromRandom/none-c2-4M-win-32                7615.11      12553.42     1.65x
BenchmarkReadFromRandom/default-c2-4M-win-32             7453.55      12528.41     1.68x
BenchmarkReadFromRandom/better-c1-pad-min-32             2329.23      4449.30      1.91x
BenchmarkReadFromRandom/best-c1-4M-win-32                545.02       617.98       1.13x
BenchmarkReadFromRandom/default-c1-32                    2882.60      5713.84      1.98x
BenchmarkReadFromRandom/better-c1-4k-win-pad-min-32      941.21       1230.53      1.31x
BenchmarkReadFromRandom/none-c2-32                       10861.07     12513.39     1.15x
BenchmarkReadFromRandom/default-c1-4M-win-32             3240.62      5781.76      1.78x
BenchmarkReadFromRandom/default-c2-4k-win-pad-min-32     968.74       1009.11      1.04x
BenchmarkReadFromRandom/better-c2-4k-win-pad-min-32      706.76       719.13       1.02x
BenchmarkReadFromRandom/better-c1-4k-win-32              949.09       1248.87      1.32x
BenchmarkReadFromRandom/better-c2-4M-win-32              7037.84      10036.09     1.43x
BenchmarkReadFromRandom/best-c1-32                       577.48       676.97       1.17x
BenchmarkReadFromRandom/best-c2-4k-win-pad-min-32        42.85        41.49        0.97x
BenchmarkReadFromRandom/none-c1-4k-win-pad-min-32        3062.77      10061.79     3.29x
BenchmarkReadFromRandom/better-c2-pad-min-32             8347.77      9485.46      1.14x
BenchmarkReadFromRandom/none-c2-4k-win-pad-min-32        1739.21      1727.37      0.99x
BenchmarkReadFromRandom/best-c2-4M-win-32                1422.27      1550.95      1.09x
BenchmarkReadFromRandom/best-c2-4M-win-pad-min-32        1479.63      1518.68      1.03x
BenchmarkReadFromRandom/default-c1-4k-win-pad-min-32     1835.21      3266.65      1.78x
BenchmarkReadFromRandom/better-c2-4M-win-pad-min-32      6810.66      9901.96      1.45x
BenchmarkReadFromRandom/default-c2-4M-win-pad-min-32     7648.27      12359.53     1.62x
BenchmarkReadFromRandom/none-c1-4k-win-32                2529.17      9940.34      3.93x
BenchmarkReadFromRandom/none-c1-4M-win-pad-min-32        3176.76      5867.69      1.85x
BenchmarkReadFromRandom/best-c1-4k-win-pad-min-32        30.10        30.70        1.02x
BenchmarkReadFromRandom/best-c1-pad-min-32               578.54       679.39       1.17x
BenchmarkReadFromRandom/none-c1-4M-win-32                3162.83      5772.97      1.83x
BenchmarkReadFromRandom/default-c1-pad-min-32            3005.20      5687.20      1.89x
BenchmarkReadFromRandom/default-c2-pad-min-32            10173.36     11857.76     1.17x
BenchmarkReadFromRandom/best-c2-32                       1564.37      1643.33      1.05x
BenchmarkReadFromRandom/better-c1-4M-win-pad-min-32      2140.24      4602.10      2.15x
BenchmarkReadFromRandom/better-c2-32                     8655.60      9526.79      1.10x

benchmark                                                old allocs     new allocs     delta
BenchmarkReadFromRandom/default-c1-4k-win-32             8196           6145           -25.02%
BenchmarkReadFromRandom/default-c2-32                    59             57             -3.39%
BenchmarkReadFromRandom/best-c2-4k-win-32                14356          14347          -0.06%
BenchmarkReadFromRandom/default-c2-4k-win-32             14344          14341          -0.02%
BenchmarkReadFromRandom/better-c1-32                     35             25             -28.57%
BenchmarkReadFromRandom/none-c1-32                       35             25             -28.57%
BenchmarkReadFromRandom/best-c1-4k-win-32                8196           6147           -25.00%
BenchmarkReadFromRandom/none-c2-pad-min-32               59             57             -3.39%
BenchmarkReadFromRandom/best-c2-pad-min-32               58             57             -1.72%
BenchmarkReadFromRandom/none-c2-4M-win-pad-min-32        18             15             -16.67%
BenchmarkReadFromRandom/better-c1-4M-win-32              11             7              -36.36%
BenchmarkReadFromRandom/none-c2-4k-win-32                14345          14343          -0.01%
BenchmarkReadFromRandom/default-c1-4M-win-pad-min-32     11             7              -36.36%
BenchmarkReadFromRandom/better-c2-4k-win-32              14343          14343          +0.00%
BenchmarkReadFromRandom/best-c1-4M-win-pad-min-32        11             7              -36.36%
BenchmarkReadFromRandom/none-c1-pad-min-32               35             25             -28.57%
BenchmarkReadFromRandom/none-c2-4M-win-32                18             15             -16.67%
BenchmarkReadFromRandom/default-c2-4M-win-32             18             15             -16.67%
BenchmarkReadFromRandom/better-c1-pad-min-32             35             25             -28.57%
BenchmarkReadFromRandom/best-c1-4M-win-32                11             7              -36.36%
BenchmarkReadFromRandom/default-c1-32                    35             25             -28.57%
BenchmarkReadFromRandom/better-c1-4k-win-pad-min-32      8196           6145           -25.02%
BenchmarkReadFromRandom/none-c2-32                       59             57             -3.39%
BenchmarkReadFromRandom/default-c1-4M-win-32             11             7              -36.36%
BenchmarkReadFromRandom/default-c2-4k-win-pad-min-32     14345          14343          -0.01%
BenchmarkReadFromRandom/better-c2-4k-win-pad-min-32      14344          14346          +0.01%
BenchmarkReadFromRandom/better-c1-4k-win-32              8196           6145           -25.02%
BenchmarkReadFromRandom/better-c2-4M-win-32              18             15             -16.67%
BenchmarkReadFromRandom/best-c1-32                       35             25             -28.57%
BenchmarkReadFromRandom/best-c2-4k-win-pad-min-32        14347          14343          -0.03%
BenchmarkReadFromRandom/none-c1-4k-win-pad-min-32        8196           6145           -25.02%
BenchmarkReadFromRandom/better-c2-pad-min-32             59             57             -3.39%
BenchmarkReadFromRandom/none-c2-4k-win-pad-min-32        14345          14343          -0.01%
BenchmarkReadFromRandom/best-c2-4M-win-32                17             15             -11.76%
BenchmarkReadFromRandom/best-c2-4M-win-pad-min-32        17             15             -11.76%
BenchmarkReadFromRandom/default-c1-4k-win-pad-min-32     8196           6145           -25.02%
BenchmarkReadFromRandom/better-c2-4M-win-pad-min-32      18             15             -16.67%
BenchmarkReadFromRandom/default-c2-4M-win-pad-min-32     18             15             -16.67%
BenchmarkReadFromRandom/none-c1-4k-win-32                8196           6145           -25.02%
BenchmarkReadFromRandom/none-c1-4M-win-pad-min-32        11             7              -36.36%
BenchmarkReadFromRandom/best-c1-4k-win-pad-min-32        8196           6148           -24.99%
BenchmarkReadFromRandom/best-c1-pad-min-32               35             25             -28.57%
BenchmarkReadFromRandom/none-c1-4M-win-32                11             7              -36.36%
BenchmarkReadFromRandom/default-c1-pad-min-32            35             25             -28.57%
BenchmarkReadFromRandom/default-c2-pad-min-32            59             57             -3.39%
BenchmarkReadFromRandom/best-c2-32                       58             57             -1.72%
BenchmarkReadFromRandom/better-c1-4M-win-pad-min-32      11             7              -36.36%
BenchmarkReadFromRandom/better-c2-32                     59             57             -3.39%

benchmark                                                old bytes     new bytes     delta
BenchmarkReadFromRandom/default-c1-4k-win-32             10119115      148076        -98.54%
BenchmarkReadFromRandom/default-c2-32                    1454676       4899          -99.66%
BenchmarkReadFromRandom/best-c2-4k-win-32                631225        625546        -0.90%
BenchmarkReadFromRandom/default-c2-4k-win-32             630732        625959        -0.76%
BenchmarkReadFromRandom/better-c1-32                     9514904       2797          -99.97%
BenchmarkReadFromRandom/none-c1-32                       9516943       2440          -99.97%
BenchmarkReadFromRandom/best-c1-4k-win-32                10119906      150564        -98.51%
BenchmarkReadFromRandom/none-c2-pad-min-32               1500410       6171          -99.59%
BenchmarkReadFromRandom/best-c2-pad-min-32               1173566       25894         -97.79%
BenchmarkReadFromRandom/none-c2-4M-win-pad-min-32        6423795       5415          -99.92%
BenchmarkReadFromRandom/better-c1-4M-win-32              12610237      6582          -99.95%
BenchmarkReadFromRandom/none-c2-4k-win-32                632307        626486        -0.92%
BenchmarkReadFromRandom/default-c1-4M-win-pad-min-32     12610296      5410          -99.96%
BenchmarkReadFromRandom/better-c2-4k-win-32              630839        626113        -0.75%
BenchmarkReadFromRandom/best-c1-4M-win-pad-min-32        12610476      49191         -99.61%
BenchmarkReadFromRandom/none-c1-pad-min-32               9514740       2479          -99.97%
BenchmarkReadFromRandom/none-c2-4M-win-32                6545265       13213         -99.80%
BenchmarkReadFromRandom/default-c2-4M-win-32             6481737       11355         -99.82%
BenchmarkReadFromRandom/better-c1-pad-min-32             9514861       2763          -99.97%
BenchmarkReadFromRandom/best-c1-4M-win-32                12610426      57138         -99.55%
BenchmarkReadFromRandom/default-c1-32                    9517523       2540          -99.97%
BenchmarkReadFromRandom/better-c1-4k-win-pad-min-32      10118890      148124        -98.54%
BenchmarkReadFromRandom/none-c2-32                       1530722       5478          -99.64%
BenchmarkReadFromRandom/default-c1-4M-win-32             12610210      5454          -99.96%
BenchmarkReadFromRandom/default-c2-4k-win-pad-min-32     630895        626457        -0.70%
BenchmarkReadFromRandom/better-c2-4k-win-pad-min-32      630794        626720        -0.65%
BenchmarkReadFromRandom/better-c1-4k-win-32              10118922      148161        -98.54%
BenchmarkReadFromRandom/better-c2-4M-win-32              6598822       13798         -99.79%
BenchmarkReadFromRandom/best-c1-32                       9516160       12594         -99.87%
BenchmarkReadFromRandom/best-c2-4k-win-pad-min-32        631572        628459        -0.49%
BenchmarkReadFromRandom/none-c1-4k-win-pad-min-32        10118789      148120        -98.54%
BenchmarkReadFromRandom/better-c2-pad-min-32             1489742       6937          -99.53%
BenchmarkReadFromRandom/none-c2-4k-win-pad-min-32        631988        626774        -0.83%
BenchmarkReadFromRandom/best-c2-4M-win-32                4734174       79313         -98.32%
BenchmarkReadFromRandom/best-c2-4M-win-pad-min-32        4859587       78584         -98.38%
BenchmarkReadFromRandom/default-c1-4k-win-pad-min-32     10118817      148094        -98.54%
BenchmarkReadFromRandom/better-c2-4M-win-pad-min-32      6497809       13671         -99.79%
BenchmarkReadFromRandom/default-c2-4M-win-pad-min-32     6482743       13380         -99.79%
BenchmarkReadFromRandom/none-c1-4k-win-32                10118669      148214        -98.54%
BenchmarkReadFromRandom/none-c1-4M-win-pad-min-32        12610006      5378          -99.96%
BenchmarkReadFromRandom/best-c1-4k-win-pad-min-32        10119906      150652        -98.51%
BenchmarkReadFromRandom/best-c1-pad-min-32               9515185       12174         -99.87%
BenchmarkReadFromRandom/none-c1-4M-win-32                12610010      5304          -99.96%
BenchmarkReadFromRandom/default-c1-pad-min-32            9514765       2516          -99.97%
BenchmarkReadFromRandom/default-c2-pad-min-32            1524283       4986          -99.67%
BenchmarkReadFromRandom/best-c2-32                       1238837       3118          -99.75%
BenchmarkReadFromRandom/better-c1-4M-win-pad-min-32      12636434      7013          -99.94%
BenchmarkReadFromRandom/better-c2-32                     1486984       6114          -99.59%
```

`magicChunkSnappyBytes` and `magicChunkBytes` not included.
  • Loading branch information
@klauspost
klauspost committed Apr 12, 2024
1 parent c0ff47e commit 8bd3916
Show file tree
Hide file tree
Showing 4 changed files with 59 additions and 13 deletions.
10 changes: 8 additions & 2 deletions s2/index.go
View file
Expand Up @@ -17,6 +17,8 @@ const (
S2IndexHeader = "s2idx\x00"
S2IndexTrailer = "\x00xdi2s"
maxIndexEntries = 1 << 16
// If distance is less than this, we do not add the entry.
minIndexDist = 1 << 20
)

// Index represents an S2/Snappy index.
Expand Down Expand Up @@ -72,6 +74,10 @@ func (i *Index) add(compressedOffset, uncompressedOffset int64) error {
if latest.compressedOffset > compressedOffset {
return fmt.Errorf("internal error: Earlier compressed received (%d > %d)", latest.uncompressedOffset, uncompressedOffset)
}
if latest.uncompressedOffset+minIndexDist > uncompressedOffset {
// Only add entry if distance is large enough.
return nil
}
}
i.info = append(i.info, struct {
compressedOffset int64
Expand Down Expand Up @@ -122,7 +128,7 @@ func (i *Index) Find(offset int64) (compressedOff, uncompressedOff int64, err er

// reduce to stay below maxIndexEntries
func (i *Index) reduce() {
if len(i.info) < maxIndexEntries && i.estBlockUncomp >= 1<<20 {
if len(i.info) < maxIndexEntries && i.estBlockUncomp >= minIndexDist {
return
}

Expand All @@ -132,7 +138,7 @@ func (i *Index) reduce() {
j := 0

// Each block should be at least 1MB, but don't reduce below 1000 entries.
for i.estBlockUncomp*(int64(removeN)+1) < 1<<20 && len(i.info)/(removeN+1) > 1000 {
for i.estBlockUncomp*(int64(removeN)+1) < minIndexDist && len(i.info)/(removeN+1) > 1000 {
removeN++
}
for idx := 0; idx < len(src); idx++ {
Expand Down
6 changes: 5 additions & 1 deletion s2/s2.go
View file
Expand Up @@ -109,7 +109,11 @@ const (
chunkTypeStreamIdentifier = 0xff
)

var crcTable = crc32.MakeTable(crc32.Castagnoli)
var (
crcTable = crc32.MakeTable(crc32.Castagnoli)
magicChunkSnappyBytes = []byte(magicChunkSnappy) // Can be passed to functions where it escapes.
magicChunkBytes = []byte(magicChunk) // Can be passed to functions where it escapes.
)

// crc implements the checksum specified in section 3 of
// https://github.com/google/snappy/blob/master/framing_format.txt
Expand Down
26 changes: 16 additions & 10 deletions s2/writer.go
View file
Expand Up @@ -239,6 +239,9 @@ func (w *Writer) ReadFrom(r io.Reader) (n int64, err error) {
}
}
if n2 == 0 {
if cap(inbuf) >= w.obufLen {
w.buffers.Put(inbuf)
}
break
}
n += int64(n2)
Expand Down Expand Up @@ -314,9 +317,9 @@ func (w *Writer) AddSkippableBlock(id uint8, data []byte) (err error) {
hWriter := make(chan result)
w.output <- hWriter
if w.snappy {
hWriter <- result{startOffset: w.uncompWritten, b: []byte(magicChunkSnappy)}
hWriter <- result{startOffset: w.uncompWritten, b: magicChunkSnappyBytes}
} else {
hWriter <- result{startOffset: w.uncompWritten, b: []byte(magicChunk)}
hWriter <- result{startOffset: w.uncompWritten, b: magicChunkBytes}
}
}

Expand Down Expand Up @@ -370,9 +373,9 @@ func (w *Writer) EncodeBuffer(buf []byte) (err error) {
hWriter := make(chan result)
w.output <- hWriter
if w.snappy {
hWriter <- result{startOffset: w.uncompWritten, b: []byte(magicChunkSnappy)}
hWriter <- result{startOffset: w.uncompWritten, b: magicChunkSnappyBytes}
} else {
hWriter <- result{startOffset: w.uncompWritten, b: []byte(magicChunk)}
hWriter <- result{startOffset: w.uncompWritten, b: magicChunkBytes}
}
}

Expand Down Expand Up @@ -478,9 +481,9 @@ func (w *Writer) write(p []byte) (nRet int, errRet error) {
hWriter := make(chan result)
w.output <- hWriter
if w.snappy {
hWriter <- result{startOffset: w.uncompWritten, b: []byte(magicChunkSnappy)}
hWriter <- result{startOffset: w.uncompWritten, b: magicChunkSnappyBytes}
} else {
hWriter <- result{startOffset: w.uncompWritten, b: []byte(magicChunk)}
hWriter <- result{startOffset: w.uncompWritten, b: magicChunkBytes}
}
}

Expand Down Expand Up @@ -560,6 +563,9 @@ func (w *Writer) writeFull(inbuf []byte) (errRet error) {

if w.concurrency == 1 {
_, err := w.writeSync(inbuf[obufHeaderLen:])
if cap(inbuf) >= w.obufLen {
w.buffers.Put(inbuf)
}
return err
}

Expand All @@ -569,9 +575,9 @@ func (w *Writer) writeFull(inbuf []byte) (errRet error) {
hWriter := make(chan result)
w.output <- hWriter
if w.snappy {
hWriter <- result{startOffset: w.uncompWritten, b: []byte(magicChunkSnappy)}
hWriter <- result{startOffset: w.uncompWritten, b: magicChunkSnappyBytes}
} else {
hWriter <- result{startOffset: w.uncompWritten, b: []byte(magicChunk)}
hWriter <- result{startOffset: w.uncompWritten, b: magicChunkBytes}
}
}

Expand Down Expand Up @@ -637,9 +643,9 @@ func (w *Writer) writeSync(p []byte) (nRet int, errRet error) {
var n int
var err error
if w.snappy {
n, err = w.writer.Write([]byte(magicChunkSnappy))
n, err = w.writer.Write(magicChunkSnappyBytes)
} else {
n, err = w.writer.Write([]byte(magicChunk))
n, err = w.writer.Write(magicChunkBytes)
}
if err != nil {
return 0, w.err(err)
Expand Down
30 changes: 30 additions & 0 deletions s2/writer_test.go
View file
Expand Up @@ -603,6 +603,36 @@ func BenchmarkWriterRandom(b *testing.B) {
}
}

func BenchmarkReadFromRandom(b *testing.B) {
rng := rand.New(rand.NewSource(1))
// Make max window so we never get matches.
data := make([]byte, 8<<20)
for i := range data {
data[i] = uint8(rng.Intn(256))
}

for name, opts := range testOptions(b) {
w := NewWriter(io.Discard, opts...)
in := bytes.NewReader(data)
w.ReadFrom(in)
b.Run(name, func(b *testing.B) {
b.ResetTimer()
b.ReportAllocs()
b.SetBytes(int64(len(data)))
for i := 0; i < b.N; i++ {
in.Reset(data)
_, err := w.ReadFrom(in)
if err != nil {
b.Fatal(err)
}
}
// Flush output
w.Flush()
})
w.Close()
}
}

func BenchmarkIndexFind(b *testing.B) {
fatalErr := func(t testing.TB, err error) {
if err != nil {
Expand Down

0 comments on commit 8bd3916

Please sign in to comment.