From db6103577931ea9661ef2f10ff7dfc1d741179f1 Mon Sep 17 00:00:00 2001 From: bill-rich Date: Tue, 6 Dec 2022 09:30:36 -0800 Subject: [PATCH 1/2] Change chunker test source --- pkg/sources/chunker.go | 2 +- pkg/sources/chunker_test.go | 9 ++------- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/pkg/sources/chunker.go b/pkg/sources/chunker.go index ad8d88acaf08..24cdb9b252b8 100644 --- a/pkg/sources/chunker.go +++ b/pkg/sources/chunker.go @@ -37,10 +37,10 @@ func Chunker(originalChunk *Chunk) chan *Chunk { } peekData, _ := reader.Peek(PeekSize) chunk.Data = append(chunkBytes[:n], peekData...) - chunkChan <- &chunk if errors.Is(err, io.EOF) { break } + chunkChan <- &chunk } }() return chunkChan diff --git a/pkg/sources/chunker_test.go b/pkg/sources/chunker_test.go index 0a3755820cad..d25963ec79e7 100644 --- a/pkg/sources/chunker_test.go +++ b/pkg/sources/chunker_test.go @@ -5,19 +5,14 @@ import ( "bytes" "errors" "io" - "net/http" "testing" diskbufferreader "github.com/bill-rich/disk-buffer-reader" ) func TestChunker(t *testing.T) { - resp, err := http.Get("https://raw.githubusercontent.com/bill-rich/bad-secrets/master/FifteenMB.gz") - if err != nil { - t.Fatal(err) - } - defer resp.Body.Close() - reReader, err := diskbufferreader.New(resp.Body) + byteBuffer := bytes.NewBuffer(make([]byte, ChunkSize*9)) + reReader, err := diskbufferreader.New(byteBuffer) if err != nil { t.Fatal(err) } From 01634edeb3c3d4edef267d82c7a11523137bc914 Mon Sep 17 00:00:00 2001 From: bill-rich Date: Tue, 6 Dec 2022 12:40:06 -0800 Subject: [PATCH 2/2] Emit chunk if the size isn't 0 --- pkg/sources/chunker.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pkg/sources/chunker.go b/pkg/sources/chunker.go index 24cdb9b252b8..e9cbf2914489 100644 --- a/pkg/sources/chunker.go +++ b/pkg/sources/chunker.go @@ -37,10 +37,12 @@ func Chunker(originalChunk *Chunk) chan *Chunk { } peekData, _ := reader.Peek(PeekSize) chunk.Data = append(chunkBytes[:n], peekData...) + if n > 0 { + chunkChan <- &chunk + } if errors.Is(err, io.EOF) { break } - chunkChan <- &chunk } }() return chunkChan