From 8ad35f65adc71c4bd29e7930385ecd84b183eef2 Mon Sep 17 00:00:00 2001
From: bill-rich <bill.rich@gmail.com>
Date: Thu, 27 Oct 2022 12:10:46 -0700
Subject: [PATCH] Add bytes counter to scans

---
 main.go              | 1 +
 pkg/engine/engine.go | 6 ++++++
 2 files changed, 7 insertions(+)

diff --git a/main.go b/main.go
index e8742c9baf36..9f25fa250188 100644
--- a/main.go
+++ b/main.go
@@ -288,6 +288,7 @@ func run(state overseer.State) {
 		}
 	}
 	logrus.Debugf("scanned %d chunks", e.ChunksScanned())
+	logrus.Debugf("scanned %d bytes", e.BytesScanned())
 
 	if *printAvgDetectorTime {
 		printAverageDetectorTime(e)
diff --git a/pkg/engine/engine.go b/pkg/engine/engine.go
index 75d813ebc697..ea59b555ee5c 100644
--- a/pkg/engine/engine.go
+++ b/pkg/engine/engine.go
@@ -27,6 +27,7 @@ type Engine struct {
 	decoders        []decoders.Decoder
 	detectors       map[bool][]detectors.Detector
 	chunksScanned   uint64
+	bytesScanned    uint64
 	detectorAvgTime sync.Map
 	sourcesWg       sync.WaitGroup
 	workersWg       sync.WaitGroup
@@ -140,6 +141,10 @@ func (e *Engine) ChunksScanned() uint64 {
 	return e.chunksScanned
 }
 
+func (e *Engine) BytesScanned() uint64 {
+	return e.bytesScanned
+}
+
 func (e *Engine) DetectorAvgTime() map[string][]time.Duration {
 	avgTime := map[string][]time.Duration{}
 	e.detectorAvgTime.Range(func(k, v interface{}) bool {
@@ -163,6 +168,7 @@ func (e *Engine) DetectorAvgTime() map[string][]time.Duration {
 func (e *Engine) detectorWorker(ctx context.Context) {
 	for originalChunk := range e.chunks {
 		for chunk := range sources.Chunker(originalChunk) {
+			atomic.AddUint64(&e.bytesScanned, uint64(len(chunk.Data)))
 			fragStart, mdLine := fragmentFirstLine(chunk)
 			for _, decoder := range e.decoders {
 				var decoderType detectorspb.DecoderType