Skip to content

Commit

Permalink
otel/runtime: remove dependency [stagingdeploy]
Browse files Browse the repository at this point in the history 
Signed-off-by: Ignacio Hagopian <jsign.uy@gmail.com>
  • Loading branch information
@jsign
jsign committed Oct 12, 2022
1 parent fc76414 commit 96fe62a
Show file tree
Hide file tree
Showing 4 changed files with 134 additions and 24 deletions.
36 changes: 18 additions & 18 deletions docker/observability/grafana/provisioning/dashboards/validator-dashboard.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -120,7 +120,7 @@
"uid": "P1809F7CD0C75ACF3"
},
"exemplar": true,
"expr": "sum by (chain_id) (tableland_wallettracker_txn_confirmation_attempts{service_name=\"tableland:api\"})",
"expr": "sum by (chain_id) (tableland_wallettracker_txn_confirmation_attempts)",
"format": "time_series",
"instant": false,
"interval": "",
Expand Down Expand Up @@ -356,7 +356,7 @@
"uid": "P1809F7CD0C75ACF3"
},
"exemplar": true,
"expr": "sum by (chain_id) (tableland_wallettracker_balance_wei{service_name=\"tableland:api\"})/1000000000",
"expr": "sum by (chain_id) (tableland_wallettracker_balance_wei)/1000000000",
"format": "time_series",
"instant": false,
"interval": "",
Expand Down Expand Up @@ -427,7 +427,7 @@
"uid": "P1809F7CD0C75ACF3"
},
"exemplar": true,
"expr": "sum by (chain_id)(tableland_wallettracker_eth_client_unhealthy{service_name=\"tableland:api\"}) < BOOL 1",
"expr": "sum by (chain_id)(tableland_wallettracker_eth_client_unhealthy) < BOOL 1",
"interval": "",
"legendFormat": "ChainID: {{chain_id}}",
"refId": "A"
Expand Down Expand Up @@ -524,7 +524,7 @@
"uid": "P1809F7CD0C75ACF3"
},
"exemplar": true,
"expr": "sum by (chain_id) (tableland_wallettracker_pending_txns{service_name=\"tableland:api\"})",
"expr": "sum by (chain_id) (tableland_wallettracker_pending_txns)",
"interval": "",
"legendFormat": "ChainID {{chain_id}}",
"refId": "A"
Expand Down Expand Up @@ -768,7 +768,7 @@
"uid": "P1809F7CD0C75ACF3"
},
"exemplar": true,
"expr": "sum by (chain_id) (tableland_eventfeed_height{service_name=\"tableland:api\"})",
"expr": "sum by (chain_id) (tableland_eventfeed_height)",
"interval": "",
"legendFormat": "ChainID: {{chain_id}}",
"refId": "A"
Expand Down Expand Up @@ -856,7 +856,7 @@
"uid": "P1809F7CD0C75ACF3"
},
"exemplar": true,
"expr": "sum by (chain_id) (tableland_eventprocessor_last_processed_height{service_name=\"tableland:api\"})",
"expr": "sum by (chain_id) (tableland_eventprocessor_last_processed_height)",
"interval": "",
"legendFormat": "ChainID: {{chain_id}}",
"refId": "A"
Expand Down Expand Up @@ -948,7 +948,7 @@
"uid": "P1809F7CD0C75ACF3"
},
"exemplar": true,
"expr": "sum by (chain_id, name) (rate(tableland_eventfeed_eventypes_count{service_name=\"tableland:api\"}[5m]))",
"expr": "sum by (chain_id, name) (rate(tableland_eventfeed_eventypes_count[5m]))",
"interval": "",
"legendFormat": "ChainID: {{chain_id}} Event: {{name}}",
"refId": "A"
Expand Down Expand Up @@ -1040,7 +1040,7 @@
"uid": "P1809F7CD0C75ACF3"
},
"exemplar": true,
"expr": "sum by (chaind_id, eventtype) (rate(tableland_eventprocessor_event_execution_count{service_name=\"tableland:api\"}[5m]))",
"expr": "sum by (chaind_id, eventtype) (rate(tableland_eventprocessor_event_execution_count[5m]))",
"interval": "",
"legendFormat": "{{chain_id}} {{eventtype}}",
"refId": "A"
Expand Down Expand Up @@ -1085,7 +1085,7 @@
"uid": "P1809F7CD0C75ACF3"
},
"exemplar": false,
"expr": "sum(rate(tableland_eventprocessor_txn_execution_latency_bucket{service_name=\"tableland:api\"}[$__interval])) by (le)",
"expr": "sum(rate(tableland_eventprocessor_txn_execution_latency_bucket[$__interval])) by (le)",
"format": "heatmap",
"instant": false,
"interval": "",
Expand Down Expand Up @@ -1145,7 +1145,7 @@
"uid": "P1809F7CD0C75ACF3"
},
"exemplar": true,
"expr": "sum(rate(tableland_eventprocessor_block_execution_latency_bucket{service_name=\"tableland:api\"}[$__interval])) by (le)",
"expr": "sum(rate(tableland_eventprocessor_block_execution_latency_bucket[$__interval])) by (le)",
"format": "heatmap",
"interval": "",
"legendFormat": "{{le}}",
Expand Down Expand Up @@ -1231,7 +1231,7 @@
"uid": "P1809F7CD0C75ACF3"
},
"exemplar": true,
"expr": "sum by (chain_id) (tableland_eventprocessor_execution_round{service_name=\"tableland:api\"})",
"expr": "sum by (chain_id) (tableland_eventprocessor_execution_round)",
"interval": "",
"legendFormat": "ChainID {{chain_id}}",
"refId": "A"
Expand Down Expand Up @@ -1364,7 +1364,7 @@
},
"editorMode": "code",
"exemplar": true,
"expr": "sum by (http_status_code, http_server_name) (\n rate(http_server_request_count{service_name=\"tableland:api\"}[10m])\n)",
"expr": "sum by (http_status_code, http_server_name) (\n rate(http_server_request_count[10m])\n)",
"interval": "",
"legendFormat": "{{http_status_code}}-{{http_server_name}}",
"range": true,
Expand Down Expand Up @@ -1472,7 +1472,7 @@
},
"editorMode": "code",
"exemplar": true,
"expr": "sum by (http_server_name) (\n rate(http_server_request_count{service_name=\"tableland:api\", http_server_name!=\"rpc\"}[5m])\n)",
"expr": "sum by (http_server_name) (\n rate(http_server_request_count{http_server_name!=\"rpc\"}[5m])\n)",
"interval": "",
"legendFormat": "__auto",
"range": true,
Expand Down Expand Up @@ -1763,7 +1763,7 @@
"uid": "P1809F7CD0C75ACF3"
},
"exemplar": true,
"expr": "sum by (method) (\n rate(tableland_mesa_call_count{service_name=\"tableland:api\"}[5m])\n)\n",
"expr": "sum by (method) (\n rate(tableland_mesa_call_count[5m])\n)\n",
"interval": "",
"legendFormat": "{{method}}",
"refId": "A"
Expand Down Expand Up @@ -1856,7 +1856,7 @@
"uid": "P1809F7CD0C75ACF3"
},
"exemplar": false,
"expr": "histogram_quantile(0.95, sum(rate(tableland_mesa_call_latency_bucket{service_name=\"tableland:api\"}[10m])) by (le, method))",
"expr": "histogram_quantile(0.95, sum(rate(tableland_mesa_call_latency_bucket[10m])) by (le, method))",
"format": "time_series",
"instant": false,
"interval": "",
Expand Down Expand Up @@ -1973,7 +1973,7 @@
"uid": "P1809F7CD0C75ACF3"
},
"exemplar": true,
"expr": "sum by (method) (\n rate(tableland_sqlstore_call_count{service_name=\"tableland:api\"}[5m])\n)",
"expr": "sum by (method) (\n rate(tableland_sqlstore_call_count[5m])\n)",
"interval": "",
"legendFormat": "{{method}}",
"refId": "A"
Expand Down Expand Up @@ -2066,7 +2066,7 @@
"uid": "P1809F7CD0C75ACF3"
},
"exemplar": false,
"expr": "histogram_quantile(0.95, sum(rate(tableland_sqlstore_call_latency_bucket{service_name=\"tableland:api\"}[5m])) by (le, method))",
"expr": "histogram_quantile(0.95, sum(rate(tableland_sqlstore_call_latency_bucket[5m])) by (le, method))",
"format": "heatmap",
"instant": false,
"interval": "",
Expand Down Expand Up @@ -2384,7 +2384,7 @@
"uid": "P1809F7CD0C75ACF3"
},
"exemplar": true,
"expr": "sum by(chain_id) (tableland_wallettracker_nonce{service_name=\"tableland:api\"})",
"expr": "sum by(chain_id) (tableland_wallettracker_nonce)",
"interval": "",
"legendFormat": "ChainID {{chain_id}}",
"refId": "A"
Expand Down
1 change: 0 additions & 1 deletion go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,6 @@ require (
github.com/tablelandnetwork/sqlparser v0.0.0-20220923130758-1b39431a2fea
github.com/textileio/cli v1.0.2
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.32.0
go.opentelemetry.io/contrib/instrumentation/runtime v0.36.2
go.opentelemetry.io/otel v1.10.0
go.opentelemetry.io/otel/exporters/prometheus v0.32.2
go.opentelemetry.io/otel/metric v0.32.2
Expand Down
2 changes: 0 additions & 2 deletions go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1397,8 +1397,6 @@ go.opentelemetry.io/contrib v0.20.0 h1:ubFQUn0VCZ0gPwIoJfBJVpeBlyRMxu8Mm/huKWYd9
go.opentelemetry.io/contrib v0.20.0/go.mod h1:G/EtFaa6qaN7+LxqfIAT3GiZa7Wv5DTBUzl5H4LY0Kc=
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.20.0/go.mod h1:oVGt1LRbBOBq1A5BQLlUg9UaU/54aiHw8cgjV3aWZ/E=
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.28.0/go.mod h1:vEhqr0m4eTc+DWxfsXoXue2GBgV2uUwVznkGIHW/e5w=
go.opentelemetry.io/contrib/instrumentation/runtime v0.36.2 h1:9fs2UD7cpFKdyqmdFk0bi+ZlQ90ngvsmzd39lRzQCIw=
go.opentelemetry.io/contrib/instrumentation/runtime v0.36.2/go.mod h1:N78CNMwF4rAFTJmSc/Ncw7fLVMPMnTi4Yrld+2bQSzg=
go.opentelemetry.io/otel v0.20.0/go.mod h1:Y3ugLH2oa81t5QO+Lty+zXf8zC9L26ax4Nzoxm/dooo=
go.opentelemetry.io/otel v1.3.0/go.mod h1:PWIKzi6JCp7sM0k9yZ43VX+T345uNbAkDKwHVjb2PTs=
go.opentelemetry.io/otel v1.6.0/go.mod h1:bfJD2DZVw0LBxghOTlgnlI0CV3hLDu9XF/QKOUXMTQQ=
Expand Down
119 changes: 116 additions & 3 deletions pkg/metrics/instrumenting.go
View file
Original file line number Diff line number Diff line change
@@ -1,15 +1,20 @@
package metrics

import (
"context"
"fmt"
"net/http"
"runtime"
"time"

"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promhttp"
otelprom "go.opentelemetry.io/otel/exporters/prometheus"
"go.opentelemetry.io/otel/metric/unit"

"go.opentelemetry.io/contrib/instrumentation/runtime"
"go.opentelemetry.io/otel/metric/global"
"go.opentelemetry.io/otel/metric/instrument"
"go.opentelemetry.io/otel/metric/instrument/asyncint64"
"go.opentelemetry.io/otel/sdk/metric"
"go.opentelemetry.io/otel/sdk/metric/aggregation"
"go.opentelemetry.io/otel/sdk/metric/view"
Expand Down Expand Up @@ -41,9 +46,117 @@ func SetupInstrumentation(prometheusAddr string) error {
_ = http.ListenAndServe(prometheusAddr, nil)
}()

if err := runtime.Start(); err != nil {
return fmt.Errorf("starting Go runtime metrics: %s", err)
if err := startCollectingRuntimeMetrics(); err != nil {
return fmt.Errorf("start collecting Go runtime metrics: %s", err)
}

if err := startCollectingMemoryMetrics(); err != nil {
return fmt.Errorf("start collecting Go memory metrics: %s", err)
}

return nil
}

func startCollectingRuntimeMetrics() error {
meter := global.MeterProvider().Meter("runtime")

uptime, err := meter.AsyncInt64().Counter(
"runtime.uptime",
instrument.WithUnit(unit.Milliseconds),
instrument.WithDescription("Milliseconds since application was initialized"),
)
if err != nil {
return fmt.Errorf("creating runtime uptime: %s", err)
}

goroutines, err := meter.AsyncInt64().Gauge(
"process.runtime.go.goroutines",
instrument.WithDescription("Number of goroutines that currently exist"),
)
if err != nil {
return fmt.Errorf("creating runtime goroutines: %s", err)
}

cgoCalls, err := meter.AsyncInt64().Counter(
"process.runtime.go.cgo.calls",
instrument.WithDescription("Number of cgo calls made by the current process"),
)
if err != nil {
return fmt.Errorf("creating runtime cgo calls: %s", err)
}

startTime := time.Now()
err = meter.RegisterCallback(
[]instrument.Asynchronous{
uptime,
goroutines,
cgoCalls,
},
func(ctx context.Context) {
uptime.Observe(ctx, time.Since(startTime).Milliseconds())
goroutines.Observe(ctx, int64(runtime.NumGoroutine()))
cgoCalls.Observe(ctx, runtime.NumCgoCall())
},
)
if err != nil {
return fmt.Errorf("registering callback: %s", err)
}

return nil
}

func startCollectingMemoryMetrics() error {
var (
err error

heapInuse asyncint64.UpDownCounter
liveObjects asyncint64.UpDownCounter
gcCount asyncint64.Counter

lastMemStats time.Time
memStats runtime.MemStats
)

meter := global.MeterProvider().Meter("runtime")
if heapInuse, err = meter.AsyncInt64().Gauge(
"process.runtime.go.mem.heap_inuse",
instrument.WithUnit(unit.Bytes),
instrument.WithDescription("Bytes in in-use spans"),
); err != nil {
return fmt.Errorf("creating heap in use: %s", err)
}

if liveObjects, err = meter.AsyncInt64().Gauge(
"process.runtime.go.mem.live_objects",
instrument.WithDescription("Number of live objects is the number of cumulative Mallocs - Frees"),
); err != nil {
return fmt.Errorf("creating heap live objects: %s", err)
}

if gcCount, err = meter.AsyncInt64().Counter(
"process.runtime.go.gc.count",
instrument.WithDescription("Number of completed garbage collection cycles"),
); err != nil {
return fmt.Errorf("creating gc count: %s", err)
}

if err := meter.RegisterCallback(
[]instrument.Asynchronous{
heapInuse,
liveObjects,
gcCount,
}, func(ctx context.Context) {
now := time.Now()
if now.Sub(lastMemStats) >= time.Second*15 {
runtime.ReadMemStats(&memStats)
lastMemStats = now
}

heapInuse.Observe(ctx, int64(memStats.HeapInuse))
liveObjects.Observe(ctx, int64(memStats.Mallocs-memStats.Frees))
gcCount.Observe(ctx, int64(memStats.NumGC))
}); err != nil {
return fmt.Errorf("registering callback: %s", err)
}
return nil
}

0 comments on commit 96fe62a

Please sign in to comment.