Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #628 from mwain/prometheus
Add Prometheus Metrics Exporter
- Loading branch information
Showing
5 changed files
with
348 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,182 @@ | ||
| package httpserver | ||
|
|
||
| import ( | ||
| "net/http" | ||
| "strconv" | ||
|
|
||
| "github.com/prometheus/client_golang/prometheus" | ||
|
|
||
| "github.com/linkedin/Burrow/core/protocol" | ||
|
|
||
| "github.com/prometheus/client_golang/prometheus/promauto" | ||
| "github.com/prometheus/client_golang/prometheus/promhttp" | ||
| ) | ||
|
|
||
| var ( | ||
| consumerTotalLagGauge = promauto.NewGaugeVec( | ||
| prometheus.GaugeOpts{ | ||
| Name: "burrow_kafka_consumer_lag_total", | ||
| Help: "The sum of all partition current lag values for the group", | ||
| }, | ||
| []string{"cluster", "consumer_group"}, | ||
| ) | ||
|
|
||
| consumerStatusGauge = promauto.NewGaugeVec( | ||
| prometheus.GaugeOpts{ | ||
| Name: "burrow_kafka_consumer_status", | ||
| Help: "The status of the consumer group. It is calculated from the highest status for the individual partitions. Statuses are an index list from NOTFOUND, OK, WARN, ERR, STOP, STALL, REWIND", | ||
| }, | ||
| []string{"cluster", "consumer_group"}, | ||
| ) | ||
|
|
||
| consumerPartitionCurrentOffset = promauto.NewGaugeVec( | ||
| prometheus.GaugeOpts{ | ||
| Name: "burrow_kafka_consumer_current_offset", | ||
| Help: "Latest offset that Burrow is storing for this partition", | ||
| }, | ||
| []string{"cluster", "consumer_group", "topic", "partition"}, | ||
| ) | ||
|
|
||
| consumerPartitionLagGauge = promauto.NewGaugeVec( | ||
| prometheus.GaugeOpts{ | ||
| Name: "burrow_kafka_consumer_partition_lag", | ||
| Help: "Number of messages the consumer group is behind by for a partition as reported by Burrow", | ||
| }, | ||
| []string{"cluster", "consumer_group", "topic", "partition"}, | ||
| ) | ||
|
|
||
| topicPartitionOffsetGauge = promauto.NewGaugeVec( | ||
| prometheus.GaugeOpts{ | ||
| Name: "burrow_kafka_topic_partition_offset", | ||
| Help: "Latest offset the topic that Burrow is storing for this partition", | ||
| }, | ||
| []string{"cluster", "topic", "partition"}, | ||
| ) | ||
| ) | ||
|
|
||
| func (hc *Coordinator) handlePrometheusMetrics() http.HandlerFunc { | ||
| promHandler := promhttp.Handler() | ||
|
|
||
| return http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) { | ||
| for _, cluster := range listClusters(hc.App) { | ||
| for _, consumer := range listConsumers(hc.App, cluster) { | ||
| consumerStatus := getFullConsumerStatus(hc.App, cluster, consumer) | ||
|
|
||
| if consumerStatus == nil || | ||
| consumerStatus.Status == protocol.StatusNotFound || | ||
| consumerStatus.Complete < 1.0 { | ||
| continue | ||
| } | ||
|
|
||
| labels := map[string]string{ | ||
| "cluster": cluster, | ||
| "consumer_group": consumer, | ||
| } | ||
|
|
||
| consumerTotalLagGauge.With(labels).Set(float64(consumerStatus.TotalLag)) | ||
| consumerStatusGauge.With(labels).Set(float64(consumerStatus.Status)) | ||
|
|
||
| for _, partition := range consumerStatus.Partitions { | ||
| if partition.Complete < 1.0 { | ||
| continue | ||
| } | ||
|
|
||
| labels := map[string]string{ | ||
| "cluster": cluster, | ||
| "consumer_group": consumer, | ||
| "topic": partition.Topic, | ||
| "partition": strconv.FormatInt(int64(partition.Partition), 10), | ||
| } | ||
|
|
||
| consumerPartitionCurrentOffset.With(labels).Set(float64(partition.End.Offset)) | ||
| consumerPartitionLagGauge.With(labels).Set(float64(partition.CurrentLag)) | ||
| } | ||
| } | ||
|
|
||
| // Topics | ||
| for _, topic := range listTopics(hc.App, cluster) { | ||
| for partitionNumber, offset := range getTopicDetail(hc.App, cluster, topic) { | ||
| topicPartitionOffsetGauge.With(map[string]string{ | ||
| "cluster": cluster, | ||
| "topic": topic, | ||
| "partition": strconv.FormatInt(int64(partitionNumber), 10), | ||
| }).Set(float64(offset)) | ||
| } | ||
| } | ||
| } | ||
|
|
||
| promHandler.ServeHTTP(resp, req) | ||
| }) | ||
| } | ||
|
|
||
| func listClusters(app *protocol.ApplicationContext) []string { | ||
| request := &protocol.StorageRequest{ | ||
| RequestType: protocol.StorageFetchClusters, | ||
| Reply: make(chan interface{}), | ||
| } | ||
| app.StorageChannel <- request | ||
| response := <-request.Reply | ||
| if response == nil { | ||
| return []string{} | ||
| } | ||
|
|
||
| return response.([]string) | ||
| } | ||
|
|
||
| func listConsumers(app *protocol.ApplicationContext, cluster string) []string { | ||
| request := &protocol.StorageRequest{ | ||
| RequestType: protocol.StorageFetchConsumers, | ||
| Cluster: cluster, | ||
| Reply: make(chan interface{}), | ||
| } | ||
| app.StorageChannel <- request | ||
| response := <-request.Reply | ||
| if response == nil { | ||
| return []string{} | ||
| } | ||
|
|
||
| return response.([]string) | ||
| } | ||
|
|
||
| func getFullConsumerStatus(app *protocol.ApplicationContext, cluster, consumer string) *protocol.ConsumerGroupStatus { | ||
| request := &protocol.EvaluatorRequest{ | ||
| Cluster: cluster, | ||
| Group: consumer, | ||
| ShowAll: true, | ||
| Reply: make(chan *protocol.ConsumerGroupStatus), | ||
| } | ||
| app.EvaluatorChannel <- request | ||
| response := <-request.Reply | ||
| return response | ||
| } | ||
|
|
||
| func listTopics(app *protocol.ApplicationContext, cluster string) []string { | ||
| request := &protocol.StorageRequest{ | ||
| RequestType: protocol.StorageFetchTopics, | ||
| Cluster: cluster, | ||
| Reply: make(chan interface{}), | ||
| } | ||
| app.StorageChannel <- request | ||
| response := <-request.Reply | ||
| if response == nil { | ||
| return []string{} | ||
| } | ||
|
|
||
| return response.([]string) | ||
| } | ||
|
|
||
| func getTopicDetail(app *protocol.ApplicationContext, cluster, topic string) []int64 { | ||
| request := &protocol.StorageRequest{ | ||
| RequestType: protocol.StorageFetchTopic, | ||
| Cluster: cluster, | ||
| Topic: topic, | ||
| Reply: make(chan interface{}), | ||
| } | ||
| app.StorageChannel <- request | ||
| response := <-request.Reply | ||
| if response == nil { | ||
| return []int64{} | ||
| } | ||
|
|
||
| return response.([]int64) | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,156 @@ | ||
| package httpserver | ||
|
|
||
| import ( | ||
| "net/http" | ||
| "net/http/httptest" | ||
| "testing" | ||
|
|
||
| "github.com/stretchr/testify/assert" | ||
|
|
||
| "github.com/linkedin/Burrow/core/protocol" | ||
| ) | ||
|
|
||
| func TestHttpServer_handlePrometheusMetrics(t *testing.T) { | ||
| coordinator := fixtureConfiguredCoordinator() | ||
|
|
||
| // Respond to the expected storage requests | ||
| go func() { | ||
| request := <-coordinator.App.StorageChannel | ||
| assert.Equalf(t, protocol.StorageFetchClusters, request.RequestType, "Expected request of type StorageFetchClusters, not %v", request.RequestType) | ||
| request.Reply <- []string{"testcluster"} | ||
| close(request.Reply) | ||
|
|
||
| // List of consumers | ||
| request = <-coordinator.App.StorageChannel | ||
| assert.Equalf(t, protocol.StorageFetchConsumers, request.RequestType, "Expected request of type StorageFetchConsumers, not %v", request.RequestType) | ||
| assert.Equalf(t, "testcluster", request.Cluster, "Expected request Cluster to be testcluster, not %v", request.Cluster) | ||
| request.Reply <- []string{"testgroup", "testgroup2"} | ||
| close(request.Reply) | ||
|
|
||
| // List of topics | ||
| request = <-coordinator.App.StorageChannel | ||
| assert.Equalf(t, protocol.StorageFetchTopics, request.RequestType, "Expected request of type StorageFetchTopics, not %v", request.RequestType) | ||
| assert.Equalf(t, "testcluster", request.Cluster, "Expected request Cluster to be testcluster, not %v", request.Cluster) | ||
| request.Reply <- []string{"testtopic", "testtopic1"} | ||
| close(request.Reply) | ||
|
|
||
| // Topic details | ||
| request = <-coordinator.App.StorageChannel | ||
| assert.Equalf(t, protocol.StorageFetchTopic, request.RequestType, "Expected request of type StorageFetchTopic, not %v", request.RequestType) | ||
| assert.Equalf(t, "testcluster", request.Cluster, "Expected request Cluster to be testcluster, not %v", request.Cluster) | ||
| assert.Equalf(t, "testtopic", request.Topic, "Expected request Topic to be testtopic, not %v", request.Topic) | ||
| request.Reply <- []int64{6556, 5566} | ||
| close(request.Reply) | ||
|
|
||
| request = <-coordinator.App.StorageChannel | ||
| assert.Equalf(t, protocol.StorageFetchTopic, request.RequestType, "Expected request of type StorageFetchTopic, not %v", request.RequestType) | ||
| assert.Equalf(t, "testcluster", request.Cluster, "Expected request Cluster to be testcluster, not %v", request.Cluster) | ||
| assert.Equalf(t, "testtopic1", request.Topic, "Expected request Topic to be testtopic, not %v", request.Topic) | ||
| request.Reply <- []int64{54} | ||
| close(request.Reply) | ||
| }() | ||
|
|
||
| // Respond to the expected evaluator requests | ||
| go func() { | ||
| // testgroup happy paths | ||
| request := <-coordinator.App.EvaluatorChannel | ||
| assert.Equalf(t, "testcluster", request.Cluster, "Expected request Cluster to be testcluster, not %v", request.Cluster) | ||
| assert.Equalf(t, "testgroup", request.Group, "Expected request Group to be testgroup, not %v", request.Group) | ||
| assert.True(t, request.ShowAll, "Expected request ShowAll to be True") | ||
| response := &protocol.ConsumerGroupStatus{ | ||
| Cluster: request.Cluster, | ||
| Group: request.Group, | ||
| Status: protocol.StatusOK, | ||
| Complete: 1.0, | ||
| Partitions: []*protocol.PartitionStatus{ | ||
| { | ||
| Topic: "testtopic", | ||
| Partition: 0, | ||
| Status: protocol.StatusOK, | ||
| CurrentLag: 100, | ||
| Complete: 1.0, | ||
| End: &protocol.ConsumerOffset{ | ||
| Offset: 22663, | ||
| }, | ||
| }, | ||
| { | ||
| Topic: "testtopic", | ||
| Partition: 1, | ||
| Status: protocol.StatusOK, | ||
| CurrentLag: 10, | ||
| Complete: 1.0, | ||
| End: &protocol.ConsumerOffset{ | ||
| Offset: 2488, | ||
| }, | ||
| }, | ||
| { | ||
| Topic: "testtopic1", | ||
| Partition: 0, | ||
| Status: protocol.StatusOK, | ||
| CurrentLag: 50, | ||
| Complete: 1.0, | ||
| End: &protocol.ConsumerOffset{ | ||
| Offset: 99888, | ||
| }, | ||
| }, | ||
| { | ||
| Topic: "incomplete", | ||
| Partition: 0, | ||
| Status: protocol.StatusOK, | ||
| CurrentLag: 0, | ||
| Complete: 0.2, | ||
| End: &protocol.ConsumerOffset{ | ||
| Offset: 5335, | ||
| }, | ||
| }, | ||
| }, | ||
| TotalPartitions: 2134, | ||
| Maxlag: &protocol.PartitionStatus{}, | ||
| TotalLag: 2345, | ||
| } | ||
| request.Reply <- response | ||
| close(request.Reply) | ||
|
|
||
| // testgroup2 not found | ||
| request = <-coordinator.App.EvaluatorChannel | ||
| assert.Equalf(t, "testcluster", request.Cluster, "Expected request Cluster to be testcluster, not %v", request.Cluster) | ||
| assert.Equalf(t, "testgroup2", request.Group, "Expected request Group to be testgroup, not %v", request.Group) | ||
| assert.True(t, request.ShowAll, "Expected request ShowAll to be True") | ||
| response = &protocol.ConsumerGroupStatus{ | ||
| Cluster: request.Cluster, | ||
| Group: request.Group, | ||
| Status: protocol.StatusNotFound, | ||
| } | ||
| request.Reply <- response | ||
| close(request.Reply) | ||
| }() | ||
|
|
||
| // Set up a request | ||
| req, err := http.NewRequest("GET", "/metrics", nil) | ||
| assert.NoError(t, err, "Expected request setup to return no error") | ||
|
|
||
| // Call the handler via httprouter | ||
| rr := httptest.NewRecorder() | ||
| coordinator.router.ServeHTTP(rr, req) | ||
|
|
||
| assert.Equalf(t, http.StatusOK, rr.Code, "Expected response code to be 200, not %v", rr.Code) | ||
|
|
||
| promExp := rr.Body.String() | ||
| assert.Contains(t, promExp, `burrow_kafka_consumer_status{cluster="testcluster",consumer_group="testgroup"} 1`) | ||
| assert.Contains(t, promExp, `burrow_kafka_consumer_lag_total{cluster="testcluster",consumer_group="testgroup"} 2345`) | ||
|
|
||
| assert.Contains(t, promExp, `burrow_kafka_consumer_partition_lag{cluster="testcluster",consumer_group="testgroup",partition="0",topic="testtopic"} 100`) | ||
| assert.Contains(t, promExp, `burrow_kafka_consumer_partition_lag{cluster="testcluster",consumer_group="testgroup",partition="1",topic="testtopic"} 10`) | ||
| assert.Contains(t, promExp, `burrow_kafka_consumer_partition_lag{cluster="testcluster",consumer_group="testgroup",partition="0",topic="testtopic1"} 50`) | ||
|
|
||
| assert.Contains(t, promExp, `burrow_kafka_consumer_current_offset{cluster="testcluster",consumer_group="testgroup",partition="0",topic="testtopic"} 22663`) | ||
| assert.Contains(t, promExp, `burrow_kafka_consumer_current_offset{cluster="testcluster",consumer_group="testgroup",partition="1",topic="testtopic"} 2488`) | ||
| assert.Contains(t, promExp, `burrow_kafka_consumer_current_offset{cluster="testcluster",consumer_group="testgroup",partition="0",topic="testtopic1"} 99888`) | ||
|
|
||
| assert.Contains(t, promExp, `burrow_kafka_topic_partition_offset{cluster="testcluster",partition="0",topic="testtopic"} 6556`) | ||
| assert.Contains(t, promExp, `burrow_kafka_topic_partition_offset{cluster="testcluster",partition="1",topic="testtopic"} 5566`) | ||
| assert.Contains(t, promExp, `burrow_kafka_topic_partition_offset{cluster="testcluster",partition="0",topic="testtopic1"} 54`) | ||
|
|
||
| assert.NotContains(t, promExp, `burrow_kafka_consumer_partition_lag{cluster="testcluster",consumer_group="testgroup",partition="0",topic="incomplete"} 0`) | ||
| assert.NotContains(t, promExp, "testgroup2") | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters