-
Notifications
You must be signed in to change notification settings - Fork 128
Run perf tests
Ondrej Fabry edited this page Mar 2, 2020
·
8 revisions
This document provides info how to run performance tests for VPP Agent along with metrics collection by Prometheus and sample dashboard in Grafana.
Before we start, we create docker network for agent and prometheus+grafana so they can access each other based on their container names. This makes the cofiguration files simpler and we don't need to lookup IP addresses.
# create docker network
docker network create --attachable perftest
# inspect created network
docker inspect perftestNow we can run containers with --network perftest to start them in network we created instead of default docker bridge.
How to run prometheus:
Guide from Prometheus docs https://prometheus.io/docs/prometheus/latest/installation/ and Docker docs: https://docs.docker.com/config/thirdparty/prometheus/
# replace `$HOME/prometheus.yml` point to the config below
docker run --name prometheus -p 9090:9090 \
-v $HOME/prometheus.yml:/etc/prometheus/prometheus.yml \
--network perftest \
prom/prometheusHere is sample config file for Prometheus. It collects the metrics exported by VPP Agent:
prometheus.yml
global:
scrape_interval: 15s
scrape_configs:
- job_name: 'prometheus'
static_configs:
- targets: ['localhost:9090']
- job_name: 'agent'
scrape_interval: 1s
static_configs:
- targets: ['agent1:9191']
# this one is only for perf testing
- job_name: 'grpc-client'
static_configs:
- targets: ['agent1:9194']
# this one is optional, collects metrics from grafana
- job_name: 'grafana'
static_configs:
- targets: ['grafana:3000']
How to run grafana:
Guide from Grafana docs: https://grafana.com/docs/installation/docker/
# without persistence
docker run --name grafana -p 3000:3000 --network perftest grafana/grafana
# to persist data in volumes
docker run --name grafana \
--network perftest \
--publish 3000:3000 \
--volume /var/lib/grafana:/var/lib/grafana \
--env "GF_SECURITY_ADMIN_PASSWORD=secret" \
grafana/grafanaAnd here is exported dashboard for Grafana. It can be imported using Grafana GUI.
grafana-dashboard.json
{
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": "-- Grafana --",
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"type": "dashboard"
},
{
"datasource": "-- Grafana --",
"enable": true,
"hide": true,
"iconColor": "rgba(255, 96, 96, 1)",
"limit": 100,
"name": "agent_version",
"showIn": 0,
"tags": [],
"type": "tags"
}
]
},
"description": "dashboard for VPP Agent metrics",
"editable": true,
"gnetId": null,
"graphTooltip": 1,
"id": 9,
"iteration": 1582723248842,
"links": [],
"panels": [
{
"aliasColors": {
"ligato_build_info{build_date=\"Mon Feb 24 13:52:28 UTC 2020\",built_by=\"root@agent1.agent.ligato.io (go1.13.7 linux/amd64)\",instance=\"agent1:9191\",job=\"agent\",revision=\"66f83a4@fixes\",version=\"v3.1.0-alpha-15-g66f83a4f4-dirty\"}": "light-green",
"vppagent_perf_test_client_settings{instance=\"agent1:9094\",job=\"grpc-client\",num_clients=\"1\",num_per_req=\"1\",num_tunnels=\"40000\",start_time=\"Feb 24 15:57:58\"}": "light-blue"
},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": null,
"decimals": null,
"fill": 2,
"fillGradient": 0,
"gridPos": {
"h": 7,
"w": 12,
"x": 0,
"y": 0
},
"hiddenSeries": false,
"id": 2,
"interval": "",
"legend": {
"alignAsTable": false,
"avg": false,
"current": false,
"hideEmpty": false,
"hideZero": false,
"max": false,
"min": false,
"rightSide": false,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 1,
"nullPointMode": "null",
"options": {
"dataLinks": []
},
"percentage": false,
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": true,
"steppedLine": true,
"targets": [
{
"expr": "up{instance=\"$agent\"}",
"format": "time_series",
"hide": true,
"instant": false,
"intervalFactor": 1,
"legendFormat": "up",
"refId": "A"
},
{
"expr": "ligato_build_info{instance=\"$agent\"}",
"format": "time_series",
"instant": false,
"intervalFactor": 1,
"legendFormat": "",
"refId": "B"
},
{
"expr": "ligato_perf_test_client_settings",
"refId": "C"
},
{
"expr": "vppagent_perf_test_client_settings",
"legendFormat": "",
"refId": "D"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "Agent state",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "individual"
},
"transparent": true,
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"decimals": 0,
"format": "none",
"label": "",
"logBase": 1,
"max": null,
"min": "0",
"show": false
},
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": false
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"cacheTimeout": null,
"columns": [
{
"text": "Max",
"value": "max"
}
],
"datasource": null,
"fontSize": "90%",
"gridPos": {
"h": 7,
"w": 6,
"x": 12,
"y": 0
},
"id": 23,
"links": [],
"options": {},
"pageSize": null,
"pluginVersion": "6.6.1",
"showHeader": true,
"sort": {
"col": 1,
"desc": true
},
"styles": [
{
"alias": "Time",
"align": "",
"dateFormat": "YYYY-MM-DD HH:mm:ss",
"link": false,
"pattern": "Metric",
"type": "string"
},
{
"alias": "Total",
"align": "center",
"colorMode": null,
"colors": [
"rgba(245, 54, 54, 0.9)",
"rgba(237, 129, 40, 0.89)",
"rgba(50, 172, 45, 0.97)"
],
"dateFormat": "YYYY-MM-DD HH:mm:ss",
"decimals": null,
"mappingType": 1,
"pattern": "Max",
"thresholds": [],
"type": "number",
"unit": "short"
}
],
"targets": [
{
"expr": "sum(grpc_server_handled_total{job=\"agent\"}) by (grpc_service, grpc_method)",
"format": "time_series",
"instant": false,
"intervalFactor": 1,
"legendFormat": "{{grpc_service}} / {{grpc_method}} ",
"refId": "B"
},
{
"expr": "sum(grpc_client_handled_total{}) by (grpc_service, grpc_method)",
"format": "time_series",
"hide": true,
"instant": false,
"intervalFactor": 1,
"legendFormat": "",
"refId": "A"
}
],
"timeFrom": null,
"timeShift": null,
"title": "grpc requests",
"transform": "timeseries_aggregations",
"type": "table"
},
{
"cacheTimeout": null,
"columns": [
{
"text": "Max",
"value": "max"
}
],
"datasource": null,
"fontSize": "90%",
"gridPos": {
"h": 7,
"w": 6,
"x": 18,
"y": 0
},
"id": 24,
"links": [],
"options": {},
"pageSize": null,
"pluginVersion": "6.6.1",
"showHeader": true,
"sort": {
"col": 1,
"desc": true
},
"styles": [
{
"alias": "VPP binapi message",
"align": "",
"dateFormat": "YYYY-MM-DD HH:mm:ss",
"link": false,
"pattern": "Metric",
"preserveFormat": false,
"type": "string"
},
{
"alias": "Total",
"align": "left",
"colorMode": null,
"colors": [
"rgba(245, 54, 54, 0.9)",
"rgba(237, 129, 40, 0.89)",
"rgba(50, 172, 45, 0.97)"
],
"dateFormat": "YYYY-MM-DD HH:mm:ss",
"decimals": null,
"mappingType": 1,
"pattern": "Max",
"rangeMaps": [
{
"from": "sent",
"text": "x",
"to": "sent"
}
],
"sanitize": false,
"thresholds": [
""
],
"type": "number",
"unit": "short"
}
],
"targets": [
{
"expr": "sum(ligato_govppmux_requests_sent{job=\"agent\"}) by (message)",
"format": "time_series",
"hide": false,
"instant": false,
"intervalFactor": 1,
"legendFormat": "{{grpc_method}} ({{grpc_service}})",
"refId": "B"
},
{
"expr": "sum(grpc_client_handled_total{}) by (grpc_service, grpc_method)",
"format": "time_series",
"hide": true,
"instant": false,
"intervalFactor": 1,
"legendFormat": "",
"refId": "A"
},
{
"expr": "sum(ligato_govppmux_requests_total{job=\"agent\"}) by (message)",
"format": "time_series",
"hide": false,
"instant": false,
"intervalFactor": 1,
"legendFormat": "requests SENT - {{message}}",
"refId": "C"
},
{
"expr": "sum(ligato_govppmux_replies_received_total{job=\"agent\"}) by (message)",
"format": "time_series",
"hide": false,
"instant": false,
"intervalFactor": 1,
"legendFormat": "replies RECEIVED - {{message}}",
"refId": "D"
}
],
"timeFrom": null,
"timeShift": null,
"title": "govpp requests",
"transform": "timeseries_aggregations",
"type": "table"
},
{
"cacheTimeout": null,
"colorBackground": false,
"colorPostfix": false,
"colorPrefix": false,
"colorValue": true,
"colors": [
"#d44a3a",
"rgb(145, 28, 29)",
"#299c46"
],
"datasource": null,
"decimals": null,
"format": "none",
"gauge": {
"maxValue": 100,
"minValue": 0,
"show": false,
"thresholdLabels": false,
"thresholdMarkers": true
},
"gridPos": {
"h": 7,
"w": 3,
"x": 0,
"y": 7
},
"id": 17,
"interval": null,
"links": [],
"mappingType": 1,
"mappingTypes": [
{
"name": "value to text",
"value": 1
},
{
"name": "range to text",
"value": 2
}
],
"maxDataPoints": 100,
"nullPointMode": "connected",
"nullText": null,
"options": {},
"postfix": "",
"postfixFontSize": "50%",
"prefix": "state",
"prefixFontSize": "50%",
"rangeMaps": [
{
"from": "",
"text": "",
"to": ""
}
],
"sparkline": {
"fillColor": "rgba(31, 118, 189, 0.18)",
"full": true,
"lineColor": "rgb(31, 120, 193)",
"show": true,
"ymax": 1,
"ymin": 0
},
"tableColumn": "",
"targets": [
{
"expr": "ligato_build_info{instance=\"$agent\"}",
"format": "time_series",
"hide": true,
"instant": false,
"legendFormat": "",
"refId": "A"
},
{
"expr": "up{instance=\"$agent\"}",
"format": "time_series",
"hide": false,
"refId": "B"
}
],
"thresholds": "0,1",
"timeFrom": null,
"timeShift": null,
"title": "Agent state",
"transparent": true,
"type": "singlestat",
"valueFontSize": "80%",
"valueMaps": [
{
"op": "=",
"text": "?",
"value": "null"
},
{
"op": "=",
"text": "DOWN",
"value": "0"
},
{
"op": "=",
"text": "UP",
"value": "1"
}
],
"valueName": "current"
},
{
"aliasColors": {
"alloc bytes": "semi-dark-orange",
"goroutines": "dark-blue"
},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": null,
"fill": 0,
"fillGradient": 0,
"gridPos": {
"h": 7,
"w": 9,
"x": 3,
"y": 7
},
"hiddenSeries": false,
"id": 9,
"legend": {
"alignAsTable": true,
"avg": false,
"current": false,
"hideEmpty": true,
"hideZero": true,
"max": true,
"min": false,
"rightSide": true,
"show": true,
"sort": null,
"sortDesc": null,
"total": false,
"values": true
},
"lines": true,
"linewidth": 2,
"nullPointMode": "null",
"options": {
"dataLinks": []
},
"percentage": false,
"pointradius": 0.5,
"points": false,
"renderer": "flot",
"seriesOverrides": [
{
"alias": "/bytes/",
"fill": 1,
"linewidth": 1,
"steppedLine": true,
"yaxis": 2
}
],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"expr": "go_goroutines{instance=\"$agent\"}",
"legendFormat": "goroutines",
"refId": "A"
},
{
"expr": "go_threads{instance=\"$agent\"}",
"format": "time_series",
"legendFormat": "threads",
"refId": "C"
},
{
"expr": "rate(go_memstats_alloc_bytes_total{instance=\"$agent\"}[$period])",
"format": "time_series",
"hide": false,
"legendFormat": "mem alloc bytes",
"refId": "D"
},
{
"expr": "go_memstats_heap_inuse_bytes{instance=\"$agent\"}",
"format": "time_series",
"hide": true,
"legendFormat": "heap in use bytes",
"refId": "E"
},
{
"expr": "go_memstats_alloc_bytes{instance=\"$agent\"}",
"format": "time_series",
"hide": false,
"legendFormat": "alloc bytes",
"refId": "B"
},
{
"expr": "go_memstats_heap_sys_bytes{instance=\"$agent\"}",
"format": "time_series",
"hide": true,
"legendFormat": "heap_sys bytes",
"refId": "F"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "Go runtime",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"format": "decbytes",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {
"le": "super-light-red",
"le 0.025": "super-light-yellow",
"under 0.002s": "light-green",
"{grpc_method=\"Update\"}": "dark-green",
"{grpc_service=\"ligato.configurator.ConfiguratorService\"}": "semi-dark-orange",
"{le=\"0.025\"}": "super-light-yellow"
},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": null,
"fill": 3,
"fillGradient": 0,
"gridPos": {
"h": 7,
"w": 5,
"x": 12,
"y": 7
},
"hiddenSeries": false,
"id": 21,
"legend": {
"alignAsTable": false,
"avg": false,
"current": false,
"hideEmpty": false,
"hideZero": true,
"max": true,
"min": false,
"rightSide": false,
"show": true,
"total": false,
"values": true
},
"lines": true,
"linewidth": 1,
"maxPerRow": 2,
"nullPointMode": "null",
"options": {
"dataLinks": []
},
"percentage": false,
"pointradius": 2,
"points": false,
"renderer": "flot",
"repeat": null,
"repeatDirection": "h",
"seriesOverrides": [
{
"alias": "/le/",
"fill": 0,
"fillGradient": 0,
"steppedLine": false,
"yaxis": 2
}
],
"spaceLength": 10,
"stack": false,
"steppedLine": true,
"targets": [
{
"expr": "100.0 - (\nsum(rate(grpc_server_handling_seconds_bucket{job=\"agent\",grpc_type=\"unary\",le=\"$grpcreqdur\"}[$period])) by (grpc_service)\n / \nsum(rate(grpc_server_handling_seconds_count{job=\"agent\",grpc_type=\"unary\"}[$period])) by (grpc_service)\n) * 100.0",
"legendFormat": "",
"refId": "A"
},
{
"expr": "sum(rate(grpc_server_handling_seconds_bucket{job=\"agent\",grpc_type=\"unary\",le=\"$grpcreqdur\"}[$period])) by (le)",
"hide": true,
"legendFormat": "",
"refId": "B"
},
{
"expr": "(1.0 - (\nsum(rate(grpc_server_handling_seconds_bucket{job=\"agent\",grpc_type=\"unary\",le=\"$grpcreqdur\"}[$period])) by (job)\n / \nsum(rate(grpc_server_handling_seconds_count{job=\"agent\",grpc_type=\"unary\"}[$period])) by (job)\n)) * sum(rate(grpc_server_handling_seconds_count{job=\"agent\",grpc_type=\"unary\"}[$period])) by (job)",
"hide": false,
"legendFormat": "le",
"refId": "C"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "grpc - req duration > X sec",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "percent",
"label": "",
"logBase": 1,
"max": null,
"min": "0",
"show": true
},
{
"decimals": null,
"format": "short",
"label": "",
"logBase": 1,
"max": null,
"min": "0",
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {
"histogram_quantile(0.5, sum(rate(grpc_server_handling_seconds_bucket[5s])) by (le))": "super-light-green",
"histogram_quantile(0.9, sum(rate(grpc_server_handling_seconds_bucket[5s])) by (le))": "light-orange",
"p0.5": "light-green",
"p0.9": "light-yellow",
"p0.95": "light-orange",
"p0.99": "light-red",
"sum(rate(grpc_server_handled_total[5s]))": "super-light-green"
},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": null,
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 7,
"w": 7,
"x": 17,
"y": 7
},
"hiddenSeries": false,
"id": 15,
"interval": "",
"legend": {
"alignAsTable": false,
"avg": false,
"current": false,
"hideEmpty": false,
"hideZero": false,
"max": true,
"min": false,
"rightSide": false,
"show": true,
"total": false,
"values": true
},
"lines": true,
"linewidth": 1,
"nullPointMode": "null",
"options": {
"dataLinks": []
},
"percentage": false,
"pointradius": 0.5,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.5, sum(rate(grpc_server_handling_seconds_bucket[$period])) by (le))",
"hide": false,
"instant": false,
"legendFormat": "p0.5",
"refId": "D"
},
{
"expr": "histogram_quantile(0.9, sum(rate(grpc_server_handling_seconds_bucket[$period])) by (le))",
"hide": false,
"instant": false,
"legendFormat": "p0.9",
"refId": "A"
},
{
"expr": "histogram_quantile(0.95, sum(rate(grpc_server_handling_seconds_bucket[$period])) by (le))",
"hide": false,
"instant": false,
"legendFormat": "p0.95",
"refId": "B"
},
{
"expr": "histogram_quantile(0.99, sum(rate(grpc_server_handling_seconds_bucket[$period])) by (le))",
"hide": false,
"instant": false,
"legendFormat": "p0.99",
"refId": "C"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "grpc - req durations",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "s",
"label": null,
"logBase": 1,
"max": null,
"min": "0",
"show": true
},
{
"format": "reqps",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": null,
"fill": 0,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 6,
"x": 0,
"y": 14
},
"hiddenSeries": false,
"id": 28,
"legend": {
"alignAsTable": false,
"avg": false,
"current": false,
"max": true,
"min": false,
"rightSide": false,
"show": true,
"sort": "max",
"sortDesc": true,
"total": false,
"values": true
},
"lines": true,
"linewidth": 1,
"nullPointMode": "null",
"options": {
"dataLinks": []
},
"percentage": false,
"pluginVersion": "6.6.1",
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"expr": "sum(ligato_kvscheduler_queue_wait_seconds{instance=\"$agent\"}) by (quantile,txn_type)",
"format": "time_series",
"intervalFactor": 1,
"legendFormat": "{{txn_type}} p{{quantile}}",
"refId": "A"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "kvscheduler - queue wait time",
"tooltip": {
"shared": true,
"sort": 2,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "s",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {
"histogram_quantile(0.5, sum(rate(grpc_server_handling_seconds_bucket[5s])) by (le))": "super-light-green",
"histogram_quantile(0.9, sum(rate(grpc_server_handling_seconds_bucket[5s])) by (le))": "light-orange",
"p0.5": "light-green",
"p0.9": "light-yellow",
"p0.95": "light-orange",
"p0.99": "light-red",
"p1": "semi-dark-red",
"sum(rate(grpc_server_handled_total[5s]))": "super-light-green"
},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": null,
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 6,
"x": 6,
"y": 14
},
"hiddenSeries": false,
"id": 30,
"interval": "",
"legend": {
"alignAsTable": false,
"avg": false,
"current": false,
"hideEmpty": false,
"hideZero": false,
"max": true,
"min": false,
"rightSide": false,
"show": true,
"total": false,
"values": true
},
"lines": true,
"linewidth": 1,
"nullPointMode": "null",
"options": {
"dataLinks": []
},
"percentage": false,
"pointradius": 0.5,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(1.0, sum(rate(ligato_kvscheduler_txn_duration_seconds_bucket[$period])) by (le))",
"hide": false,
"legendFormat": "p1",
"refId": "D"
},
{
"expr": "histogram_quantile(0.99, sum(rate(ligato_kvscheduler_txn_duration_seconds_bucket[$period])) by (le))",
"hide": false,
"legendFormat": "p0.99",
"refId": "C"
},
{
"expr": "histogram_quantile(0.95, sum(rate(ligato_kvscheduler_txn_duration_seconds_bucket[$period])) by (le))",
"legendFormat": "p0.95",
"refId": "E"
},
{
"expr": "histogram_quantile(0.9, sum(rate(ligato_kvscheduler_txn_duration_seconds_bucket[$period])) by (le))",
"legendFormat": "p0.9",
"refId": "A"
},
{
"expr": "histogram_quantile(0.5, sum(rate(ligato_kvscheduler_txn_duration_seconds_bucket[$period])) by (le))",
"legendFormat": "p0.5",
"refId": "B"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "kvscheduler - txn durations",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "s",
"label": null,
"logBase": 2,
"max": null,
"min": null,
"show": true
},
{
"format": "reqps",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": null,
"decimals": null,
"fill": 0,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 14
},
"hiddenSeries": false,
"id": 29,
"legend": {
"alignAsTable": false,
"avg": false,
"current": false,
"hideZero": false,
"max": true,
"min": false,
"rightSide": false,
"show": true,
"sort": null,
"sortDesc": null,
"total": false,
"values": true
},
"lines": true,
"linewidth": 2,
"nullPointMode": "null",
"options": {
"dataLinks": []
},
"percentage": false,
"pluginVersion": "6.6.1",
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [
{
"alias": "/processTransaction .*/",
"fill": 1,
"stack": false
},
{}
],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"expr": "ligato_kvscheduler_txn_process_duration_seconds{instance=\"$agent\",quantile=\"0.99\"}",
"intervalFactor": 1,
"legendFormat": "{{slice}} p99",
"refId": "B"
},
{
"expr": "sum(ligato_kvscheduler_queue_wait_seconds{instance=\"$agent\",quantile=\"0.99\"}) by (quantile)",
"legendFormat": "queueWait p99",
"refId": "A"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "kvscheduler - txn timings",
"tooltip": {
"shared": true,
"sort": 2,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "s",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {
"client - ligato.configurator.ConfiguratorService / grpc_method - OK": "super-light-blue"
},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": null,
"decimals": 1,
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 9,
"w": 12,
"x": 0,
"y": 22
},
"hiddenSeries": false,
"id": 7,
"legend": {
"alignAsTable": true,
"avg": true,
"current": false,
"hideEmpty": false,
"hideZero": true,
"max": true,
"min": true,
"rightSide": false,
"show": true,
"sort": "total",
"sortDesc": true,
"total": false,
"values": true
},
"lines": true,
"linewidth": 1,
"nullPointMode": "null",
"options": {
"dataLinks": []
},
"percentage": false,
"pointradius": 0.5,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"expr": "sum(rate(grpc_server_handled_total{instance=\"$agent\"}[$period])) by (grpc_service, grpc_code)",
"hide": false,
"instant": false,
"interval": "",
"intervalFactor": 1,
"legendFormat": "server - {{grpc_service}} / {{grpc_method}} - {{grpc_code}}",
"refId": "A"
},
{
"expr": "sum(rate(grpc_client_handled_total{job=\"grpc-client\"}[$period])) by (grpc_service, grpc_code)",
"hide": false,
"instant": false,
"interval": "",
"intervalFactor": 1,
"legendFormat": "client - {{grpc_service}} / {{grpc_method}} - {{grpc_code}}",
"refId": "E"
},
{
"expr": "rate(grpc_server_started_total{instance=\"$agent\"}[$period])",
"hide": true,
"legendFormat": "started: {{grpc_method}}",
"refId": "B"
},
{
"expr": "rate(grpc_server_msg_sent_total{instance=\"$agent\"}[$period])",
"hide": true,
"legendFormat": "msg sent: {{grpc_method}}",
"refId": "C"
},
{
"expr": "rate(grpc_server_msg_received_total{instance=\"$agent\"}[$period])",
"hide": true,
"legendFormat": "msg recv: {{grpc_method}}",
"refId": "D"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "grpc server - request rate",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"decimals": null,
"format": "short",
"label": "",
"logBase": 1,
"max": null,
"min": "0",
"show": true
},
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": null,
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 9,
"w": 12,
"x": 12,
"y": 22
},
"hiddenSeries": false,
"id": 3,
"interval": "",
"legend": {
"alignAsTable": true,
"avg": true,
"current": false,
"hideEmpty": false,
"hideZero": true,
"max": true,
"min": true,
"rightSide": false,
"show": true,
"sort": "total",
"sortDesc": true,
"total": false,
"values": true
},
"lines": true,
"linewidth": 1,
"nullPointMode": "null",
"options": {
"dataLinks": []
},
"percentage": false,
"pointradius": 0.5,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": true,
"targets": [
{
"expr": "rate(grpc_server_handling_seconds_sum[$period]) / rate(grpc_server_handling_seconds_count[$period])",
"hide": false,
"instant": false,
"intervalFactor": 1,
"legendFormat": "server - {{grpc_service}} / {{grpc_method}}",
"refId": "C"
},
{
"expr": "rate(grpc_client_handling_seconds_sum[$period]) / rate(grpc_client_handling_seconds_count[$period])",
"hide": false,
"instant": false,
"intervalFactor": 1,
"legendFormat": "client - {{grpc_service}} / {{grpc_method}}",
"refId": "B"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "grpc - request duration",
"tooltip": {
"shared": true,
"sort": 2,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "s",
"label": "",
"logBase": 1,
"max": null,
"min": "0",
"show": true
},
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {
"dropped": "semi-dark-red",
"processed": "semi-dark-blue"
},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": null,
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 31
},
"hiddenSeries": false,
"id": 26,
"legend": {
"avg": true,
"current": false,
"hideEmpty": false,
"hideZero": false,
"max": true,
"min": false,
"show": true,
"total": false,
"values": true
},
"lines": true,
"linewidth": 2,
"nullPointMode": "null",
"options": {
"dataLinks": []
},
"percentage": false,
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"expr": "rate(ligato_kvscheduler_txn_processed[$period])",
"hide": false,
"legendFormat": "processed",
"refId": "A"
},
{
"expr": "rate(ligato_kvscheduler_txn_dropped[$period])",
"legendFormat": "dropped",
"refId": "B"
},
{
"expr": "rate(ligato_kvscheduler_txn_process_duration_seconds_count[$period])",
"hide": true,
"refId": "C"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "kvscheduler - txn rate",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {
"NB Transaction": "light-orange",
"SB Notification": "light-purple"
},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": null,
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 31
},
"hiddenSeries": false,
"id": 31,
"interval": "",
"legend": {
"alignAsTable": false,
"avg": true,
"current": false,
"hideEmpty": false,
"hideZero": true,
"max": true,
"min": true,
"rightSide": false,
"show": true,
"sort": "total",
"sortDesc": true,
"total": false,
"values": true
},
"lines": true,
"linewidth": 1,
"nullPointMode": "null",
"options": {
"dataLinks": []
},
"percentage": false,
"pointradius": 0.5,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": true,
"targets": [
{
"expr": "rate(ligato_kvscheduler_txn_duration_seconds_sum[$period])\n/\nrate(ligato_kvscheduler_txn_duration_seconds_count[$period])",
"hide": false,
"instant": false,
"intervalFactor": 1,
"legendFormat": "{{txn_type}}",
"refId": "C"
},
{
"expr": "rate(ligato_kvscheduler_txn_process_duration_seconds_sum{slice=\"processTransaction\"}[$period])\n/\nrate(ligato_kvscheduler_txn_process_duration_seconds_count{slice=\"processTransaction\"}[$period])",
"hide": true,
"instant": false,
"intervalFactor": 1,
"legendFormat": "",
"refId": "A"
},
{
"expr": "topk(2, ligato_kvscheduler_txn_duration_seconds_sum)",
"hide": true,
"instant": false,
"intervalFactor": 1,
"legendFormat": "",
"refId": "B"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "kvscheduler - txn duration",
"tooltip": {
"shared": true,
"sort": 2,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "s",
"label": "",
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": null,
"decimals": 1,
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 9,
"w": 12,
"x": 0,
"y": 39
},
"hiddenSeries": false,
"id": 10,
"legend": {
"alignAsTable": true,
"avg": true,
"current": false,
"hideEmpty": false,
"hideZero": true,
"max": true,
"min": true,
"rightSide": false,
"show": true,
"sort": "max",
"sortDesc": true,
"total": false,
"values": true
},
"lines": true,
"linewidth": 1,
"nullPointMode": "null",
"options": {
"dataLinks": []
},
"percentage": false,
"pointradius": 0.5,
"points": false,
"renderer": "flot",
"seriesOverrides": [
{
"alias": "/.*total.*/",
"bars": true,
"fill": 0,
"lines": false,
"stack": false,
"yaxis": 2
},
{
"alias": "/received.*/",
"bars": true,
"color": "rgba(70, 131, 140, 0.29)",
"fill": 0,
"lines": false,
"stack": false,
"yaxis": 2,
"zindex": -3
}
],
"spaceLength": 10,
"stack": true,
"steppedLine": false,
"targets": [
{
"expr": "rate(ligato_govppmux_requests_total[$period])",
"hide": false,
"legendFormat": "sent {msg={{message}}}",
"refId": "D"
},
{
"expr": "rate(ligato_govppmux_requests_failed_total[$period])",
"hide": false,
"legendFormat": "failed {msg={{message}}}",
"refId": "G"
},
{
"expr": "rate(ligato_govppmux_replies_received_total[$period])",
"hide": true,
"legendFormat": "received {msg={{message}}}",
"refId": "E"
},
{
"expr": "sum(rate(ligato_govppmux_replies_received_total[$period]))",
"hide": false,
"legendFormat": "received replies total",
"refId": "B"
},
{
"expr": "sum(rate(ligato_govppmux_successful_request_duration_seconds_count[$period])) by (instance)",
"hide": true,
"legendFormat": "requests total",
"refId": "A"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "govppmux - requests rate",
"tooltip": {
"shared": true,
"sort": 2,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "short",
"label": "",
"logBase": 1,
"max": null,
"min": "0",
"show": true
},
{
"decimals": null,
"format": "short",
"label": "",
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {},
"bars": true,
"dashLength": 10,
"dashes": false,
"datasource": null,
"decimals": 1,
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 9,
"w": 12,
"x": 12,
"y": 39
},
"hiddenSeries": false,
"id": 4,
"legend": {
"alignAsTable": true,
"avg": true,
"current": false,
"hideEmpty": false,
"hideZero": true,
"max": true,
"min": true,
"rightSide": false,
"show": true,
"sort": "max",
"sortDesc": true,
"total": false,
"values": true
},
"lines": false,
"linewidth": 1,
"nullPointMode": "null",
"options": {
"dataLinks": []
},
"percentage": false,
"pointradius": 0.5,
"points": false,
"renderer": "flot",
"seriesOverrides": [
{
"alias": "/.*total.*/",
"bars": false,
"fill": 0,
"lines": true,
"stack": false,
"steppedLine": true,
"yaxis": 2
}
],
"spaceLength": 10,
"stack": true,
"steppedLine": false,
"targets": [
{
"expr": "rate(ligato_govppmux_successful_request_duration_seconds_sum{message!~\".*dump$\"}[$period])\n/\nrate(ligato_govppmux_successful_request_duration_seconds_count{message!~\".*dump$\"}[$period])",
"format": "time_series",
"hide": false,
"legendFormat": "{{message}}",
"refId": "A"
},
{
"expr": "sum(rate(ligato_govppmux_successful_request_duration_seconds_sum{message!~\".*dump$\"}[$period])) by (instance)",
"format": "time_series",
"hide": false,
"legendFormat": "duration total",
"refId": "D"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "govppmux - requests duration",
"tooltip": {
"shared": true,
"sort": 2,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"decimals": null,
"format": "s",
"label": "",
"logBase": 1,
"max": null,
"min": "0",
"show": true
},
{
"decimals": null,
"format": "s",
"label": "",
"logBase": 1,
"max": null,
"min": "0",
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
}
],
"refresh": false,
"schemaVersion": 22,
"style": "dark",
"tags": [],
"templating": {
"list": [
{
"auto": true,
"auto_count": 50,
"auto_min": "10s",
"current": {
"selected": true,
"text": "5s",
"value": "5s"
},
"hide": 0,
"label": "Period",
"name": "period",
"options": [
{
"selected": false,
"text": "auto",
"value": "$__auto_interval_period"
},
{
"selected": false,
"text": "1s",
"value": "1s"
},
{
"selected": true,
"text": "5s",
"value": "5s"
},
{
"selected": false,
"text": "10s",
"value": "10s"
},
{
"selected": false,
"text": "15s",
"value": "15s"
},
{
"selected": false,
"text": "30s",
"value": "30s"
},
{
"selected": false,
"text": "1m",
"value": "1m"
},
{
"selected": false,
"text": "5m",
"value": "5m"
},
{
"selected": false,
"text": "10m",
"value": "10m"
},
{
"selected": false,
"text": "30m",
"value": "30m"
},
{
"selected": false,
"text": "1h",
"value": "1h"
},
{
"selected": false,
"text": "6h",
"value": "6h"
},
{
"selected": false,
"text": "12h",
"value": "12h"
},
{
"selected": false,
"text": "24h",
"value": "24h"
}
],
"query": "1s,5s,10s,15s,30s,1m,5m,10m,30m,1h,6h,12h,24h",
"refresh": 2,
"skipUrlSync": false,
"type": "interval"
},
{
"allValue": null,
"current": {
"text": "agent1:9191",
"value": "agent1:9191"
},
"datasource": "Prometheus",
"definition": "label_values(ligato_build_info{job=\"agent\"}, instance)",
"hide": 0,
"includeAll": false,
"label": "",
"multi": false,
"name": "agent",
"options": [],
"query": "label_values(ligato_build_info{job=\"agent\"}, instance)",
"refresh": 2,
"regex": "",
"skipUrlSync": false,
"sort": 0,
"tagValuesQuery": "",
"tags": [],
"tagsQuery": "",
"type": "query",
"useTags": false
},
{
"allValue": null,
"current": {
"selected": true,
"text": "0.005",
"value": "0.005"
},
"datasource": "Prometheus",
"definition": "label_values(grpc_server_handling_seconds_bucket{job=\"agent\"}, le)",
"hide": 0,
"includeAll": false,
"label": "grpc request dur",
"multi": false,
"name": "grpcreqdur",
"options": [],
"query": "label_values(grpc_server_handling_seconds_bucket{job=\"agent\"}, le)",
"refresh": 1,
"regex": "",
"skipUrlSync": false,
"sort": 0,
"tagValuesQuery": "",
"tags": [],
"tagsQuery": "",
"type": "query",
"useTags": false
}
]
},
"time": {
"from": "2020-02-26T13:26:00.526Z",
"to": "2020-02-26T13:27:26.486Z"
},
"timepicker": {
"hidden": false,
"nowDelay": "5s",
"refresh_intervals": [
"1s",
"5s",
"10s",
"30s",
"1m",
"5m",
"15m",
"30m",
"1h",
"2h",
"1d"
]
},
"timezone": "",
"title": "VPP Agent Copy",
"uid": "gygjs3wWk",
"version": 12
}git clone https://github.com/ligato/vpp-agent
cd vpp-agent
# run basic perf test
bash ./scripts/run_perf_test.shHere's perf test output:
perf test output [open]
➤ AGENT_IMG=dev_vpp_agent bash ./scripts/run_perf_test.sh 1000 10 10
Starting perf test run: 1000-req
Preparing PERF testing..
================================================================================
PERF TEST - grpc-perf
================================================================================
report dir: /report
settings:
- requests per client: 1000
- tunnels per request: 10
- clients: 10
--------------------------------------------------------------------------------
-> compiling test client grpc-perf..
-> starting VPP (/usr/bin/vpp).. ok! (PID:641)
-> starting agent.. ok! (PID:647)
-> sleeping for 5 seconds before starting test
-> starting grpc-perf test..
--------------------------------------------------------------
time="2020-02-28 09:58:37.22111" level=info msg="Starting agent version: v3.1.0-dev" BuildDate="Mon Jan 1 00:00:00 UTC 0001" CommitHash=unknown loc="agent/agent.go(134)" logger=agent
time="2020-02-28 09:58:37.22162" level=info msg="Agent started with 1 plugins (took 0s)" loc="agent/agent.go(179)" logger=agent
INFO[0001] Configuring memif interfaces..
INFO[0001] ----------------------------------------
INFO[0001] SETTINGS:
INFO[0001] ----------------------------------------
INFO[0001] -> Clients: 10
INFO[0001] -> Requests: 1000
INFO[0001] -> Tunnels per request: 10
INFO[0001] ----------------------------------------
INFO[0001] Launching all clients..
INFO[0004] client #8 - progress 20% -> 200 tunnels took 3.254s (61.5 tunnels/sec)
INFO[0004] client #9 - progress 20% -> 200 tunnels took 3.646s (54.9 tunnels/sec)
INFO[0005] client #4 - progress 20% -> 200 tunnels took 3.817s (52.4 tunnels/sec)
INFO[0005] client #1 - progress 20% -> 200 tunnels took 3.946s (50.7 tunnels/sec)
INFO[0005] client #2 - progress 20% -> 200 tunnels took 3.956s (50.6 tunnels/sec)
INFO[0005] client #3 - progress 20% -> 200 tunnels took 3.968s (50.4 tunnels/sec)
INFO[0005] client #6 - progress 20% -> 200 tunnels took 4.000s (50.0 tunnels/sec)
INFO[0005] client #5 - progress 20% -> 200 tunnels took 4.039s (49.5 tunnels/sec)
INFO[0005] client #7 - progress 20% -> 200 tunnels took 4.124s (48.5 tunnels/sec)
INFO[0005] client #0 - progress 20% -> 200 tunnels took 4.168s (48.0 tunnels/sec)
INFO[0007] client #8 - progress 40% -> 200 tunnels took 3.327s (60.1 tunnels/sec)
INFO[0007] client #9 - progress 40% -> 200 tunnels took 3.111s (64.3 tunnels/sec)
INFO[0007] client #4 - progress 40% -> 200 tunnels took 2.958s (67.6 tunnels/sec)
INFO[0008] client #1 - progress 40% -> 200 tunnels took 2.856s (70.0 tunnels/sec)
INFO[0008] client #2 - progress 40% -> 200 tunnels took 2.865s (69.8 tunnels/sec)
INFO[0008] client #3 - progress 40% -> 200 tunnels took 2.871s (69.7 tunnels/sec)
INFO[0008] client #6 - progress 40% -> 200 tunnels took 2.873s (69.6 tunnels/sec)
INFO[0008] client #5 - progress 40% -> 200 tunnels took 2.852s (70.1 tunnels/sec)
INFO[0008] client #7 - progress 40% -> 200 tunnels took 2.780s (71.9 tunnels/sec)
INFO[0008] client #0 - progress 40% -> 200 tunnels took 2.753s (72.6 tunnels/sec)
INFO[0011] client #8 - progress 60% -> 200 tunnels took 3.898s (51.3 tunnels/sec)
INFO[0012] client #9 - progress 60% -> 200 tunnels took 4.096s (48.8 tunnels/sec)
INFO[0012] client #4 - progress 60% -> 200 tunnels took 4.097s (48.8 tunnels/sec)
INFO[0012] client #1 - progress 60% -> 200 tunnels took 4.093s (48.9 tunnels/sec)
INFO[0012] client #2 - progress 60% -> 200 tunnels took 4.102s (48.8 tunnels/sec)
INFO[0012] client #3 - progress 60% -> 200 tunnels took 4.281s (46.7 tunnels/sec)
INFO[0012] client #6 - progress 60% -> 200 tunnels took 4.265s (46.9 tunnels/sec)
INFO[0012] client #5 - progress 60% -> 200 tunnels took 4.287s (46.6 tunnels/sec)
INFO[0012] client #7 - progress 60% -> 200 tunnels took 4.293s (46.6 tunnels/sec)
INFO[0012] client #0 - progress 60% -> 200 tunnels took 4.292s (46.6 tunnels/sec)
INFO[0015] client #8 - progress 80% -> 200 tunnels took 3.929s (50.9 tunnels/sec)
INFO[0015] client #9 - progress 80% -> 200 tunnels took 3.856s (51.9 tunnels/sec)
INFO[0015] client #4 - progress 80% -> 200 tunnels took 3.850s (51.9 tunnels/sec)
INFO[0015] client #1 - progress 80% -> 200 tunnels took 3.840s (52.1 tunnels/sec)
INFO[0015] client #2 - progress 80% -> 200 tunnels took 3.830s (52.2 tunnels/sec)
INFO[0015] client #3 - progress 80% -> 200 tunnels took 3.648s (54.8 tunnels/sec)
INFO[0016] client #6 - progress 80% -> 200 tunnels took 3.646s (54.9 tunnels/sec)
INFO[0016] client #5 - progress 80% -> 200 tunnels took 3.661s (54.6 tunnels/sec)
INFO[0016] client #7 - progress 80% -> 200 tunnels took 3.660s (54.6 tunnels/sec)
INFO[0016] client #0 - progress 80% -> 200 tunnels took 3.666s (54.6 tunnels/sec)
INFO[0018] client #8 - progress 100% -> 200 tunnels took 2.903s (68.9 tunnels/sec)
INFO[0018] client #8 done => 1000 tunnels took 17.311s (57.8 tunnels/sec)
INFO[0018] client #9 - progress 100% -> 200 tunnels took 2.805s (71.3 tunnels/sec)
INFO[0018] client #9 done => 1000 tunnels took 17.514s (57.1 tunnels/sec)
INFO[0018] client #4 - progress 100% -> 200 tunnels took 2.805s (71.3 tunnels/sec)
INFO[0018] client #4 done => 1000 tunnels took 17.526s (57.1 tunnels/sec)
INFO[0018] client #1 - progress 100% -> 200 tunnels took 2.809s (71.2 tunnels/sec)
INFO[0018] client #1 done => 1000 tunnels took 17.544s (57.0 tunnels/sec)
INFO[0018] client #2 - progress 100% -> 200 tunnels took 2.807s (71.3 tunnels/sec)
INFO[0018] client #2 done => 1000 tunnels took 17.560s (56.9 tunnels/sec)
INFO[0018] client #6 - progress 100% -> 200 tunnels took 2.916s (68.6 tunnels/sec)
INFO[0018] client #6 done => 1000 tunnels took 17.700s (56.5 tunnels/sec)
INFO[0018] client #3 - progress 100% -> 200 tunnels took 2.959s (67.6 tunnels/sec)
INFO[0018] client #3 done => 1000 tunnels took 17.726s (56.4 tunnels/sec)
INFO[0018] client #5 - progress 100% -> 200 tunnels took 2.925s (68.4 tunnels/sec)
INFO[0018] client #5 done => 1000 tunnels took 17.764s (56.3 tunnels/sec)
INFO[0019] client #7 - progress 100% -> 200 tunnels took 2.924s (68.4 tunnels/sec)
INFO[0019] client #7 done => 1000 tunnels took 17.782s (56.2 tunnels/sec)
INFO[0019] client #0 - progress 100% -> 200 tunnels took 2.960s (67.6 tunnels/sec)
INFO[0019] client #0 done => 1000 tunnels took 17.839s (56.1 tunnels/sec)
INFO[0019] All clients done!
INFO[0019] ========================================
INFO[0019] RESULTS:
INFO[0019] ========================================
INFO[0019] Elapsed: 17.84 sec
INFO[0019] Average: 560.6 req/sec
INFO[0019] ========================================
time="2020-02-28 09:58:56.27255" level=info msg="Stopping agent" loc="agent/agent.go(269)" logger=agent
time="2020-02-28 09:58:56.27261" level=info msg="Agent stopped" loc="agent/agent.go(291)" logger=agent
--------------------------------------------------------------
-> grpc-perf test finished (exit code: 0)
-> checking VPP.. ok!
-> checking vpp-agent.. ok!
-> collecting system info to: /report/sys-info.txt
-> collecting agent data..
-> collecting VPP data to: /report/vpp-info.txt
--------------------------------------------------------------------------------
✓ Test run passed!
--------------------------------------------------------------------------------
-> sleeping for 5 seconds before stopping
-> stopping vpp-agent (PID: 647)..
-> stopping VPP (PID: 641)..
-> processing profiles..
./scripts/run_perf_test.sh: line 41: ests/perf/perf_test.sh: No such file or directory
a72e536333854d6e68aa2953fe8bcb9f1527b5abf22d298842a07b722110d507# usage: run_perf_test.sh <client_req> <tunnels_per_req> <num_clients>
# run perf test with 1 client, 20k requests per client, 100 tunnels per request (20k tunnels total)
bash ./scripts/run_perf_test.sh 20000 100
# run perf test with 5 clients, 10k requests per client, 5 tunnels per request (50k tunnels total)
bash ./scripts/run_perf_test.sh 10000 10 5# build dev image
make dev-image
# run with local image
AGENT_IMG=dev_vpp_agent bash ./scripts/run/perf_test.sh