{ "__inputs": [ { "name": "DS_PROMETHEUS", "label": "Prometheus", "description": "", "type": "datasource", "pluginId": "prometheus", "pluginName": "Prometheus" } ], "__requires": [ { "type": "grafana", "id": "grafana", "name": "Grafana", "version": "7.5.2" }, { "type": "panel", "id": "graph", "name": "Graph", "version": "" }, { "type": "datasource", "id": "prometheus", "name": "Prometheus", "version": "1.0.0" }, { "type": "panel", "id": "table", "name": "Table", "version": "" } ], "annotations": { "list": [ { "builtIn": 1, "datasource": "-- Grafana --", "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", "name": "Annotations & Alerts", "type": "dashboard" }, { "datasource": "${DS_PROMETHEUS}", "enable": true, "expr": "resets(ClickHouseAsyncMetrics_Uptime{instance=~\"$instance\"}[$__rate_interval])", "hide": false, "iconColor": "rgba(255, 96, 96, 1)", "name": "Restarts", "showIn": 0, "step": "60s", "tagKeys": "instance", "titleFormat": "Restart", "useValueForTime": false } ] }, "description": "ClickHouse internal exporter metrics", "editable": true, "gnetId": 14192, "graphTooltip": 1, "id": null, "iteration": 1617958425076, "links": [ { "asDropdown": false, "icon": "external link", "includeVars": false, "keepTime": false, "tags": [], "targetBlank": true, "title": "ClickHouse monitoring docs", "tooltip": "", "type": "link", "url": "https://clickhouse.tech/docs/en/operations/monitoring/" } ], "panels": [ { "datasource": "${DS_PROMETHEUS}", "description": "", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "custom": { "align": null, "filterable": false }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [ { "matcher": { "id": "byName", "options": "uptime" }, "properties": [ { "id": "unit", "value": "s" } ] } ] }, "gridPos": { "h": 8, "w": 24, "x": 0, "y": 0 }, "id": 14, "options": { "frameIndex": 1, "showHeader": true }, "pluginVersion": "7.5.2", "targets": [ { "exemplar": false, "expr": "ClickHouseMetrics_VersionInteger", "format": "table", "instant": true, "interval": "", "legendFormat": "", "refId": "A" }, { "exemplar": false, "expr": "ClickHouseMetrics_Revision", "format": "table", "hide": false, "instant": true, "interval": "", "legendFormat": "", "refId": "B" }, { "exemplar": false, "expr": "ClickHouseAsyncMetrics_NumberOfTables", "format": "table", "hide": false, "instant": true, "interval": "", "legendFormat": "", "refId": "C" }, { "exemplar": false, "expr": "ClickHouseAsyncMetrics_Uptime", "format": "table", "hide": false, "instant": true, "interval": "", "legendFormat": "", "refId": "D" }, { "exemplar": false, "expr": "ClickHouseAsyncMetrics_NumberOfDatabases", "format": "table", "hide": false, "instant": true, "interval": "", "legendFormat": "", "refId": "E" } ], "title": "Nodes", "transformations": [ { "id": "seriesToColumns", "options": { "byField": "instance" } }, { "id": "organize", "options": { "excludeByName": { "Time 1": true, "Time 10": true, "Time 2": true, "Time 3": true, "Time 4": true, "Time 5": true, "Time 6": true, "Time 7": true, "Time 8": true, "Time 9": true, "Value #A 2": true, "Value #B 2": true, "Value #C 2": true, "Value #D 2": true, "Value #E 2": true, "__name__ 1": true, "__name__ 10": true, "__name__ 2": true, "__name__ 3": true, "__name__ 4": true, "__name__ 5": true, "__name__ 6": true, "__name__ 7": true, "__name__ 8": true, "__name__ 9": true, "job 1": true, "job 10": true, "job 2": true, "job 3": true, "job 4": true, "job 5": true, "job 6": true, "job 7": true, "job 8": true, "job 9": true }, "indexByName": { "Time 1": 6, "Time 10": 37, "Time 2": 9, "Time 3": 13, "Time 4": 16, "Time 5": 20, "Time 6": 23, "Time 7": 27, "Time 8": 30, "Time 9": 34, "Value #A 1": 1, "Value #A 2": 12, "Value #B 1": 2, "Value #B 2": 19, "Value #C 1": 3, "Value #C 2": 26, "Value #D 1": 5, "Value #D 2": 33, "Value #E 1": 4, "Value #E 2": 40, "__name__ 1": 7, "__name__ 10": 38, "__name__ 2": 10, "__name__ 3": 14, "__name__ 4": 17, "__name__ 5": 21, "__name__ 6": 24, "__name__ 7": 28, "__name__ 8": 31, "__name__ 9": 35, "instance": 0, "job 1": 8, "job 10": 39, "job 2": 11, "job 3": 15, "job 4": 18, "job 5": 22, "job 6": 25, "job 7": 29, "job 8": 32, "job 9": 36 }, "renameByName": { "Value #A 1": "version", "Value #B 1": "revision", "Value #C 1": "tables", "Value #D 1": "uptime", "Value #E 1": "databases" } } } ], "transparent": true, "type": "table" }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_PROMETHEUS}", "description": "Total amount of memory (bytes) allocated by the server.", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 0, "fillGradient": 0, "gridPos": { "h": 5, "w": 12, "x": 0, "y": 8 }, "hiddenSeries": false, "id": 29, "legend": { "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, "rightSide": true, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 2, "maxDataPoints": 200, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.2", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": false, "expr": "ClickHouseMetrics_MemoryTracking", "interval": "", "legendFormat": "{{instance}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Memory", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "transparent": true, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:176", "format": "bytes", "label": "", "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:177", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_PROMETHEUS}", "description": "Number of connections to TCP server (clients with native interface), also included server-server distributed query connections. \nNumber of connections to HTTP server", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 0, "fillGradient": 0, "gridPos": { "h": 5, "w": 12, "x": 12, "y": 8 }, "hiddenSeries": false, "id": 73, "legend": { "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, "rightSide": true, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 2, "maxDataPoints": 200, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.2", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": false, "expr": "ClickHouseMetrics_TCPConnection", "interval": "", "legendFormat": "tcp - {{instance}}", "refId": "A" }, { "exemplar": false, "expr": "ClickHouseMetrics_HTTPConnection", "hide": false, "interval": "", "legendFormat": "http - {{instance}}", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Connections", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "transparent": true, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:176", "format": "short", "label": "", "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:177", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_PROMETHEUS}", "description": "Number of times the number of query processing threads was lowered due to slow reads.", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 0, "fillGradient": 0, "gridPos": { "h": 5, "w": 12, "x": 0, "y": 13 }, "hiddenSeries": false, "id": 72, "legend": { "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, "rightSide": true, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 2, "maxDataPoints": 200, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.2", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": false, "expr": "max_over_time( (irate(ClickHouseProfileEvents_ReadBackoff{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks", "interval": "", "legendFormat": "peaks - {{instance}}", "refId": "A" }, { "exemplar": false, "expr": "rate(ClickHouseProfileEvents_ReadBackoff{instance=~\"$instance\"}[$__rate_interval]) * $trends", "hide": false, "interval": "", "legendFormat": "trend - {{instance}}", "refId": "C" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Read backoff", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "transparent": true, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:176", "format": "cps", "label": "", "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:177", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_PROMETHEUS}", "description": "Number of reads from a file that were slow. This indicate system overload. Thresholds are controlled by read_backoff_* settings.", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 0, "fillGradient": 0, "gridPos": { "h": 5, "w": 12, "x": 12, "y": 13 }, "hiddenSeries": false, "id": 71, "legend": { "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, "rightSide": true, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 2, "maxDataPoints": 200, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.2", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": false, "expr": "max_over_time( (irate(ClickHouseProfileEvents_SlowRead{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks", "interval": "", "legendFormat": "peaks - {{instance}}", "refId": "A" }, { "exemplar": false, "expr": "rate(ClickHouseProfileEvents_SlowRead{instance=~\"$instance\"}[$__rate_interval]) * $trends", "hide": false, "interval": "", "legendFormat": "trend - {{instance}}", "refId": "C" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Slow reads", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "transparent": true, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:176", "format": "cps", "label": "", "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:177", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "collapsed": true, "datasource": "${DS_PROMETHEUS}", "gridPos": { "h": 1, "w": 24, "x": 0, "y": 18 }, "id": 12, "panels": [ { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_PROMETHEUS}", "description": "Number of queries to be interpreted and potentially executed. Does not include queries that failed to parse or were rejected due to AST size limits, quota limits or limits on the number of simultaneously running queries. May include internal queries initiated by ClickHouse itself. Does not count subqueries.", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 0, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 19 }, "hiddenSeries": false, "id": 2, "legend": { "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, "rightSide": true, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 2, "maxDataPoints": 200, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.2", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": false, "expr": "max_over_time( (irate(ClickHouseProfileEvents_Query{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks", "interval": "", "legendFormat": "peaks - {{instance}}", "refId": "A" }, { "exemplar": false, "expr": "rate(ClickHouseProfileEvents_Query{instance=~\"$instance\"}[$__rate_interval]) * $trends", "hide": false, "interval": "", "legendFormat": "trend - {{instance}}", "refId": "C" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Queries", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "transparent": true, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:176", "format": "reqps", "label": "", "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:177", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_PROMETHEUS}", "description": "Same as Queries, but only for SELECT queries.", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 0, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 19 }, "hiddenSeries": false, "id": 3, "legend": { "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, "rightSide": true, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 2, "maxDataPoints": 200, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.2", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": false, "expr": "max_over_time( (irate(ClickHouseProfileEvents_SelectQuery{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks", "interval": "", "legendFormat": "peaks - {{instance}}", "refId": "A" }, { "exemplar": false, "expr": "rate(ClickHouseProfileEvents_SelectQuery{instance=~\"$instance\"}[$__rate_interval]) * $trends", "hide": false, "interval": "", "legendFormat": "trend - {{instance}}", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "SELECT queries", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "transparent": true, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:176", "format": "reqps", "label": "", "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:177", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_PROMETHEUS}", "description": "Same as Queries, but only for INSERT queries.", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 0, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 27 }, "hiddenSeries": false, "id": 4, "legend": { "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, "rightSide": true, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 2, "maxDataPoints": 200, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.2", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": false, "expr": "max_over_time( (irate(ClickHouseProfileEvents_InsertQuery{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks", "interval": "", "legendFormat": "peaks - {{instance}}", "refId": "A" }, { "exemplar": false, "expr": "rate(ClickHouseProfileEvents_InsertQuery{instance=~\"$instance\"}[$__rate_interval]) * $trends", "hide": false, "interval": "", "legendFormat": "trend - {{instance}}", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "INSERT queries", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "transparent": true, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:176", "format": "reqps", "label": "", "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:177", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_PROMETHEUS}", "description": "Same as Failed queries, but only for SELECT queries.", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 0, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 27 }, "hiddenSeries": false, "id": 6, "legend": { "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, "rightSide": true, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 2, "maxDataPoints": 200, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.2", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": false, "expr": "max_over_time( (irate(ClickHouseProfileEvents_FailedSelectQuery{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks", "interval": "", "legendFormat": "peaks - {{instance}}", "refId": "A" }, { "exemplar": false, "expr": "rate(ClickHouseProfileEvents_FailedSelectQuery{instance=~\"$instance\"}[$__rate_interval]) * $trends", "hide": false, "interval": "", "legendFormat": "trend - {{instance}}", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Failed SELECT queries", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "transparent": true, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:176", "format": "reqps", "label": "", "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:177", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_PROMETHEUS}", "description": "Number of failed queries.", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 0, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 35 }, "hiddenSeries": false, "id": 5, "legend": { "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, "rightSide": true, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 2, "maxDataPoints": 200, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.2", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": false, "expr": "max_over_time( (irate(ClickHouseProfileEvents_FailedQuery{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks", "interval": "", "legendFormat": "peaks - {{instance}}", "refId": "A" }, { "exemplar": false, "expr": "rate(ClickHouseProfileEvents_FailedQuery{instance=~\"$instance\"}[$__rate_interval]) * $trends", "hide": false, "interval": "", "legendFormat": "trend - {{instance}}", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Failed queries", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "transparent": true, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:176", "format": "reqps", "label": "", "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:177", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_PROMETHEUS}", "description": "Number of times when memory limit exceeded for query.", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 0, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 35 }, "hiddenSeries": false, "id": 85, "legend": { "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, "rightSide": true, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 2, "maxDataPoints": 200, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.2", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": false, "expr": "max_over_time( (irate(ClickHouseProfileEvents_QueryMemoryLimitExceeded{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks", "interval": "", "legendFormat": "peaks - {{instance}}", "refId": "A" }, { "exemplar": false, "expr": "rate(ClickHouseProfileEvents_QueryMemoryLimitExceeded{instance=~\"$instance\"}[$__rate_interval]) * $trends", "hide": false, "interval": "", "legendFormat": "trend - {{instance}}", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Queries memory limit", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "transparent": true, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:176", "format": "reqps", "label": "", "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:177", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_PROMETHEUS}", "description": "Number of queries that are stopped and waiting due to 'priority' setting.", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 0, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 43 }, "hiddenSeries": false, "id": 86, "legend": { "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, "rightSide": true, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 2, "maxDataPoints": 200, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.2", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": false, "expr": "max_over_time( (irate(ClickHouseMetrics_QueryPreempted{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks", "interval": "", "legendFormat": "peaks - {{instance}}", "refId": "A" }, { "exemplar": false, "expr": "rate(ClickHouseMetrics_QueryPreempted{instance=~\"$instance\"}[$__rate_interval]) * $trends", "hide": false, "interval": "", "legendFormat": "trend - {{instance}}", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Query preemted", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "transparent": true, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:176", "format": "reqps", "label": "", "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:177", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_PROMETHEUS}", "description": "Same as Failed queries, but only for INSERT queries.", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 0, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 43 }, "hiddenSeries": false, "id": 7, "legend": { "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, "rightSide": true, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 2, "maxDataPoints": 200, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.2", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": false, "expr": "max_over_time( (irate(ClickHouseProfileEvents_FailedInsertQuery{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks", "interval": "", "legendFormat": "peaks - {{instance}}", "refId": "A" }, { "exemplar": false, "expr": "rate(ClickHouseProfileEvents_FailedInsertQuery{instance=~\"$instance\"}[$__rate_interval]) * $trends", "hide": false, "interval": "", "legendFormat": "trend - {{instance}}", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Failed INSERT queries", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "transparent": true, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:176", "format": "reqps", "label": "", "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:177", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_PROMETHEUS}", "description": "Avg queries latencies", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 0, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 51 }, "hiddenSeries": false, "id": 8, "legend": { "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, "rightSide": true, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 2, "maxDataPoints": 200, "nullPointMode": "null as zero", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.2", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": false, "expr": "increase(ClickHouseProfileEvents_QueryTimeMicroseconds{instance=~\"$instance\"}[$__rate_interval]) / increase(ClickHouseProfileEvents_Query{instance=~\"$instance\"}[$__rate_interval])", "interval": "", "legendFormat": "{{instance}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Query latencies (avg)", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "transparent": true, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:176", "format": "µs", "label": "", "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:177", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 1 } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_PROMETHEUS}", "description": "Avg SELECT queries latencies", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 0, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 51 }, "hiddenSeries": false, "id": 9, "legend": { "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, "rightSide": true, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 2, "maxDataPoints": 200, "nullPointMode": "null as zero", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.2", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": false, "expr": "increase(ClickHouseProfileEvents_SelectQueryTimeMicroseconds{instance=~\"$instance\"}[$__rate_interval]) / increase(ClickHouseProfileEvents_SelectQuery{instance=~\"$instance\"}[$__rate_interval])", "interval": "", "legendFormat": "{{instance}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "SELECT query latencies (avg)", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "transparent": true, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:176", "format": "µs", "label": "", "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:177", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_PROMETHEUS}", "description": "Avg INSERT queries latencies", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 0, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 59 }, "hiddenSeries": false, "id": 10, "legend": { "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, "rightSide": true, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 2, "maxDataPoints": 200, "nullPointMode": "null as zero", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.2", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": false, "expr": "increase(ClickHouseProfileEvents_InsertQueryTimeMicroseconds{instance=~\"$instance\"}[$__rate_interval]) / increase(ClickHouseProfileEvents_InsertQuery{instance=~\"$instance\"}[$__rate_interval])", "interval": "", "legendFormat": "{{instance}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "INSERT query latencies (avg)", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "transparent": true, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:176", "format": "µs", "label": "", "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:177", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } } ], "title": "Queries", "type": "row" }, { "collapsed": true, "datasource": "${DS_PROMETHEUS}", "gridPos": { "h": 1, "w": 24, "x": 0, "y": 19 }, "id": 64, "panels": [ { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_PROMETHEUS}", "description": "Number of rows INSERTed to all tables.", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 0, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 30 }, "hiddenSeries": false, "id": 62, "legend": { "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, "rightSide": true, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 2, "maxDataPoints": 200, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.2", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": false, "expr": "max_over_time( (irate(ClickHouseProfileEvents_InsertedRows{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks", "interval": "", "legendFormat": "peaks - {{instance}}", "refId": "A" }, { "exemplar": false, "expr": "rate(ClickHouseProfileEvents_InsertedRows{instance=~\"$instance\"}[$__rate_interval]) * $trends", "hide": false, "interval": "", "legendFormat": "trend - {{instance}}", "refId": "C" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Inserted rows", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "transparent": true, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:176", "format": "cps", "label": "", "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:177", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_PROMETHEUS}", "description": "Number of bytes (uncompressed; for columns as they stored in memory) INSERTed to all tables.", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 0, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 30 }, "hiddenSeries": false, "id": 65, "legend": { "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, "rightSide": true, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 2, "maxDataPoints": 200, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.2", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": false, "expr": "max_over_time( (irate(ClickHouseProfileEvents_InsertedBytes{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks", "interval": "", "legendFormat": "peaks - {{instance}}", "refId": "A" }, { "exemplar": false, "expr": "rate(ClickHouseProfileEvents_InsertedBytes{instance=~\"$instance\"}[$__rate_interval]) * $trends", "hide": false, "interval": "", "legendFormat": "trend - {{instance}}", "refId": "C" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Inserted bytes", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "transparent": true, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:176", "format": "binBps", "label": "", "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:177", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_PROMETHEUS}", "description": "Number of times the INSERT of a block to a MergeTree table was throttled due to high number of active data parts for partition.", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 0, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 38 }, "hiddenSeries": false, "id": 66, "legend": { "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, "rightSide": true, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 2, "maxDataPoints": 200, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.2", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": false, "expr": "max_over_time( (irate(ClickHouseProfileEvents_DelayedInserts{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks", "interval": "", "legendFormat": "peaks - {{instance}}", "refId": "A" }, { "exemplar": false, "expr": "rate(ClickHouseProfileEvents_DelayedInserts{instance=~\"$instance\"}[$__rate_interval]) * $trends", "hide": false, "interval": "", "legendFormat": "trend - {{instance}}", "refId": "C" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Delayed inserts", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "transparent": true, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:176", "format": "cps", "label": "", "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:177", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_PROMETHEUS}", "description": "Number of times the INSERT of a block to a MergeTree table was rejected with 'Too many parts' exception due to high number of active data parts for partition.", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 0, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 38 }, "hiddenSeries": false, "id": 67, "legend": { "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, "rightSide": true, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 2, "maxDataPoints": 200, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.2", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": false, "expr": "max_over_time( (irate(ClickHouseProfileEvents_RejectedInserts{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks", "interval": "", "legendFormat": "peaks - {{instance}}", "refId": "A" }, { "exemplar": false, "expr": "rate(ClickHouseProfileEvents_RejectedInserts{instance=~\"$instance\"}[$__rate_interval]) * $trends", "hide": false, "interval": "", "legendFormat": "trend - {{instance}}", "refId": "C" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Rejected inserts", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "transparent": true, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:176", "format": "cps", "label": "", "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:177", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_PROMETHEUS}", "description": "Total number of milliseconds spent while the INSERT of a block to a MergeTree table was throttled due to high number of active data parts for partition.", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 0, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 46 }, "hiddenSeries": false, "id": 69, "legend": { "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, "rightSide": true, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 2, "maxDataPoints": 200, "nullPointMode": "null as zero", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.2", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": false, "expr": "increase(ClickHouseProfileEvents_DelayedInsertsMilliseconds{instance=~\"$instance\"}[$__rate_interval]) / increase(ClickHouseProfileEvents_DelayedInserts{instance=~\"$instance\"}[$__rate_interval])", "interval": "", "legendFormat": "{{instance}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Delayed inserts blocks (avg)", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "transparent": true, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:176", "format": "µs", "label": "", "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:177", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 1 } } ], "title": "Insert", "type": "row" }, { "collapsed": true, "datasource": "${DS_PROMETHEUS}", "gridPos": { "h": 1, "w": 24, "x": 0, "y": 20 }, "id": 125, "panels": [ { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_PROMETHEUS}", "description": "Number of data parts selected to read from a MergeTree table.", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 0, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 27 }, "hiddenSeries": false, "id": 126, "legend": { "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, "rightSide": true, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 2, "maxDataPoints": 200, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.2", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": false, "expr": "max_over_time( (irate(ClickHouseProfileEvents_SelectedParts{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks", "interval": "", "legendFormat": "peaks - {{instance}}", "refId": "A" }, { "exemplar": false, "expr": "rate(ClickHouseProfileEvents_SelectedParts{instance=~\"$instance\"}[$__rate_interval]) * $trends", "hide": false, "interval": "", "legendFormat": "trend - {{instance}}", "refId": "C" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Parts", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "transparent": true, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:176", "format": "cps", "label": "", "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:177", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_PROMETHEUS}", "description": "Number of (non-adjacent) ranges in all data parts selected to read from a MergeTree table.", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 0, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 27 }, "hiddenSeries": false, "id": 127, "legend": { "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, "rightSide": true, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 2, "maxDataPoints": 200, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.2", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": false, "expr": "max_over_time( (irate(ClickHouseProfileEvents_SelectedRanges{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks", "interval": "", "legendFormat": "peaks - {{instance}}", "refId": "A" }, { "exemplar": false, "expr": "rate(ClickHouseProfileEvents_SelectedRanges{instance=~\"$instance\"}[$__rate_interval]) * $trends", "hide": false, "interval": "", "legendFormat": "trend - {{instance}}", "refId": "C" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Ranges", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "transparent": true, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:176", "format": "cps", "label": "", "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:177", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_PROMETHEUS}", "description": "Number of marks (index granules) selected to read from a MergeTree table.", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 0, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 35 }, "hiddenSeries": false, "id": 128, "legend": { "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, "rightSide": true, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 2, "maxDataPoints": 200, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.2", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": false, "expr": "max_over_time( (irate(ClickHouseProfileEvents_SelectedMarks{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks", "interval": "", "legendFormat": "peaks - {{instance}}", "refId": "A" }, { "exemplar": false, "expr": "rate(ClickHouseProfileEvents_SelectedMarks{instance=~\"$instance\"}[$__rate_interval]) * $trends", "hide": false, "interval": "", "legendFormat": "trend - {{instance}}", "refId": "C" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Marks", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "transparent": true, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:176", "format": "cps", "label": "", "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:177", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_PROMETHEUS}", "description": "Number of rows SELECTed from all tables.", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 0, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 35 }, "hiddenSeries": false, "id": 129, "legend": { "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, "rightSide": true, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 2, "maxDataPoints": 200, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.2", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": false, "expr": "max_over_time( (irate(ClickHouseProfileEvents_SelectedRows{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks", "interval": "", "legendFormat": "peaks - {{instance}}", "refId": "A" }, { "exemplar": false, "expr": "rate(ClickHouseProfileEvents_SelectedRows{instance=~\"$instance\"}[$__rate_interval]) * $trends", "hide": false, "interval": "", "legendFormat": "trend - {{instance}}", "refId": "C" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Rows", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "transparent": true, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:176", "format": "cps", "label": "", "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:177", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_PROMETHEUS}", "description": "Number of bytes (uncompressed; for columns as they stored in memory) SELECTed from all tables.", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 0, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 43 }, "hiddenSeries": false, "id": 130, "legend": { "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, "rightSide": true, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 2, "maxDataPoints": 200, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.2", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": false, "expr": "max_over_time( (irate(ClickHouseProfileEvents_SelectedBytes{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks", "interval": "", "legendFormat": "peaks - {{instance}}", "refId": "A" }, { "exemplar": false, "expr": "rate(ClickHouseProfileEvents_SelectedBytes{instance=~\"$instance\"}[$__rate_interval]) * $trends", "hide": false, "interval": "", "legendFormat": "trend - {{instance}}", "refId": "C" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Bytes", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "transparent": true, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:176", "format": "binBps", "label": "", "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:177", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } } ], "title": "Select", "type": "row" }, { "collapsed": true, "datasource": "${DS_PROMETHEUS}", "gridPos": { "h": 1, "w": 24, "x": 0, "y": 21 }, "id": 17, "panels": [ { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_PROMETHEUS}", "description": "Number of times the 'lseek' function was called.", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 0, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 33 }, "hiddenSeries": false, "id": 20, "legend": { "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, "rightSide": true, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 2, "maxDataPoints": 200, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.2", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": false, "expr": "max_over_time( (irate(ClickHouseProfileEvents_Seek{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks", "interval": "", "legendFormat": "peaks - {{instance}}", "refId": "A" }, { "exemplar": false, "expr": "rate(ClickHouseProfileEvents_Seek{instance=~\"$instance\"}[$__rate_interval]) * $trends", "hide": false, "interval": "", "legendFormat": "trend - {{instance}}", "refId": "C" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "lseek", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "transparent": true, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:176", "format": "ops", "label": "", "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:177", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_PROMETHEUS}", "description": "Number of files opened", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 0, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 33 }, "hiddenSeries": false, "id": 19, "legend": { "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, "rightSide": true, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 2, "maxDataPoints": 200, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.2", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": false, "expr": "max_over_time( (irate(ClickHouseProfileEvents_FileOpen{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks", "interval": "", "legendFormat": "peaks - {{instance}}", "refId": "A" }, { "exemplar": false, "expr": "rate(ClickHouseProfileEvents_FileOpen{instance=~\"$instance\"}[$__rate_interval]) * $trends", "hide": false, "interval": "", "legendFormat": "trend - {{instance}}", "refId": "C" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Open Files", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "transparent": true, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:176", "format": "ops", "label": "", "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:177", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_PROMETHEUS}", "description": "Number of reads (read/pread) from a file descriptor. Does not include sockets.", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 0, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 41 }, "hiddenSeries": false, "id": 22, "legend": { "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, "rightSide": true, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 2, "maxDataPoints": 200, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.2", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": false, "expr": "max_over_time( (irate(ClickHouseProfileEvents_ReadBufferFromFileDescriptorRead{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks", "interval": "", "legendFormat": "peaks - {{instance}}", "refId": "A" }, { "exemplar": false, "expr": "rate(ClickHouseProfileEvents_ReadBufferFromFileDescriptorRead{instance=~\"$instance\"}[$__rate_interval]) * $trends", "hide": false, "interval": "", "legendFormat": "trend - {{instance}}", "refId": "C" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Read from FD", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "transparent": true, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:176", "format": "ops", "label": "", "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:177", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_PROMETHEUS}", "description": "Number of writes (write/pwrite) to a file descriptor. Does not include sockets.", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 0, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 41 }, "hiddenSeries": false, "id": 23, "legend": { "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, "rightSide": true, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 2, "maxDataPoints": 200, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.2", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": false, "expr": "max_over_time( (irate(ClickHouseProfileEvents_WriteBufferFromFileDescriptorWrite{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks", "interval": "", "legendFormat": "peaks - {{instance}}", "refId": "A" }, { "exemplar": false, "expr": "rate(ClickHouseProfileEvents_WriteBufferFromFileDescriptorWrite{instance=~\"$instance\"}[$__rate_interval]) * $trends", "hide": false, "interval": "", "legendFormat": "trend - {{instance}}", "refId": "C" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Write to FD", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "transparent": true, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:176", "format": "ops", "label": "", "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:177", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_PROMETHEUS}", "description": "Number of times the read (read/pread) from a file descriptor have failed.", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 0, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 49 }, "hiddenSeries": false, "id": 24, "legend": { "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, "rightSide": true, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 2, "maxDataPoints": 200, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.2", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": false, "expr": "max_over_time( (irate(ClickHouseProfileEvents_ReadBufferFromFileDescriptorReadFailed{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks", "interval": "", "legendFormat": "peaks - {{instance}}", "refId": "A" }, { "exemplar": false, "expr": "rate(ClickHouseProfileEvents_ReadBufferFromFileDescriptorReadFailed{instance=~\"$instance\"}[$__rate_interval]) * $trends", "hide": false, "interval": "", "legendFormat": "trend - {{instance}}", "refId": "C" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Read from FD failed", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "transparent": true, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:176", "format": "ops", "label": "", "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:177", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_PROMETHEUS}", "description": "Number of times the read (read/pread) from a file descriptor have failed.", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 0, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 49 }, "hiddenSeries": false, "id": 25, "legend": { "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, "rightSide": true, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 2, "maxDataPoints": 200, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.2", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": false, "expr": "max_over_time( (irate(ClickHouseProfileEvents_ReadBufferFromFileDescriptorReadFailed{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks", "interval": "", "legendFormat": "peaks - {{instance}}", "refId": "A" }, { "exemplar": false, "expr": "rate(ClickHouseProfileEvents_ReadBufferFromFileDescriptorReadFailed{instance=~\"$instance\"}[$__rate_interval]) * $trends", "hide": false, "interval": "", "legendFormat": "trend - {{instance}}", "refId": "C" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Read from FD failed", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "transparent": true, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:176", "format": "ops", "label": "", "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:177", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_PROMETHEUS}", "description": "Number of bytes read from file descriptors. If the file is compressed, this will show the compressed data size.", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 0, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 57 }, "hiddenSeries": false, "id": 21, "legend": { "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, "rightSide": true, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 2, "maxDataPoints": 200, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.2", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": false, "expr": "max_over_time( (irate(ClickHouseProfileEvents_ReadBufferFromFileDescriptorReadBytes{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks", "interval": "", "legendFormat": "peaks - {{instance}}", "refId": "A" }, { "exemplar": false, "expr": "rate(ClickHouseProfileEvents_ReadBufferFromFileDescriptorReadBytes{instance=~\"$instance\"}[$__rate_interval]) * $trends", "hide": false, "interval": "", "legendFormat": "trend - {{instance}}", "refId": "C" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Read bytes from FD", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "transparent": true, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:176", "format": "binBps", "label": "", "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:177", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_PROMETHEUS}", "description": "Number of bytes written to file descriptors. If the file is compressed, this will show compressed data size.", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 0, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 57 }, "hiddenSeries": false, "id": 26, "legend": { "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, "rightSide": true, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 2, "maxDataPoints": 200, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.2", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": false, "expr": "max_over_time( (irate(ClickHouseProfileEvents_WriteBufferFromFileDescriptorWriteBytes{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks", "interval": "", "legendFormat": "peaks - {{instance}}", "refId": "A" }, { "exemplar": false, "expr": "rate(ClickHouseProfileEvents_WriteBufferFromFileDescriptorWriteBytes{instance=~\"$instance\"}[$__rate_interval]) * $trends", "hide": false, "interval": "", "legendFormat": "trend - {{instance}}", "refId": "C" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Write bytes to FD", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "transparent": true, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:176", "format": "binBps", "label": "", "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:177", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_PROMETHEUS}", "description": "Number of bytes (the number of bytes before decompression) read from compressed sources (files, network).", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 0, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 65 }, "hiddenSeries": false, "id": 51, "legend": { "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, "rightSide": true, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 2, "maxDataPoints": 200, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.2", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": false, "expr": "max_over_time( (irate(ClickHouseProfileEvents_ReadCompressedBytes{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks", "interval": "", "legendFormat": "peaks - {{instance}}", "refId": "A" }, { "exemplar": false, "expr": "rate(ClickHouseProfileEvents_ReadCompressedBytes{instance=~\"$instance\"}[$__rate_interval]) * $trends", "hide": false, "interval": "", "legendFormat": "trend - {{instance}}", "refId": "C" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Read compressed bytes", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "transparent": true, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:176", "format": "binBps", "label": "", "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:177", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_PROMETHEUS}", "description": "Number of uncompressed bytes (the number of bytes after decompression) read from compressed sources (files, network).", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 0, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 65 }, "hiddenSeries": false, "id": 53, "legend": { "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, "rightSide": true, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 2, "maxDataPoints": 200, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.2", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": false, "expr": "max_over_time( (irate(ClickHouseProfileEvents_CompressedReadBufferBytes{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks", "interval": "", "legendFormat": "peaks - {{instance}}", "refId": "A" }, { "exemplar": false, "expr": "rate(ClickHouseProfileEvents_CompressedReadBufferBytes{instance=~\"$instance\"}[$__rate_interval]) * $trends", "hide": false, "interval": "", "legendFormat": "trend - {{instance}}", "refId": "C" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Read uncompressed bytes", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "transparent": true, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:176", "format": "binBps", "label": "", "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:177", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_PROMETHEUS}", "description": "Number of compressed blocks (the blocks of data that are compressed independent of each other) read from compressed sources (files, network).", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 0, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 73 }, "hiddenSeries": false, "id": 52, "legend": { "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, "rightSide": true, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 2, "maxDataPoints": 200, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.2", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": false, "expr": "max_over_time( (irate(ClickHouseProfileEvents_CompressedReadBufferBlocks{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks", "interval": "", "legendFormat": "peaks - {{instance}}", "refId": "A" }, { "exemplar": false, "expr": "rate(ClickHouseProfileEvents_CompressedReadBufferBlocks{instance=~\"$instance\"}[$__rate_interval]) * $trends", "hide": false, "interval": "", "legendFormat": "trend - {{instance}}", "refId": "C" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Compressed blocks", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "transparent": true, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:176", "format": "cps", "label": "", "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:177", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } } ], "title": "IO", "type": "row" }, { "collapsed": true, "datasource": "${DS_PROMETHEUS}", "gridPos": { "h": 1, "w": 24, "x": 0, "y": 22 }, "id": 56, "panels": [ { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_PROMETHEUS}", "description": "Number of Replicated tables that are currently in readonly state due to re-initialization after ZooKeeper session loss or due to startup without ZooKeeper configured.", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 0, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 31 }, "hiddenSeries": false, "id": 74, "legend": { "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, "rightSide": true, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 2, "maxDataPoints": 200, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.2", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": false, "expr": "ClickHouseMetrics_ReadonlyReplica", "interval": "", "legendFormat": "{{instance}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Readonly replica", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "transparent": true, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:176", "format": "short", "label": "", "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:177", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_PROMETHEUS}", "description": "Number of times a data part was failed to download from replica of a ReplicatedMergeTree table.", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 0, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 31 }, "hiddenSeries": false, "id": 58, "legend": { "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, "rightSide": true, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 2, "maxDataPoints": 200, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.2", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": false, "expr": "max_over_time( (irate(ClickHouseProfileEvents_ReplicatedPartFailedFetches{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks", "interval": "", "legendFormat": "peaks - {{instance}}", "refId": "A" }, { "exemplar": false, "expr": "rate(ClickHouseProfileEvents_ReplicatedPartFailedFetches{instance=~\"$instance\"}[$__rate_interval]) * $trends", "hide": false, "interval": "", "legendFormat": "trend - {{instance}}", "refId": "C" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Replicated part failed fetches", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "transparent": true, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:176", "format": "cps", "label": "", "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:177", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_PROMETHEUS}", "description": "Number of times a data part was downloaded from replica of a ReplicatedMergeTree table.", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 0, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 39 }, "hiddenSeries": false, "id": 57, "legend": { "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, "rightSide": true, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 2, "maxDataPoints": 200, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.2", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": false, "expr": "max_over_time( (irate(ClickHouseProfileEvents_ReplicatedPartFetches{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks", "interval": "", "legendFormat": "peaks - {{instance}}", "refId": "A" }, { "exemplar": false, "expr": "rate(ClickHouseProfileEvents_ReplicatedPartFetches{instance=~\"$instance\"}[$__rate_interval]) * $trends", "hide": false, "interval": "", "legendFormat": "trend - {{instance}}", "refId": "C" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Replicated part fetches", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "transparent": true, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:176", "format": "cps", "label": "", "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:177", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_PROMETHEUS}", "description": "Number of times we prefer to download already merged part from replica of ReplicatedMergeTree table instead of performing a merge ourself (usually we prefer doing a merge ourself to save network traffic). This happens when we have not all source parts to perform a merge or when the data part is old enough.", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 0, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 39 }, "hiddenSeries": false, "id": 60, "legend": { "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, "rightSide": true, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 2, "maxDataPoints": 200, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.2", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": false, "expr": "max_over_time( (irate(ClickHouseProfileEvents_ReplicatedPartFetchesOfMerged{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks", "interval": "", "legendFormat": "peaks - {{instance}}", "refId": "A" }, { "exemplar": false, "expr": "rate(ClickHouseProfileEvents_ReplicatedPartFetchesOfMerged{instance=~\"$instance\"}[$__rate_interval]) * $trends", "hide": false, "interval": "", "legendFormat": "trend - {{instance}}", "refId": "C" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Replicated part fetches or merges", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "transparent": true, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:176", "format": "cps", "label": "", "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:177", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_PROMETHEUS}", "description": "Number of times data parts of ReplicatedMergeTree tables were successfully merged.", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 0, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 47 }, "hiddenSeries": false, "id": 59, "legend": { "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, "rightSide": true, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 2, "maxDataPoints": 200, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.2", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": false, "expr": "max_over_time( (irate(ClickHouseProfileEvents_ReplicatedPartMerges{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks", "interval": "", "legendFormat": "peaks - {{instance}}", "refId": "A" }, { "exemplar": false, "expr": "rate(ClickHouseProfileEvents_ReplicatedPartMerges{instance=~\"$instance\"}[$__rate_interval]) * $trends", "hide": false, "interval": "", "legendFormat": "trend - {{instance}}", "refId": "C" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Replicated part merges", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "transparent": true, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:176", "format": "cps", "label": "", "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:177", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_PROMETHEUS}", "description": "Number of times a data part that we wanted doesn't exist on any replica (even on replicas that are offline right now). That data parts are definitely lost. This is normal due to asynchronous replication (if quorum inserts were not enabled), when the replica on which the data part was written was failed and when it became online after fail it doesn't contain that data part.", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 0, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 47 }, "hiddenSeries": false, "id": 61, "legend": { "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, "rightSide": true, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 2, "maxDataPoints": 200, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.2", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": false, "expr": "max_over_time( (irate(ClickHouseProfileEvents_ReplicatedDataLoss{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks", "interval": "", "legendFormat": "peaks - {{instance}}", "refId": "A" }, { "exemplar": false, "expr": "rate(ClickHouseProfileEvents_ReplicatedDataLoss{instance=~\"$instance\"}[$__rate_interval]) * $trends", "hide": false, "interval": "", "legendFormat": "trend - {{instance}}", "refId": "C" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Replicated part data loss", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "transparent": true, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:176", "format": "cps", "label": "", "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:177", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } } ], "title": "Replicas", "type": "row" }, { "collapsed": true, "datasource": "${DS_PROMETHEUS}", "gridPos": { "h": 1, "w": 24, "x": 0, "y": 23 }, "id": 76, "panels": [ { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_PROMETHEUS}", "description": "Number of launched background merges.", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 0, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 29 }, "hiddenSeries": false, "id": 77, "legend": { "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, "rightSide": true, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 2, "maxDataPoints": 200, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.2", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": false, "expr": "max_over_time( (irate(ClickHouseProfileEvents_Merge{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks", "interval": "", "legendFormat": "peaks - {{instance}}", "refId": "A" }, { "exemplar": false, "expr": "rate(ClickHouseProfileEvents_Merge{instance=~\"$instance\"}[$__rate_interval]) * $trends", "hide": false, "interval": "", "legendFormat": "trend - {{instance}}", "refId": "C" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Merge", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "transparent": true, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:176", "format": "cps", "label": "", "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:177", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_PROMETHEUS}", "description": "Rows read for background merges. This is the number of rows before merge.", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 0, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 29 }, "hiddenSeries": false, "id": 78, "legend": { "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, "rightSide": true, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 2, "maxDataPoints": 200, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.2", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": false, "expr": "max_over_time( (irate(ClickHouseProfileEvents_MergedRows{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks", "interval": "", "legendFormat": "peaks - {{instance}}", "refId": "A" }, { "exemplar": false, "expr": "rate(ClickHouseProfileEvents_MergedRows{instance=~\"$instance\"}[$__rate_interval]) * $trends", "hide": false, "interval": "", "legendFormat": "trend - {{instance}}", "refId": "C" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Merged rows", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "transparent": true, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:176", "format": "cps", "label": "", "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:177", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_PROMETHEUS}", "description": "Uncompressed bytes (for columns as they stored in memory) that was read for background merges. This is the number before merge.", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 0, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 37 }, "hiddenSeries": false, "id": 79, "legend": { "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, "rightSide": true, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 2, "maxDataPoints": 200, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.2", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": false, "expr": "max_over_time( (irate(ClickHouseProfileEvents_MergedUncompressedBytes{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks", "interval": "", "legendFormat": "peaks - {{instance}}", "refId": "A" }, { "exemplar": false, "expr": "rate(ClickHouseProfileEvents_MergedUncompressedBytes{instance=~\"$instance\"}[$__rate_interval]) * $trends", "hide": false, "interval": "", "legendFormat": "trend - {{instance}}", "refId": "C" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Merge uncompressed bytes", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "transparent": true, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:176", "format": "binBps", "label": "", "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:177", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_PROMETHEUS}", "description": "Avg merge duration", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 0, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 37 }, "hiddenSeries": false, "id": 80, "legend": { "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, "rightSide": true, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 2, "maxDataPoints": 200, "nullPointMode": "null as zero", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.2", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": false, "expr": "increase(ClickHouseProfileEvents_MergesTimeMilliseconds{instance=~\"$instance\"}[$__rate_interval]) / increase(ClickHouseProfileEvents_Merge{instance=~\"$instance\"}[$__rate_interval])", "interval": "", "legendFormat": "{{instance}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Merge duration (avg)", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "transparent": true, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:176", "format": "µs", "label": "", "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:177", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 1 } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_PROMETHEUS}", "description": "Number of rows INSERTed to MergeTree tables.", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 0, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 45 }, "hiddenSeries": false, "id": 81, "legend": { "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, "rightSide": true, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 2, "maxDataPoints": 200, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.2", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": false, "expr": "max_over_time( (irate(ClickHouseProfileEvents_MergeTreeDataWriterRows{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks", "interval": "", "legendFormat": "peaks - {{instance}}", "refId": "A" }, { "exemplar": false, "expr": "rate(ClickHouseProfileEvents_MergeTreeDataWriterRows{instance=~\"$instance\"}[$__rate_interval]) * $trends", "hide": false, "interval": "", "legendFormat": "trend - {{instance}}", "refId": "C" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "MergeTree rows", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "transparent": true, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:176", "format": "cps", "label": "", "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:177", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_PROMETHEUS}", "description": "Number of blocks INSERTed to MergeTree tables. Each block forms a data part of level zero.", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 0, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 45 }, "hiddenSeries": false, "id": 82, "legend": { "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, "rightSide": true, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 2, "maxDataPoints": 200, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.2", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": false, "expr": "max_over_time( (irate(ClickHouseProfileEvents_MergeTreeDataWriterBlocks{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks", "interval": "", "legendFormat": "peaks - {{instance}}", "refId": "A" }, { "exemplar": false, "expr": "rate(ClickHouseProfileEvents_MergeTreeDataWriterBlocks{instance=~\"$instance\"}[$__rate_interval]) * $trends", "hide": false, "interval": "", "legendFormat": "trend - {{instance}}", "refId": "C" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "MergeTree blocks", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "transparent": true, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:176", "format": "cps", "label": "", "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:177", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_PROMETHEUS}", "description": "Uncompressed bytes (for columns as they stored in memory) INSERTed to MergeTree tables.", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 0, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 53 }, "hiddenSeries": false, "id": 83, "legend": { "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, "rightSide": true, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 2, "maxDataPoints": 200, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.2", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": false, "expr": "max_over_time( (irate(ClickHouseProfileEvents_MergeTreeDataWriterUncompressedBytes{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks", "interval": "", "legendFormat": "peaks - {{instance}}", "refId": "A" }, { "exemplar": false, "expr": "rate(ClickHouseProfileEvents_MergeTreeDataWriterUncompressedBytes{instance=~\"$instance\"}[$__rate_interval]) * $trends", "hide": false, "interval": "", "legendFormat": "trend - {{instance}}", "refId": "C" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "MergeTree uncompressed bytes", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "transparent": true, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:176", "format": "binBps", "label": "", "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:177", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_PROMETHEUS}", "description": "Bytes written to filesystem for data INSERTed to MergeTree tables.", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 0, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 53 }, "hiddenSeries": false, "id": 84, "legend": { "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, "rightSide": true, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 2, "maxDataPoints": 200, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.2", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": false, "expr": "max_over_time( (irate(ClickHouseProfileEvents_MergeTreeDataWriterCompressedBytes{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks", "interval": "", "legendFormat": "peaks - {{instance}}", "refId": "A" }, { "exemplar": false, "expr": "rate(ClickHouseProfileEvents_MergeTreeDataWriterCompressedBytes{instance=~\"$instance\"}[$__rate_interval]) * $trends", "hide": false, "interval": "", "legendFormat": "trend - {{instance}}", "refId": "C" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "MergeTree compressed bytes", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "transparent": true, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:176", "format": "binBps", "label": "", "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:177", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_PROMETHEUS}", "description": "Disk space reserved for currently running background merges. It is slightly more than the total size of currently merging parts.", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 0, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 61 }, "hiddenSeries": false, "id": 123, "legend": { "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, "rightSide": true, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 2, "maxDataPoints": 200, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.2", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": false, "expr": "ClickHouseMetrics_DiskSpaceReservedForMerge", "interval": "", "legendFormat": "{{instance}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Reserved space", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "transparent": true, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:176", "format": "bytes", "label": "", "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:177", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } } ], "title": "Merge", "type": "row" }, { "collapsed": true, "datasource": "${DS_PROMETHEUS}", "gridPos": { "h": 1, "w": 24, "x": 0, "y": 24 }, "id": 92, "panels": [ { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_PROMETHEUS}", "description": "", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 0, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 26 }, "hiddenSeries": false, "id": 102, "legend": { "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, "rightSide": true, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 2, "maxDataPoints": 200, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.2", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": false, "expr": "max_over_time( (irate(ClickHouseProfileEvents_UncompressedCacheHits{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks", "interval": "", "legendFormat": "peaks - {{instance}}", "refId": "A" }, { "exemplar": false, "expr": "rate(ClickHouseProfileEvents_UncompressedCacheHits{instance=~\"$instance\"}[$__rate_interval]) * $trends", "hide": false, "interval": "", "legendFormat": "trend - {{instance}}", "refId": "C" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Uncompressed Cache Hits", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "transparent": true, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:176", "format": "reqps", "label": "", "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:177", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_PROMETHEUS}", "description": "", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 0, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 26 }, "hiddenSeries": false, "id": 103, "legend": { "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, "rightSide": true, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 2, "maxDataPoints": 200, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.2", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": false, "expr": "max_over_time( (irate(ClickHouseProfileEvents_UncompressedCacheMisses{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks", "interval": "", "legendFormat": "peaks - {{instance}}", "refId": "A" }, { "exemplar": false, "expr": "rate(ClickHouseProfileEvents_UncompressedCacheMisses{instance=~\"$instance\"}[$__rate_interval]) * $trends", "hide": false, "interval": "", "legendFormat": "trend - {{instance}}", "refId": "C" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Uncompressed Cache Misses", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "transparent": true, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:176", "format": "reqps", "label": "", "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:177", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_PROMETHEUS}", "description": "", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 0, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 34 }, "hiddenSeries": false, "id": 104, "legend": { "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, "rightSide": true, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 2, "maxDataPoints": 200, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.2", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": false, "expr": "max_over_time( (irate(ClickHouseProfileEvents_MarkCacheHits{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks", "interval": "", "legendFormat": "peaks - {{instance}}", "refId": "A" }, { "exemplar": false, "expr": "rate(ClickHouseProfileEvents_MarkCacheHits{instance=~\"$instance\"}[$__rate_interval]) * $trends", "hide": false, "interval": "", "legendFormat": "trend - {{instance}}", "refId": "C" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Mark Cache Hits", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "transparent": true, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:176", "format": "reqps", "label": "", "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:177", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_PROMETHEUS}", "description": "", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 0, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 34 }, "hiddenSeries": false, "id": 105, "legend": { "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, "rightSide": true, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 2, "maxDataPoints": 200, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.2", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": false, "expr": "max_over_time( (irate(ClickHouseProfileEvents_MarkCacheMisses{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks", "interval": "", "legendFormat": "peaks - {{instance}}", "refId": "A" }, { "exemplar": false, "expr": "rate(ClickHouseProfileEvents_MarkCacheMisses{instance=~\"$instance\"}[$__rate_interval]) * $trends", "hide": false, "interval": "", "legendFormat": "trend - {{instance}}", "refId": "C" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Mark Cache Misses", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "transparent": true, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:176", "format": "reqps", "label": "", "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:177", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } } ], "title": "Cache", "type": "row" }, { "collapsed": true, "datasource": "${DS_PROMETHEUS}", "gridPos": { "h": 1, "w": 24, "x": 0, "y": 25 }, "id": 90, "panels": [ { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_PROMETHEUS}", "description": "The part is generating now, it is not in data_parts list.", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 0, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 27 }, "hiddenSeries": false, "id": 114, "legend": { "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, "rightSide": true, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 2, "maxDataPoints": 200, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.2", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": false, "expr": "ClickHouseMetrics_PartsTemporary", "interval": "", "legendFormat": "{{instance}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Temporary", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "transparent": true, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:176", "format": "short", "label": "", "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:177", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_PROMETHEUS}", "description": "The part is in data_parts, but not used for SELECTs.", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 0, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 27 }, "hiddenSeries": false, "id": 115, "legend": { "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, "rightSide": true, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 2, "maxDataPoints": 200, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.2", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": false, "expr": "ClickHouseMetrics_PartsPreCommitted", "interval": "", "legendFormat": "{{instance}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Pre commited", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "transparent": true, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:176", "format": "short", "label": "", "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:177", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_PROMETHEUS}", "description": "The part is in data_parts, but not used for SELECTs.", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 0, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 35 }, "hiddenSeries": false, "id": 116, "legend": { "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, "rightSide": true, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 2, "maxDataPoints": 200, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.2", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": false, "expr": "ClickHouseMetrics_PartsCommitted", "interval": "", "legendFormat": "{{instance}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Commited", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "transparent": true, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:176", "format": "short", "label": "", "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:177", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_PROMETHEUS}", "description": "Not active data part, but could be used by only current SELECTs, could be deleted after SELECTs finishes.", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 0, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 35 }, "hiddenSeries": false, "id": 117, "legend": { "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, "rightSide": true, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 2, "maxDataPoints": 200, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.2", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": false, "expr": "ClickHouseMetrics_PartsOutdated", "interval": "", "legendFormat": "{{instance}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Outdated", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "transparent": true, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:176", "format": "short", "label": "", "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:177", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_PROMETHEUS}", "description": "Not active data part with identity refcounter, it is deleting right now by a cleaner.", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 0, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 43 }, "hiddenSeries": false, "id": 118, "legend": { "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, "rightSide": true, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 2, "maxDataPoints": 200, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.2", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": false, "expr": "ClickHouseMetrics_PartsDeleting", "interval": "", "legendFormat": "{{instance}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Deleting", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "transparent": true, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:176", "format": "short", "label": "", "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:177", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_PROMETHEUS}", "description": "Part was moved to another disk and should be deleted in own destructor.", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 0, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 43 }, "hiddenSeries": false, "id": 119, "legend": { "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, "rightSide": true, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 2, "maxDataPoints": 200, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.2", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": false, "expr": "ClickHouseMetrics_PartsDeleteOnDestroy", "interval": "", "legendFormat": "{{instance}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Delete on destroy", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "transparent": true, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:176", "format": "short", "label": "", "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:177", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_PROMETHEUS}", "description": "Wide parts.", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 0, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 51 }, "hiddenSeries": false, "id": 120, "legend": { "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, "rightSide": true, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 2, "maxDataPoints": 200, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.2", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": false, "expr": "ClickHouseMetrics_PartsWide", "interval": "", "legendFormat": "{{instance}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Wide", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "transparent": true, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:176", "format": "short", "label": "", "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:177", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_PROMETHEUS}", "description": "Compact parts.", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 0, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 51 }, "hiddenSeries": false, "id": 122, "legend": { "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, "rightSide": true, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 2, "maxDataPoints": 200, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.2", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": false, "expr": "ClickHouseMetrics_PartsCompact", "interval": "", "legendFormat": "{{instance}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Compact", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "transparent": true, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:176", "format": "short", "label": "", "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:177", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_PROMETHEUS}", "description": "In-memory parts.", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 0, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 59 }, "hiddenSeries": false, "id": 121, "legend": { "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, "rightSide": true, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 2, "maxDataPoints": 200, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.2", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": false, "expr": "ClickHouseMetrics_PartsInMemory", "interval": "", "legendFormat": "{{instance}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "In-memory", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "transparent": true, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:176", "format": "short", "label": "", "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:177", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } } ], "title": "Parts", "type": "row" }, { "collapsed": true, "datasource": "${DS_PROMETHEUS}", "gridPos": { "h": 1, "w": 24, "x": 0, "y": 26 }, "id": 94, "panels": [ { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_PROMETHEUS}", "description": "Number of connections to remote servers sending data that was INSERTed into Distributed tables. Both synchronous and asynchronous mode.", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 0, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 25 }, "hiddenSeries": false, "id": 112, "legend": { "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, "rightSide": true, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 2, "maxDataPoints": 200, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.2", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": false, "expr": "ClickHouseMetrics_DistributedSend", "interval": "", "legendFormat": "{{instance}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Send", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "transparent": true, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:176", "format": "short", "label": "", "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:177", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_PROMETHEUS}", "description": "Number of pending files to process for asynchronous insertion into Distributed tables. Number of files for every shard is summed.", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 0, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 25 }, "hiddenSeries": false, "id": 113, "legend": { "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, "rightSide": true, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 2, "maxDataPoints": 200, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.2", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": false, "expr": "ClickHouseMetrics_DistributedFilesToInsert", "interval": "", "legendFormat": "{{instance}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Files to insert", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "transparent": true, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:176", "format": "short", "label": "", "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:177", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_PROMETHEUS}", "description": "Number of times the INSERT of a block to a Distributed table was throttled due to high number of pending bytes.", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 0, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 33 }, "hiddenSeries": false, "id": 106, "legend": { "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, "rightSide": true, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 2, "maxDataPoints": 200, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.2", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": false, "expr": "max_over_time( (irate(ClickHouseProfileEvents_DistributedDelayedInserts{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks", "interval": "", "legendFormat": "peaks - {{instance}}", "refId": "A" }, { "exemplar": false, "expr": "rate(ClickHouseProfileEvents_DistributedDelayedInserts{instance=~\"$instance\"}[$__rate_interval]) * $trends", "hide": false, "interval": "", "legendFormat": "trend - {{instance}}", "refId": "C" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Distributed delayed inserts", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "transparent": true, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:176", "format": "reqps", "label": "", "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:177", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_PROMETHEUS}", "description": "Number of times the INSERT of a block to a Distributed table was rejected with 'Too many bytes' exception due to high number of pending bytes.", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 0, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 33 }, "hiddenSeries": false, "id": 107, "legend": { "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, "rightSide": true, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 2, "maxDataPoints": 200, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.2", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": false, "expr": "max_over_time( (irate(ClickHouseProfileEvents_DistributedRejectedInserts{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks", "interval": "", "legendFormat": "peaks - {{instance}}", "refId": "A" }, { "exemplar": false, "expr": "rate(ClickHouseProfileEvents_DistributedRejectedInserts{instance=~\"$instance\"}[$__rate_interval]) * $trends", "hide": false, "interval": "", "legendFormat": "trend - {{instance}}", "refId": "C" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Distributed rejected inserts", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "transparent": true, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:176", "format": "reqps", "label": "", "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:177", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_PROMETHEUS}", "description": "Total number of milliseconds spent while the INSERT of a block to a Distributed table was throttled due to high number of pending bytes.", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 0, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 41 }, "hiddenSeries": false, "id": 108, "legend": { "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, "rightSide": true, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 2, "maxDataPoints": 200, "nullPointMode": "null as zero", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.2", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": false, "expr": "increase(ClickHouseProfileEvents_DistributedDelayedInsertsMilliseconds{instance=~\"$instance\"}[$__rate_interval]) / increase(ClickHouseProfileEvents_DistributedDelayedInserts{instance=~\"$instance\"}[$__rate_interval])", "interval": "", "legendFormat": "{{instance}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Distributed delayed inserts blocks (avg)", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "transparent": true, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:176", "format": "µs", "label": "", "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:177", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 1 } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_PROMETHEUS}", "description": "Total count when distributed connection fails with retry", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 0, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 41 }, "hiddenSeries": false, "id": 109, "legend": { "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, "rightSide": true, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 2, "maxDataPoints": 200, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.2", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": false, "expr": "max_over_time( (irate(ClickHouseProfileEvents_DistributedConnectionFailTry{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks", "interval": "", "legendFormat": "peaks - {{instance}}", "refId": "A" }, { "exemplar": false, "expr": "rate(ClickHouseProfileEvents_DistributedConnectionFailTry{instance=~\"$instance\"}[$__rate_interval]) * $trends", "hide": false, "interval": "", "legendFormat": "trend - {{instance}}", "refId": "C" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Distributed connection fail try", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "transparent": true, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:176", "format": "reqps", "label": "", "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:177", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_PROMETHEUS}", "description": "Total count when distributed connection fails after all retries finished", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 0, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 49 }, "hiddenSeries": false, "id": 110, "legend": { "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, "rightSide": true, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 2, "maxDataPoints": 200, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.2", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": false, "expr": "max_over_time( (irate(ClickHouseProfileEvents_DistributedConnectionFailAtAll{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks", "interval": "", "legendFormat": "peaks - {{instance}}", "refId": "A" }, { "exemplar": false, "expr": "rate(ClickHouseProfileEvents_DistributedConnectionFailAtAll{instance=~\"$instance\"}[$__rate_interval]) * $trends", "hide": false, "interval": "", "legendFormat": "trend - {{instance}}", "refId": "C" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Distributed connection fail at all", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "transparent": true, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:176", "format": "reqps", "label": "", "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:177", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_PROMETHEUS}", "description": "", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 0, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 49 }, "hiddenSeries": false, "id": 111, "legend": { "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, "rightSide": true, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 2, "maxDataPoints": 200, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.2", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": false, "expr": "max_over_time( (irate(ClickHouseProfileEvents_DistributedSyncInsertionTimeoutExceeded{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks", "interval": "", "legendFormat": "peaks - {{instance}}", "refId": "A" }, { "exemplar": false, "expr": "rate(ClickHouseProfileEvents_DistributedSyncInsertionTimeoutExceeded{instance=~\"$instance\"}[$__rate_interval]) * $trends", "hide": false, "interval": "", "legendFormat": "trend - {{instance}}", "refId": "C" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Distributed sync insertation timeout", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "transparent": true, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:176", "format": "reqps", "label": "", "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:177", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } } ], "title": "Distributed", "type": "row" }, { "collapsed": true, "datasource": "${DS_PROMETHEUS}", "gridPos": { "h": 1, "w": 24, "x": 0, "y": 27 }, "id": 88, "panels": [ { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_PROMETHEUS}", "description": "Number of active tasks in BackgroundProcessingPool (merges, mutations, or replication queue bookkeeping)", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 0, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 28 }, "hiddenSeries": false, "id": 95, "legend": { "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, "rightSide": true, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 2, "maxDataPoints": 200, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.2", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": false, "expr": "ClickHouseMetrics_BackgroundPoolTask", "interval": "", "legendFormat": "{{instance}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "BackgroundPool task", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "transparent": true, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:176", "format": "short", "label": "", "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:177", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_PROMETHEUS}", "description": "Number of active tasks in BackgroundFetchesPool", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 0, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 28 }, "hiddenSeries": false, "id": 96, "legend": { "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, "rightSide": true, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 2, "maxDataPoints": 200, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.2", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": false, "expr": "ClickHouseMetrics_BackgroundFetchesPoolTask", "interval": "", "legendFormat": "{{instance}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "BackgroundFetchesPool task", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "transparent": true, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:176", "format": "short", "label": "", "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:177", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_PROMETHEUS}", "description": "Number of active tasks in BackgroundProcessingPool for moves", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 0, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 36 }, "hiddenSeries": false, "id": 97, "legend": { "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, "rightSide": true, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 2, "maxDataPoints": 200, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.2", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": false, "expr": "ClickHouseMetrics_BackgroundMovePoolTask", "interval": "", "legendFormat": "{{instance}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "BackgroundMovePool task", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "transparent": true, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:176", "format": "short", "label": "", "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:177", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_PROMETHEUS}", "description": "Number of active tasks in BackgroundSchedulePool. This pool is used for periodic ReplicatedMergeTree tasks, like cleaning old data parts, altering data parts, replica re-initialization, etc.", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 0, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 36 }, "hiddenSeries": false, "id": 98, "legend": { "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, "rightSide": true, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 2, "maxDataPoints": 200, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.2", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": false, "expr": "ClickHouseMetrics_BackgroundSchedulePoolTask", "interval": "", "legendFormat": "{{instance}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "BackgroundSchedulePool task", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "transparent": true, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:176", "format": "short", "label": "", "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:177", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_PROMETHEUS}", "description": "Number of active tasks in BackgroundBufferFlushSchedulePool. This pool is used for periodic Buffer flushes", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 0, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 44 }, "hiddenSeries": false, "id": 99, "legend": { "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, "rightSide": true, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 2, "maxDataPoints": 200, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.2", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": false, "expr": "ClickHouseMetrics_BackgroundBufferFlushSchedulePoolTask", "interval": "", "legendFormat": "{{instance}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "BackgroundBufferFlushSchedulePool task", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "transparent": true, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:176", "format": "short", "label": "", "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:177", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_PROMETHEUS}", "description": "Number of active tasks in BackgroundDistributedSchedulePool. This pool is used for distributed sends that is done in background.", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 0, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 44 }, "hiddenSeries": false, "id": 100, "legend": { "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, "rightSide": true, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 2, "maxDataPoints": 200, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.2", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": false, "expr": "ClickHouseMetrics_BackgroundDistributedSchedulePoolTask", "interval": "", "legendFormat": "{{instance}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "BackgroundDistributedSchedulePool task", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "transparent": true, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:176", "format": "short", "label": "", "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:177", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_PROMETHEUS}", "description": "Number of active tasks in BackgroundProcessingPool for message streaming", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 0, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 52 }, "hiddenSeries": false, "id": 101, "legend": { "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, "rightSide": true, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 2, "maxDataPoints": 200, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.2", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": false, "expr": "ClickHouseMetrics_BackgroundMessageBrokerSchedulePoolTask", "interval": "", "legendFormat": "{{instance}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "BackgroundMessageBrokerSchedulePool task", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "transparent": true, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:176", "format": "short", "label": "", "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:177", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } } ], "title": "Background pool", "type": "row" }, { "collapsed": true, "datasource": "${DS_PROMETHEUS}", "gridPos": { "h": 1, "w": 24, "x": 0, "y": 28 }, "id": 28, "panels": [ { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_PROMETHEUS}", "description": "", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 0, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 29 }, "hiddenSeries": false, "id": 30, "legend": { "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, "rightSide": true, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 2, "maxDataPoints": 200, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.2", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": false, "expr": "max_over_time( (irate(ClickHouseProfileEvents_ZooKeeperInit{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks", "interval": "", "legendFormat": "peaks - {{instance}}", "refId": "A" }, { "exemplar": false, "expr": "rate(ClickHouseProfileEvents_ZooKeeperInit{instance=~\"$instance\"}[$__rate_interval]) * $trends", "hide": false, "interval": "", "legendFormat": "trend - {{instance}}", "refId": "C" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Init", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "transparent": true, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:176", "format": "reqps", "label": "", "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:177", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_PROMETHEUS}", "description": "", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 0, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 29 }, "hiddenSeries": false, "id": 31, "legend": { "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, "rightSide": true, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 2, "maxDataPoints": 200, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.2", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": false, "expr": "max_over_time( (irate(ClickHouseProfileEvents_ZooKeeperTransactions{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks", "interval": "", "legendFormat": "peaks - {{instance}}", "refId": "A" }, { "exemplar": false, "expr": "rate(ClickHouseProfileEvents_ZooKeeperTransactions{instance=~\"$instance\"}[$__rate_interval]) * $trends", "hide": false, "interval": "", "legendFormat": "trend - {{instance}}", "refId": "C" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Transactions", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "transparent": true, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:176", "format": "reqps", "label": "", "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:177", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_PROMETHEUS}", "description": "", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 0, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 37 }, "hiddenSeries": false, "id": 32, "legend": { "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, "rightSide": true, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 2, "maxDataPoints": 200, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.2", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": false, "expr": "max_over_time( (irate(ClickHouseProfileEvents_ZooKeeperList{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks", "interval": "", "legendFormat": "peaks - {{instance}}", "refId": "A" }, { "exemplar": false, "expr": "rate(ClickHouseProfileEvents_ZooKeeperList{instance=~\"$instance\"}[$__rate_interval]) * $trends", "hide": false, "interval": "", "legendFormat": "trend - {{instance}}", "refId": "C" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "List", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "transparent": true, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:176", "format": "reqps", "label": "", "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:177", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_PROMETHEUS}", "description": "", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 0, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 37 }, "hiddenSeries": false, "id": 33, "legend": { "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, "rightSide": true, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 2, "maxDataPoints": 200, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.2", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": false, "expr": "max_over_time( (irate(ClickHouseProfileEvents_ZooKeeperCreate{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks", "interval": "", "legendFormat": "peaks - {{instance}}", "refId": "A" }, { "exemplar": false, "expr": "rate(ClickHouseProfileEvents_ZooKeeperCreate{instance=~\"$instance\"}[$__rate_interval]) * $trends", "hide": false, "interval": "", "legendFormat": "trend - {{instance}}", "refId": "C" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Create", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "transparent": true, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:176", "format": "reqps", "label": "", "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:177", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_PROMETHEUS}", "description": "", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 0, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 45 }, "hiddenSeries": false, "id": 34, "legend": { "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, "rightSide": true, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 2, "maxDataPoints": 200, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.2", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": false, "expr": "max_over_time( (irate(ClickHouseProfileEvents_ZooKeeperRemove{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks", "interval": "", "legendFormat": "peaks - {{instance}}", "refId": "A" }, { "exemplar": false, "expr": "rate(ClickHouseProfileEvents_ZooKeeperRemove{instance=~\"$instance\"}[$__rate_interval]) * $trends", "hide": false, "interval": "", "legendFormat": "trend - {{instance}}", "refId": "C" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Remove", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "transparent": true, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:176", "format": "reqps", "label": "", "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:177", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_PROMETHEUS}", "description": "", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 0, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 45 }, "hiddenSeries": false, "id": 35, "legend": { "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, "rightSide": true, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 2, "maxDataPoints": 200, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.2", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": false, "expr": "max_over_time( (irate(ClickHouseProfileEvents_ZooKeeperExists{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks", "interval": "", "legendFormat": "peaks - {{instance}}", "refId": "A" }, { "exemplar": false, "expr": "rate(ClickHouseProfileEvents_ZooKeeperExists{instance=~\"$instance\"}[$__rate_interval]) * $trends", "hide": false, "interval": "", "legendFormat": "trend - {{instance}}", "refId": "C" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Exists", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "transparent": true, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:176", "format": "reqps", "label": "", "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:177", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_PROMETHEUS}", "description": "", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 0, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 53 }, "hiddenSeries": false, "id": 36, "legend": { "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, "rightSide": true, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 2, "maxDataPoints": 200, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.2", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": false, "expr": "max_over_time( (irate(ClickHouseProfileEvents_ZooKeeperGet{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks", "interval": "", "legendFormat": "peaks - {{instance}}", "refId": "A" }, { "exemplar": false, "expr": "rate(ClickHouseProfileEvents_ZooKeeperGet{instance=~\"$instance\"}[$__rate_interval]) * $trends", "hide": false, "interval": "", "legendFormat": "trend - {{instance}}", "refId": "C" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Get", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "transparent": true, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:176", "format": "reqps", "label": "", "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:177", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_PROMETHEUS}", "description": "", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 0, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 53 }, "hiddenSeries": false, "id": 37, "legend": { "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, "rightSide": true, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 2, "maxDataPoints": 200, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.2", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": false, "expr": "max_over_time( (irate(ClickHouseProfileEvents_ZooKeeperSet{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks", "interval": "", "legendFormat": "peaks - {{instance}}", "refId": "A" }, { "exemplar": false, "expr": "rate(ClickHouseProfileEvents_ZooKeeperSet{instance=~\"$instance\"}[$__rate_interval]) * $trends", "hide": false, "interval": "", "legendFormat": "trend - {{instance}}", "refId": "C" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Set", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "transparent": true, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:176", "format": "reqps", "label": "", "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:177", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_PROMETHEUS}", "description": "", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 0, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 61 }, "hiddenSeries": false, "id": 38, "legend": { "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, "rightSide": true, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 2, "maxDataPoints": 200, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.2", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": false, "expr": "max_over_time( (irate(ClickHouseProfileEvents_ZooKeeperMulti{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks", "interval": "", "legendFormat": "peaks - {{instance}}", "refId": "A" }, { "exemplar": false, "expr": "rate(ClickHouseProfileEvents_ZooKeeperMulti{instance=~\"$instance\"}[$__rate_interval]) * $trends", "hide": false, "interval": "", "legendFormat": "trend - {{instance}}", "refId": "C" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Multi", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "transparent": true, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:176", "format": "reqps", "label": "", "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:177", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_PROMETHEUS}", "description": "", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 0, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 61 }, "hiddenSeries": false, "id": 39, "legend": { "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, "rightSide": true, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 2, "maxDataPoints": 200, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.2", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": false, "expr": "max_over_time( (irate(ClickHouseProfileEvents_ZooKeeperCheck{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks", "interval": "", "legendFormat": "peaks - {{instance}}", "refId": "A" }, { "exemplar": false, "expr": "rate(ClickHouseProfileEvents_ZooKeeperCheck{instance=~\"$instance\"}[$__rate_interval]) * $trends", "hide": false, "interval": "", "legendFormat": "trend - {{instance}}", "refId": "C" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Check", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "transparent": true, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:176", "format": "reqps", "label": "", "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:177", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_PROMETHEUS}", "description": "", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 0, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 69 }, "hiddenSeries": false, "id": 40, "legend": { "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, "rightSide": true, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 2, "maxDataPoints": 200, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.2", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": false, "expr": "max_over_time( (irate(ClickHouseProfileEvents_ZooKeeperClose{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks", "interval": "", "legendFormat": "peaks - {{instance}}", "refId": "A" }, { "exemplar": false, "expr": "rate(ClickHouseProfileEvents_ZooKeeperClose{instance=~\"$instance\"}[$__rate_interval]) * $trends", "hide": false, "interval": "", "legendFormat": "trend - {{instance}}", "refId": "C" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Close", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "transparent": true, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:176", "format": "reqps", "label": "", "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:177", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_PROMETHEUS}", "description": "", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 0, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 69 }, "hiddenSeries": false, "id": 41, "legend": { "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, "rightSide": true, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 2, "maxDataPoints": 200, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.2", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": false, "expr": "max_over_time( (irate(ClickHouseProfileEvents_ZooKeeperWatchResponse{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks", "interval": "", "legendFormat": "peaks - {{instance}}", "refId": "A" }, { "exemplar": false, "expr": "rate(ClickHouseProfileEvents_ZooKeeperWatchResponse{instance=~\"$instance\"}[$__rate_interval]) * $trends", "hide": false, "interval": "", "legendFormat": "trend - {{instance}}", "refId": "C" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Watch response", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "transparent": true, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:176", "format": "reqps", "label": "", "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:177", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_PROMETHEUS}", "description": "", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 0, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 77 }, "hiddenSeries": false, "id": 42, "legend": { "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, "rightSide": true, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 2, "maxDataPoints": 200, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.2", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": false, "expr": "max_over_time( (irate(ClickHouseProfileEvents_ZooKeeperUserExceptions{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks", "interval": "", "legendFormat": "peaks - {{instance}}", "refId": "A" }, { "exemplar": false, "expr": "rate(ClickHouseProfileEvents_ZooKeeperUserExceptions{instance=~\"$instance\"}[$__rate_interval]) * $trends", "hide": false, "interval": "", "legendFormat": "trend - {{instance}}", "refId": "C" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "User Exceptions", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "transparent": true, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:176", "format": "cps", "label": "", "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:177", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_PROMETHEUS}", "description": "", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 0, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 77 }, "hiddenSeries": false, "id": 43, "legend": { "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, "rightSide": true, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 2, "maxDataPoints": 200, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.2", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": false, "expr": "max_over_time( (irate(ClickHouseProfileEvents_ZooKeeperHardwareExceptions{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks", "interval": "", "legendFormat": "peaks - {{instance}}", "refId": "A" }, { "exemplar": false, "expr": "rate(ClickHouseProfileEvents_ZooKeeperHardwareExceptions{instance=~\"$instance\"}[$__rate_interval]) * $trends", "hide": false, "interval": "", "legendFormat": "trend - {{instance}}", "refId": "C" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Hardware Exceptions", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "transparent": true, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:176", "format": "cps", "label": "", "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:177", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_PROMETHEUS}", "description": "", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 0, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 85 }, "hiddenSeries": false, "id": 46, "legend": { "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, "rightSide": true, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 2, "maxDataPoints": 200, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.2", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": false, "expr": "max_over_time( (irate(ClickHouseProfileEvents_ZooKeeperBytesReceived{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks", "interval": "", "legendFormat": "peaks - {{instance}}", "refId": "A" }, { "exemplar": false, "expr": "rate(ClickHouseProfileEvents_ZooKeeperBytesReceived{instance=~\"$instance\"}[$__rate_interval]) * $trends", "hide": false, "interval": "", "legendFormat": "trend - {{instance}}", "refId": "C" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Bytes received", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "transparent": true, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:176", "format": "binBps", "label": "", "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:177", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_PROMETHEUS}", "description": "", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 0, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 85 }, "hiddenSeries": false, "id": 45, "legend": { "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, "rightSide": true, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 2, "maxDataPoints": 200, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.2", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": false, "expr": "max_over_time( (irate(ClickHouseProfileEvents_ZooKeeperBytesSent{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks", "interval": "", "legendFormat": "peaks - {{instance}}", "refId": "A" }, { "exemplar": false, "expr": "rate(ClickHouseProfileEvents_ZooKeeperBytesSent{instance=~\"$instance\"}[$__rate_interval]) * $trends", "hide": false, "interval": "", "legendFormat": "trend - {{instance}}", "refId": "C" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Bytes sent", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "transparent": true, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:176", "format": "binBps", "label": "", "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:177", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_PROMETHEUS}", "description": "Number of sessions (connections) to ZooKeeper. Should be no more than one, because using more than one connection to ZooKeeper may lead to bugs due to lack of linearizability (stale reads) that ZooKeeper consistency model allows.", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 0, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 93 }, "hiddenSeries": false, "id": 48, "legend": { "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, "rightSide": true, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 2, "maxDataPoints": 200, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.2", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": false, "expr": "ClickHouseMetrics_ZooKeeperSession", "interval": "", "legendFormat": "{{instance}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Sessions", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "transparent": true, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:176", "format": "short", "label": "", "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:177", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_PROMETHEUS}", "description": "Number of ephemeral nodes hold in ZooKeeper.", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 0, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 93 }, "hiddenSeries": false, "id": 47, "legend": { "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, "rightSide": true, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 2, "maxDataPoints": 200, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.2", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": false, "expr": "ClickHouseMetrics_EphemeralNode", "interval": "", "legendFormat": "{{instance}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Ephemeral Node", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "transparent": true, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:176", "format": "short", "label": "", "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:177", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_PROMETHEUS}", "description": "Number of watches (event subscriptions) in ZooKeeper.", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 0, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 101 }, "hiddenSeries": false, "id": 50, "legend": { "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, "rightSide": true, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 2, "maxDataPoints": 200, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.2", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": false, "expr": "ClickHouseMetrics_ZooKeeperWatch", "interval": "", "legendFormat": "{{instance}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Watches", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "transparent": true, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:176", "format": "short", "label": "", "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:177", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_PROMETHEUS}", "description": "Number of requests to ZooKeeper in fly.", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 0, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 101 }, "hiddenSeries": false, "id": 49, "legend": { "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, "rightSide": true, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 2, "maxDataPoints": 200, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.2", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": false, "expr": "ClickHouseMetrics_ZooKeeperRequest", "interval": "", "legendFormat": "{{instance}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Requests", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "transparent": true, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:176", "format": "short", "label": "", "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:177", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_PROMETHEUS}", "description": "Number of times an error happened while trying to remove ephemeral node. This is not an issue, because our implementation of ZooKeeper library guarantee that the session will expire and the node will be removed.", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 0, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 109 }, "hiddenSeries": false, "id": 70, "legend": { "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, "rightSide": true, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 2, "maxDataPoints": 200, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.2", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": false, "expr": "max_over_time( (irate(ClickHouseProfileEvents_CannotRemoveEphemeralNode{instance=~\"$instance\"}[2m]))[$__rate_interval:15s] ) * $peaks", "interval": "", "legendFormat": "peaks - {{instance}}", "refId": "A" }, { "exemplar": false, "expr": "rate(ClickHouseProfileEvents_CannotRemoveEphemeralNode{instance=~\"$instance\"}[$__rate_interval]) * $trends", "hide": false, "interval": "", "legendFormat": "trend - {{instance}}", "refId": "C" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Remove ephemeral node failed", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "transparent": true, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:176", "format": "cps", "label": "", "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:177", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } } ], "title": "Zookeeper", "type": "row" } ], "refresh": "30s", "schemaVersion": 27, "style": "dark", "tags": [ "ClickHouse", "DB" ], "templating": { "list": [ { "current": { "selected": false, "text": "Prometheus", "value": "Prometheus" }, "description": null, "error": null, "hide": 0, "includeAll": false, "label": "DataSource", "multi": false, "name": "datasource", "options": [], "query": "prometheus", "queryValue": "", "refresh": 1, "regex": "", "skipUrlSync": false, "type": "datasource" }, { "allValue": null, "current": {}, "datasource": "${DS_PROMETHEUS}", "definition": "label_values(ClickHouseAsyncMetrics_Uptime, instance)", "description": null, "error": null, "hide": 0, "includeAll": true, "label": "Node", "multi": true, "name": "instance", "options": [], "query": { "query": "label_values(ClickHouseAsyncMetrics_Uptime, instance)", "refId": "StandardVariableQuery" }, "refresh": 1, "regex": "", "skipUrlSync": false, "sort": 5, "tagValuesQuery": "", "tags": [], "tagsQuery": "", "type": "query", "useTags": false }, { "allValue": null, "current": { "selected": false, "text": "Yes", "value": "1" }, "description": null, "error": null, "hide": 0, "includeAll": false, "label": "Show trends", "multi": false, "name": "trends", "options": [ { "selected": true, "text": "Yes", "value": "1" }, { "selected": false, "text": "No", "value": "null" } ], "query": "Yes : 1, No : null", "queryValue": "", "skipUrlSync": false, "type": "custom" }, { "allValue": null, "current": { "selected": false, "text": "No", "value": "null" }, "description": "Be aware of the points limit per timeseries for Grafana. This option may not work for a time range > 24h", "error": null, "hide": 0, "includeAll": false, "label": "Show peaks", "multi": false, "name": "peaks", "options": [ { "selected": false, "text": "Yes", "value": "1" }, { "selected": true, "text": "No", "value": "null" } ], "query": "Yes : 1, No : null", "queryValue": "", "skipUrlSync": false, "type": "custom" } ] }, "time": { "from": "now-3h", "to": "now" }, "timepicker": {}, "timezone": "", "title": "ClickHouse", "uid": "thEkJB_Mz", "version": 40 }