
{{- $Values := (.helm).Values | default .Values }}
{{- $multicluster := ((($Values.grafana).sidecar).dashboards).multicluster | default false }}
{{- $defaultDatasource := "prometheus" -}}
{{- $clusterLabel := ($Values.global).clusterLabel | default "cluster" }}
{{- range (($Values.defaultDatasources).victoriametrics).datasources | default list }}
  {{- if and .isDefault .type }}{{ $defaultDatasource = .type }}{{- end }}
{{- end }}
title: etcd
templating:
  list:
  - name: datasource
    label: Data Source
    type: datasource
    query: {{ $defaultDatasource }}
  - name: cluster
    label: cluster
    datasource:
      type: {{ $defaultDatasource }}
      uid: ${datasource}
    type: {{ ternary "query" "constant" $multicluster }}
    query: {{ ternary (printf "label_values(etcd_server_has_leader{job=~\".*etcd.*\"}, %s)" $clusterLabel) ".*" $multicluster }}
    hide: {{ ternary 0 2 $multicluster }}
    refresh: 2
editable: false
timezone: {{ default "utc" ($Values.defaultDashboards).defaultTimezone }}
tags:
- etcd-mixin
- vm-k8s-stack
panels:
- title: Up
  targets:
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $datasource
    expr: {{ printf "sum(etcd_server_has_leader{job=~\".*etcd.*\",%s=~\"$cluster\"}) by(%s)" $clusterLabel $clusterLabel }}
    legendFormat: |
      {{`{{`}}cluster{{`}}`}} - {{`{{`}}namespace{{`}}`}}
  datasource:
    type: datasource
    uid: -- Mixed --
  gridPos:
    h: 7
    w: 6
    x: 0
    "y": 0
  id: 1
  interval: 1m
  options:
    colorMode: none
    graphMode: none
    reduceOptions:
      calcs:
      - lastNotNull
  pluginVersion: v10.0.0
  type: stat
- title: RPC rate
  targets:
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $datasource
    expr: {{ printf "sum(rate(grpc_server_started_total{job=~\".*etcd.*\",%s=~\"$cluster\",grpc_type=\"unary\"}[$__rate_interval])) by(%s)" $clusterLabel $clusterLabel }}
    legendFormat: RPC rate
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $datasource
    expr: {{ printf "sum(rate(grpc_server_handled_total{job=~\".*etcd.*\",%s=~\"$cluster\",grpc_type=\"unary\",grpc_code=~\"Unknown|FailedPrecondition|ResourceExhausted|Internal|Unavailable|DataLoss|DeadlineExceeded\"}[$__rate_interval])) by(%s)" $clusterLabel $clusterLabel }}
    legendFormat: RPC failed rate
  datasource:
    type: datasource
    uid: -- Mixed --
  fieldConfig:
    defaults:
      custom:
        fillOpacity: 0
        lineWidth: 2
        showPoints: never
      unit: ops
  gridPos:
    h: 7
    w: 10
    x: 6
    "y": 0
  id: 2
  interval: 1m
  pluginVersion: v10.0.0
  type: timeseries
- title: Active streams
  targets:
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $datasource
    expr: {{ printf "sum(grpc_server_started_total{job=~\".*etcd.*\",%s=~\"$cluster\",grpc_service=\"etcdserverpb.Watch\",grpc_type=\"bidi_stream\"}) by(%s) - sum(grpc_server_handled_total{%s=~\"$cluster\",grpc_service=\"etcdserverpb.Watch\",grpc_type=\"bidi_stream\"}) by(%s)" $clusterLabel $clusterLabel $clusterLabel $clusterLabel }}
    legendFormat: Watch streams
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $datasource
    expr: {{ printf "sum(grpc_server_started_total{job=~\".*etcd.*\",%s=~\"$cluster\",grpc_service=\"etcdserverpb.Lease\",grpc_type=\"bidi_stream\"}) by(%s) - sum(grpc_server_handled_total{%s=~\"$cluster\",grpc_service=\"etcdserverpb.Lease\",grpc_type=\"bidi_stream\"}) by(%s)" $clusterLabel $clusterLabel $clusterLabel $clusterLabel }}
    legendFormat: Lease streams
  datasource:
    type: datasource
    uid: -- Mixed --
  fieldConfig:
    defaults:
      custom:
        fillOpacity: 0
        lineWidth: 2
        showPoints: never
  gridPos:
    h: 7
    w: 8
    x: 16
    "y": 0
  id: 3
  interval: 1m
  pluginVersion: v10.0.0
  type: timeseries
- title: DB size
  targets:
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $datasource
    expr: {{ printf "etcd_mvcc_db_total_size_in_bytes{job=~\".*etcd.*\",%s=~\"$cluster\"}" $clusterLabel }}
    legendFormat: '{{`{{`}}instance{{`}}`}} DB size'
  datasource:
    type: datasource
    uid: -- Mixed --
  fieldConfig:
    defaults:
      custom:
        fillOpacity: 0
        lineWidth: 2
        showPoints: never
      unit: bytes
  gridPos:
    h: 7
    w: 8
    x: 0
    "y": 25
  id: 4
  interval: 1m
  pluginVersion: v10.0.0
  type: timeseries
- title: Disk sync duration
  targets:
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $datasource
    expr: {{ printf "histogram_quantile(0.99, sum(rate(etcd_disk_wal_fsync_duration_seconds_bucket{job=~\".*etcd.*\",%s=~\"$cluster\"}[$__rate_interval])) by(instance,le,%s))" $clusterLabel $clusterLabel }}
    legendFormat: '{{`{{`}}instance{{`}}`}} WAL fsync'
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $datasource
    expr: {{ printf "histogram_quantile(0.99, sum(rate(etcd_disk_backend_commit_duration_seconds_bucket{job=~\".*etcd.*\",%s=~\"$cluster\"}[$__rate_interval])) by(instance,le,%s))" $clusterLabel $clusterLabel }}
    legendFormat: '{{`{{`}}instance{{`}}`}} DB fsync'
  datasource:
    type: datasource
    uid: -- Mixed --
  fieldConfig:
    defaults:
      custom:
        fillOpacity: 0
        lineWidth: 2
        showPoints: never
      unit: s
  gridPos:
    h: 7
    w: 8
    x: 8
    "y": 25
  id: 5
  interval: 1m
  pluginVersion: v10.0.0
  type: timeseries
- title: Memory
  targets:
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $datasource
    expr: {{ printf "process_resident_memory_bytes{job=~\".*etcd.*\",%s=~\"$cluster\"}" $clusterLabel }}
    legendFormat: '{{`{{`}}instance{{`}}`}} resident memory'
  datasource:
    type: datasource
    uid: -- Mixed --
  fieldConfig:
    defaults:
      custom:
        fillOpacity: 0
        lineWidth: 2
        showPoints: never
      unit: bytes
  gridPos:
    h: 7
    w: 8
    x: 16
    "y": 25
  id: 6
  interval: 1m
  pluginVersion: v10.0.0
  type: timeseries
- title: Client traffic in
  targets:
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $datasource
    expr: {{ printf "rate(etcd_network_client_grpc_received_bytes_total{job=~\".*etcd.*\",%s=~\"$cluster\"}[$__rate_interval])" $clusterLabel }}
    legendFormat: '{{`{{`}}instance{{`}}`}} client traffic in'
  datasource:
    type: datasource
    uid: -- Mixed --
  fieldConfig:
    defaults:
      custom:
        fillOpacity: 0
        lineWidth: 2
        showPoints: never
      unit: Bps
  gridPos:
    h: 7
    w: 6
    x: 0
    "y": 50
  id: 7
  interval: 1m
  pluginVersion: v10.0.0
  type: timeseries
- title: Client traffic out
  targets:
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $datasource
    expr: {{ printf "rate(etcd_network_client_grpc_sent_bytes_total{job=~\".*etcd.*\",%s=~\"$cluster\"}[$__rate_interval])" $clusterLabel }}
    legendFormat: '{{`{{`}}instance{{`}}`}} client traffic out'
  datasource:
    type: datasource
    uid: -- Mixed --
  fieldConfig:
    defaults:
      custom:
        fillOpacity: 0
        lineWidth: 2
        showPoints: never
      unit: Bps
  gridPos:
    h: 7
    w: 6
    x: 6
    "y": 50
  id: 8
  interval: 1m
  pluginVersion: v10.0.0
  type: timeseries
- title: Peer traffic in
  targets:
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $datasource
    expr: {{ printf "sum(rate(etcd_network_peer_received_bytes_total{job=~\".*etcd.*\",%s=~\"$cluster\"}[$__rate_interval])) by(instance,%s)" $clusterLabel $clusterLabel }}
    legendFormat: '{{`{{`}}instance{{`}}`}} peer traffic in'
  datasource:
    type: datasource
    uid: -- Mixed --
  fieldConfig:
    defaults:
      custom:
        fillOpacity: 0
        lineWidth: 2
        showPoints: never
      unit: Bps
  gridPos:
    h: 7
    w: 6
    x: 12
    "y": 50
  id: 9
  interval: 1m
  pluginVersion: v10.0.0
  type: timeseries
- title: Peer traffic out
  targets:
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $datasource
    expr: {{ printf "sum(rate(etcd_network_peer_sent_bytes_total{job=~\".*etcd.*\",%s=~\"$cluster\"}[$__rate_interval])) by(instance,%s)" $clusterLabel $clusterLabel }}
    legendFormat: '{{`{{`}}instance{{`}}`}} peer traffic out'
  datasource:
    type: datasource
    uid: -- Mixed --
  fieldConfig:
    defaults:
      custom:
        fillOpacity: 0
        lineWidth: 2
        showPoints: never
      unit: Bps
  gridPos:
    h: 7
    w: 6
    x: 18
    "y": 50
  id: 10
  interval: 1m
  pluginVersion: v10.0.0
  type: timeseries
- title: Raft proposals
  targets:
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $datasource
    expr: {{ printf "changes(etcd_server_leader_changes_seen_total{job=~\".*etcd.*\",%s=~\"$cluster\"}[1d])" $clusterLabel }}
    legendFormat: '{{`{{`}}instance{{`}}`}} total leader elections per day'
  datasource:
    type: datasource
    uid: -- Mixed --
  fieldConfig:
    defaults:
      custom:
        fillOpacity: 0
        lineWidth: 2
        showPoints: never
  gridPos:
    h: 7
    w: 8
    x: 0
    "y": 75
  id: 11
  interval: 1m
  pluginVersion: v10.0.0
  type: timeseries
- title: Total leader elections per day
  targets:
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $datasource
    expr: {{ printf "changes(etcd_server_leader_changes_seen_total{job=~\".*etcd.*\",%s=~\"$cluster\"}[1d])" $clusterLabel }}
    legendFormat: '{{`{{`}}instance{{`}}`}} total leader elections per day'
  datasource:
    type: datasource
    uid: -- Mixed --
  fieldConfig:
    defaults:
      custom:
        fillOpacity: 0
        lineWidth: 2
        showPoints: never
  gridPos:
    h: 7
    w: 8
    x: 8
    "y": 75
  id: 12
  interval: 1m
  pluginVersion: v10.0.0
  type: timeseries
- title: Peer round trip time
  targets:
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $datasource
    expr: {{ printf "histogram_quantile(0.99, sum(rate(etcd_network_peer_round_trip_time_seconds_bucket{job=~\".*etcd.*\",%s=~\"$cluster\"}[$__rate_interval])) by(instance,le,%s))" $clusterLabel $clusterLabel }}
    legendFormat: '{{`{{`}}instance{{`}}`}} peer round trip time'
  datasource:
    type: datasource
    uid: -- Mixed --
  fieldConfig:
    defaults:
      custom:
        fillOpacity: 0
        lineWidth: 2
        showPoints: never
      unit: s
  gridPos:
    h: 7
    w: 8
    x: 16
    "y": 75
  id: 13
  interval: 1m
  pluginVersion: v10.0.0
  type: timeseries
condition: {{ ($Values.kubeEtcd).enabled }}
description: etcd sample Grafana dashboard with Prometheus
refresh: 10s
schemaVersion: 36
time:
  from: now-15m
  to: now
uid: c2f4e12cdf69feb95caa41a5a1b423d9
