
{{- $Values := (.helm).Values | default .Values }}
{{- $runbookUrl := ($Values.defaultRules).runbookUrl | default "https://runbooks.prometheus-operator.dev/runbooks" }}
{{- $clusterLabel := ($Values.global).clusterLabel | default "cluster" }}
{{- $additionalGroupByLabels := append $Values.defaultRules.additionalGroupByLabels $clusterLabel }}
{{- $groupLabels := join "," (uniq $additionalGroupByLabels) }}
{{- $grafanaAddr := include "vm-k8s-stack.grafana.addr" . }}
name: etcd
rules:
- alert: etcdMembersDown
  expr: max(sum(up{job=~".*etcd.*"} ==bool 0) without(instance) or count(sum(rate(etcd_network_peer_sent_failures_total{job=~".*etcd.*"}[120s])) without(instance) > 0.01) without(To)) without(endpoint) > 0
  annotations:
    description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": members are down ({{`{{`}} $value {{`}}`}}).'
    summary: etcd cluster members are down.
  condition: true
- alert: etcdInsufficientMembers
  expr: sum(up{job=~".*etcd.*"} ==bool 1) without(instance) < ((count(up{job=~".*etcd.*"}) without(instance) + 1) / 2)
  annotations:
    description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": insufficient members ({{`{{`}} $value {{`}}`}}).'
    summary: etcd cluster has insufficient number of members.
  condition: true
- alert: etcdNoLeader
  expr: etcd_server_has_leader{job=~".*etcd.*"} == 0
  annotations:
    description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": member {{`{{`}} $labels.instance {{`}}`}} has no leader.'
    summary: etcd cluster has no leader.
  condition: true
- alert: etcdHighNumberOfLeaderChanges
  expr: increase((max(etcd_server_leader_changes_seen_total{job=~".*etcd.*"}) without(instance) or (0 * absent(etcd_server_leader_changes_seen_total{job=~".*etcd.*"})))[15m:1m]) >= 4
  annotations:
    description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": {{`{{`}} $value {{`}}`}} leader changes within the last 15 minutes. Frequent elections may be a sign of insufficient resources, high network latency, or disruptions by other components and should be investigated.'
    summary: etcd cluster has high number of leader changes.
  condition: true
- alert: etcdHighNumberOfFailedGRPCRequests
  expr: ((100 * sum(rate(grpc_server_handled_total{job=~".*etcd.*",grpc_code=~"Unknown|FailedPrecondition|ResourceExhausted|Internal|Unavailable|DataLoss|DeadlineExceeded"}[5m])) without(grpc_type,grpc_code)) / sum(rate(grpc_server_handled_total{job=~".*etcd.*"}[5m])) without(grpc_type,grpc_code)) > 1
  annotations:
    description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": {{`{{`}} $value {{`}}`}}% of requests for {{`{{`}} $labels.grpc_method {{`}}`}} failed on etcd instance {{`{{`}} $labels.instance {{`}}`}}.'
    summary: etcd cluster has high number of failed grpc requests.
  condition: true
- alert: etcdHighNumberOfFailedGRPCRequests
  expr: ((100 * sum(rate(grpc_server_handled_total{job=~".*etcd.*",grpc_code=~"Unknown|FailedPrecondition|ResourceExhausted|Internal|Unavailable|DataLoss|DeadlineExceeded"}[5m])) without(grpc_type,grpc_code)) / sum(rate(grpc_server_handled_total{job=~".*etcd.*"}[5m])) without(grpc_type,grpc_code)) > 5
  annotations:
    description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": {{`{{`}} $value {{`}}`}}% of requests for {{`{{`}} $labels.grpc_method {{`}}`}} failed on etcd instance {{`{{`}} $labels.instance {{`}}`}}.'
    summary: etcd cluster has high number of failed grpc requests.
  condition: true
- alert: etcdGRPCRequestsSlow
  expr: histogram_quantile(0.99, sum(rate(grpc_server_handling_seconds_bucket{job=~".*etcd.*",grpc_method!="Defragment",grpc_type="unary"}[5m])) without(grpc_type)) > 0.15
  annotations:
    description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": 99th percentile of gRPC requests is {{`{{`}} $value {{`}}`}}s on etcd instance {{`{{`}} $labels.instance {{`}}`}} for {{`{{`}} $labels.grpc_method {{`}}`}} method.'
    summary: etcd grpc requests are slow
  condition: true
- alert: etcdMemberCommunicationSlow
  expr: histogram_quantile(0.99, rate(etcd_network_peer_round_trip_time_seconds_bucket{job=~".*etcd.*"}[5m])) > 0.15
  annotations:
    description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": member communication with {{`{{`}} $labels.To {{`}}`}} is taking {{`{{`}} $value {{`}}`}}s on etcd instance {{`{{`}} $labels.instance {{`}}`}}.'
    summary: etcd cluster member communication is slow.
  condition: true
- alert: etcdHighNumberOfFailedProposals
  expr: rate(etcd_server_proposals_failed_total{job=~".*etcd.*"}[15m]) > 5
  annotations:
    description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": {{`{{`}} $value {{`}}`}} proposal failures within the last 30 minutes on etcd instance {{`{{`}} $labels.instance {{`}}`}}.'
    summary: etcd cluster has high number of proposal failures.
  condition: true
- alert: etcdHighFsyncDurations
  expr: histogram_quantile(0.99, rate(etcd_disk_wal_fsync_duration_seconds_bucket{job=~".*etcd.*"}[5m])) > 0.5
  annotations:
    description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": 99th percentile fsync durations are {{`{{`}} $value {{`}}`}}s on etcd instance {{`{{`}} $labels.instance {{`}}`}}.'
    summary: etcd cluster 99th percentile fsync durations are too high.
  condition: true
- alert: etcdHighFsyncDurations
  expr: histogram_quantile(0.99, rate(etcd_disk_wal_fsync_duration_seconds_bucket{job=~".*etcd.*"}[5m])) > 1
  annotations:
    description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": 99th percentile fsync durations are {{`{{`}} $value {{`}}`}}s on etcd instance {{`{{`}} $labels.instance {{`}}`}}.'
    summary: etcd cluster 99th percentile fsync durations are too high.
  condition: true
- alert: etcdHighCommitDurations
  expr: histogram_quantile(0.99, rate(etcd_disk_backend_commit_duration_seconds_bucket{job=~".*etcd.*"}[5m])) > 0.25
  annotations:
    description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": 99th percentile commit durations {{`{{`}} $value {{`}}`}}s on etcd instance {{`{{`}} $labels.instance {{`}}`}}.'
    summary: etcd cluster 99th percentile commit durations are too high.
  condition: true
- alert: etcdDatabaseQuotaLowSpace
  expr: ((last_over_time(etcd_mvcc_db_total_size_in_bytes{job=~".*etcd.*"}[5m]) / last_over_time(etcd_server_quota_backend_bytes{job=~".*etcd.*"}[5m])) * 100) > 95
  annotations:
    description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": database size exceeds the defined quota on etcd instance {{`{{`}} $labels.instance {{`}}`}}, please defrag or increase the quota as the writes to etcd will be disabled when it is full.'
    summary: etcd cluster database is running full.
  condition: true
- alert: etcdExcessiveDatabaseGrowth
  expr: predict_linear(etcd_mvcc_db_total_size_in_bytes{job=~".*etcd.*"}[4h], 14400) > etcd_server_quota_backend_bytes{job=~".*etcd.*"}
  annotations:
    description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": Predicting running out of disk space in the next four hours, based on write observations within the past four hours on etcd instance {{`{{`}} $labels.instance {{`}}`}}, please check as it might be disruptive.'
    summary: etcd cluster database growing very fast.
  condition: true
- alert: etcdDatabaseHighFragmentationRatio
  expr: ((last_over_time(etcd_mvcc_db_total_size_in_use_in_bytes{job=~".*etcd.*"}[5m]) / last_over_time(etcd_mvcc_db_total_size_in_bytes{job=~".*etcd.*"}[5m])) < 0.5) and (etcd_mvcc_db_total_size_in_use_in_bytes{job=~".*etcd.*"} > 104857600)
  annotations:
    description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": database size in use on instance {{`{{`}} $labels.instance {{`}}`}} is {{`{{`}} $value | humanizePercentage {{`}}`}} of the actual allocated disk space, please run defragmentation (e.g. etcdctl defrag) to retrieve the unused fragmented disk space.'
    runbook_url: https://etcd.io/docs/v3.5/op-guide/maintenance/#defragmentation
    summary: etcd database size in use is less than 50% of the actual allocated storage.
  condition: true
condition: {{ ($Values.kubeEtcd).enabled }}
