
{{- $Values := (.helm).Values | default .Values }}
{{- $runbookUrl := ($Values.defaultRules).runbookUrl | default "https://runbooks.prometheus-operator.dev/runbooks" }}
{{- $clusterLabel := ($Values.global).clusterLabel | default "cluster" }}
{{- $additionalGroupByLabels := append $Values.defaultRules.additionalGroupByLabels $clusterLabel }}
{{- $groupLabels := join "," (uniq $additionalGroupByLabels) }}
{{- $grafanaAddr := include "vm-k8s-stack.grafana.addr" . }}
name: kubernetes-system-kubelet
rules:
- alert: KubeNodeNotReady
  expr: {{ printf "(kube_node_status_condition{job=\"kube-state-metrics\",condition=\"Ready\",status=\"true\"} == 0) and on(%s,node) (kube_node_spec_unschedulable{job=\"kube-state-metrics\"} == 0)" $groupLabels }}
  annotations:
    description: '{{`{{`}} $labels.node {{`}}`}} has been unready for more than 15 minutes on cluster {{`{{`}} $labels.{{ $clusterLabel }} {{`}}`}}.'
    runbook_url: {{ $runbookUrl }}/kubernetes/kubenodenotready
    summary: Node is not ready.
  condition: true
  for: 15m
  labels:
    severity: warning
- alert: KubeNodePressure
  expr: {{ printf "(kube_node_status_condition{job=\"kube-state-metrics\",condition=~\"(MemoryPressure|DiskPressure|PIDPressure)\",status=\"true\"} == 1) and on(%s,node) (kube_node_spec_unschedulable{job=\"kube-state-metrics\"} == 0)" $groupLabels }}
  annotations:
    description: '{{`{{`}} $labels.node {{`}}`}} on cluster {{`{{`}} $labels.{{ $clusterLabel }} {{`}}`}} has active Condition {{`{{`}} $labels.condition {{`}}`}}. This is caused by resource usage exceeding eviction thresholds.'
    runbook_url: {{ $runbookUrl }}/kubernetes/kubenodepressure
    summary: Node has as active Condition.
  condition: true
  for: 10m
  labels:
    severity: info
- alert: KubeNodeUnreachable
  expr: (kube_node_spec_taint{job="kube-state-metrics",key="node.kubernetes.io/unreachable",effect="NoSchedule"} unless ignoring(key,value) kube_node_spec_taint{job="kube-state-metrics",key=~"ToBeDeletedByClusterAutoscaler|cloud.google.com/impending-node-termination|aws-node-termination-handler/spot-itn"}) == 1
  annotations:
    description: '{{`{{`}} $labels.node {{`}}`}} is unreachable and some workloads may be rescheduled on cluster {{`{{`}} $labels.{{ $clusterLabel }} {{`}}`}}.'
    runbook_url: {{ $runbookUrl }}/kubernetes/kubenodeunreachable
    summary: Node is unreachable.
  condition: true
  for: 15m
  labels:
    severity: warning
- alert: KubeletTooManyPods
  expr: {{ printf "((max(kubelet_running_pods{job=\"kubelet\",metrics_path=\"/metrics\"} > 1) by(%s,instance) * on(%s,instance) group_left(node) max(kubelet_node_name{job=\"kubelet\",metrics_path=\"/metrics\"}) by(%s,instance,node)) / on(%s,node) group_left() max(kube_node_status_capacity{job=\"kube-state-metrics\",resource=\"pods\"} != 1) by(%s,node)) > 0.95" $groupLabels $groupLabels $groupLabels $groupLabels $groupLabels }}
  annotations:
    description: Kubelet '{{`{{`}} $labels.node {{`}}`}}' is running at {{`{{`}} $value | humanizePercentage {{`}}`}} of its Pod capacity on cluster {{`{{`}} $labels.{{ $clusterLabel }} {{`}}`}}.
    runbook_url: {{ $runbookUrl }}/kubernetes/kubelettoomanypods
    summary: Kubelet is running at capacity.
  condition: true
  for: 15m
  labels:
    severity: info
- alert: KubeNodeReadinessFlapping
  expr: {{ printf "(sum(changes(kube_node_status_condition{job=\"kube-state-metrics\",status=\"true\",condition=\"Ready\"}[15m])) by(%s,node) > 2) and on(%s,node) (kube_node_spec_unschedulable{job=\"kube-state-metrics\"} == 0)" $groupLabels $groupLabels }}
  annotations:
    description: The readiness status of node {{`{{`}} $labels.node {{`}}`}} has changed {{`{{`}} $value {{`}}`}} times in the last 15 minutes on cluster {{`{{`}} $labels.{{ $clusterLabel }} {{`}}`}}.
    runbook_url: {{ $runbookUrl }}/kubernetes/kubenodereadinessflapping
    summary: Node readiness status is flapping.
  condition: true
  for: 15m
  labels:
    severity: warning
- alert: KubeNodeEviction
  expr: {{ printf "(sum(rate(kubelet_evictions{job=\"kubelet\",metrics_path=\"/metrics\"}[15m])) by(%s,eviction_signal,instance) * on(%s,instance) group_left(node) max(kubelet_node_name{job=\"kubelet\",metrics_path=\"/metrics\"}) by(%s,instance,node)) > 0" $groupLabels $groupLabels $groupLabels }}
  annotations:
    description: Node {{`{{`}} $labels.node {{`}}`}} on {{`{{`}} $labels.{{ $clusterLabel }} {{`}}`}} is evicting Pods due to {{`{{`}} $labels.eviction_signal {{`}}`}}.  Eviction occurs when eviction thresholds are crossed, typically caused by Pods exceeding RAM/ephemeral-storage limits.
    runbook_url: {{ $runbookUrl }}/kubernetes/kubenodeeviction
    summary: Node is evicting pods.
  condition: true
  for: 0s
  labels:
    severity: info
- alert: KubeletPlegDurationHigh
  expr: node_quantile:kubelet_pleg_relist_duration_seconds:histogram_quantile{quantile="0.99"} >= 10
  annotations:
    description: The Kubelet Pod Lifecycle Event Generator has a 99th percentile duration of {{`{{`}} $value {{`}}`}} seconds on node {{`{{`}} $labels.node {{`}}`}} on cluster {{`{{`}} $labels.{{ $clusterLabel }} {{`}}`}}.
    runbook_url: {{ $runbookUrl }}/kubernetes/kubeletplegdurationhigh
    summary: Kubelet Pod Lifecycle Event Generator is taking too long to relist.
  condition: true
  for: 5m
  labels:
    severity: warning
- alert: KubeletPodStartUpLatencyHigh
  expr: {{ printf "(histogram_quantile(0.99, sum(topk(1, rate(kubelet_pod_worker_duration_seconds_bucket{job=\"kubelet\",metrics_path=\"/metrics\"}[5m])) by(%s,instance,le,operation_type)) by(%s,instance,le)) * on(%s,instance) group_left(node) topk(1, kubelet_node_name{job=\"kubelet\",metrics_path=\"/metrics\"}) by(%s,instance,node)) > 60" $groupLabels $groupLabels $groupLabels $groupLabels }}
  annotations:
    description: Kubelet Pod startup 99th percentile latency is {{`{{`}} $value {{`}}`}} seconds on node {{`{{`}} $labels.node {{`}}`}} on cluster {{`{{`}} $labels.{{ $clusterLabel }} {{`}}`}}.
    runbook_url: {{ $runbookUrl }}/kubernetes/kubeletpodstartuplatencyhigh
    summary: Kubelet Pod startup latency is too high.
  condition: true
  for: 15m
  labels:
    severity: warning
- alert: KubeletClientCertificateExpiration
  expr: kubelet_certificate_manager_client_ttl_seconds < 604800
  annotations:
    description: Client certificate for Kubelet on node {{`{{`}} $labels.node {{`}}`}} expires in {{`{{`}} $value | humanizeDuration {{`}}`}} on cluster {{`{{`}} $labels.{{ $clusterLabel }} {{`}}`}}.
    runbook_url: {{ $runbookUrl }}/kubernetes/kubeletclientcertificateexpiration
    summary: Kubelet client certificate is about to expire.
  condition: true
  labels:
    severity: warning
- alert: KubeletClientCertificateExpiration
  expr: kubelet_certificate_manager_client_ttl_seconds < 86400
  annotations:
    description: Client certificate for Kubelet on node {{`{{`}} $labels.node {{`}}`}} expires in {{`{{`}} $value | humanizeDuration {{`}}`}} on cluster {{`{{`}} $labels.{{ $clusterLabel }} {{`}}`}}.
    runbook_url: {{ $runbookUrl }}/kubernetes/kubeletclientcertificateexpiration
    summary: Kubelet client certificate is about to expire.
  condition: true
  labels:
    severity: critical
- alert: KubeletServerCertificateExpiration
  expr: kubelet_certificate_manager_server_ttl_seconds < 604800
  annotations:
    description: Server certificate for Kubelet on node {{`{{`}} $labels.node {{`}}`}} expires in {{`{{`}} $value | humanizeDuration {{`}}`}} on cluster {{`{{`}} $labels.{{ $clusterLabel }} {{`}}`}}.
    runbook_url: {{ $runbookUrl }}/kubernetes/kubeletservercertificateexpiration
    summary: Kubelet server certificate is about to expire.
  condition: true
  labels:
    severity: warning
- alert: KubeletServerCertificateExpiration
  expr: kubelet_certificate_manager_server_ttl_seconds < 86400
  annotations:
    description: Server certificate for Kubelet on node {{`{{`}} $labels.node {{`}}`}} expires in {{`{{`}} $value | humanizeDuration {{`}}`}} on cluster {{`{{`}} $labels.{{ $clusterLabel }} {{`}}`}}.
    runbook_url: {{ $runbookUrl }}/kubernetes/kubeletservercertificateexpiration
    summary: Kubelet server certificate is about to expire.
  condition: true
  labels:
    severity: critical
- alert: KubeletClientCertificateRenewalErrors
  expr: increase(kubelet_certificate_manager_client_expiration_renew_errors[5m]) > 0
  annotations:
    description: Kubelet on node {{`{{`}} $labels.node {{`}}`}} has failed to renew its client certificate ({{`{{`}} $value | humanize {{`}}`}} errors in the last 5 minutes) on cluster {{`{{`}} $labels.{{ $clusterLabel }} {{`}}`}}.
    runbook_url: {{ $runbookUrl }}/kubernetes/kubeletclientcertificaterenewalerrors
    summary: Kubelet has failed to renew its client certificate.
  condition: true
  for: 15m
  labels:
    severity: warning
- alert: KubeletServerCertificateRenewalErrors
  expr: increase(kubelet_server_expiration_renew_errors[5m]) > 0
  annotations:
    description: Kubelet on node {{`{{`}} $labels.node {{`}}`}} has failed to renew its server certificate ({{`{{`}} $value | humanize {{`}}`}} errors in the last 5 minutes) on cluster {{`{{`}} $labels.{{ $clusterLabel }} {{`}}`}}.
    runbook_url: {{ $runbookUrl }}/kubernetes/kubeletservercertificaterenewalerrors
    summary: Kubelet has failed to renew its server certificate.
  condition: true
  for: 15m
  labels:
    severity: warning
- alert: KubeletDown
  expr: {{ printf "count(kube_node_info{job=\"kube-state-metrics\"}) by(%s) unless on(%s) count(up{job=\"kubelet\",metrics_path=\"/metrics\"} == 1) by(%s)" $groupLabels $groupLabels $groupLabels }}
  annotations:
    description: Kubelet has disappeared from Prometheus target discovery on cluster {{`{{`}} $labels.{{ $clusterLabel }} {{`}}`}}.
    runbook_url: {{ $runbookUrl }}/kubernetes/kubeletdown
    summary: Target disappeared from Prometheus target discovery.
  condition: {{ ($Values.kubelet).enabled }}
  for: 15m
  labels:
    severity: critical
condition: true
