
{{- $Values := (.helm).Values | default .Values }}
{{- $runbookUrl := ($Values.defaultRules).runbookUrl | default "https://runbooks.prometheus-operator.dev/runbooks" }}
{{- $clusterLabel := ($Values.global).clusterLabel | default "cluster" }}
{{- $additionalGroupByLabels := append $Values.defaultRules.additionalGroupByLabels $clusterLabel }}
{{- $groupLabels := join "," (uniq $additionalGroupByLabels) }}
{{- $grafanaAddr := include "vm-k8s-stack.grafana.addr" . }}
name: kubernetes-apps
rules:
- alert: KubePodCrashLooping
  expr: {{ printf "max_over_time(kube_pod_container_status_waiting_reason{reason=\"CrashLoopBackOff\",namespace=~\"%s\",job=\"kube-state-metrics\"}[5m]) >= 1" .targetNamespace }}
  annotations:
    description: 'Pod {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.pod {{`}}`}} ({{`{{`}} $labels.container {{`}}`}}) is in waiting state (reason: "CrashLoopBackOff") on cluster {{`{{`}} $labels.{{ $clusterLabel }} {{`}}`}}.'
    runbook_url: {{ $runbookUrl }}/kubernetes/kubepodcrashlooping
    summary: Pod is crash looping.
  condition: true
  for: 15m
  labels:
    severity: warning
- alert: KubePodNotReady
  expr: {{ printf "sum(max(kube_pod_status_phase{namespace=~\"%s\",job=\"kube-state-metrics\",phase=~\"Pending|Unknown\"}) by(namespace,pod,job,%s) * on(namespace,pod,%s) group_left(owner_kind) topk(1, max(kube_pod_owner{owner_kind!=\"Job\"}) by(namespace,pod,owner_kind,%s)) by(namespace,pod,%s)) by(namespace,pod,job,%s) > 0" .targetNamespace $groupLabels $groupLabels $groupLabels $groupLabels $groupLabels }}
  annotations:
    description: Pod {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.pod {{`}}`}} has been in a non-ready state for longer than 15 minutes on cluster {{`{{`}} $labels.{{ $clusterLabel }} {{`}}`}}.
    runbook_url: {{ $runbookUrl }}/kubernetes/kubepodnotready
    summary: Pod has been in a non-ready state for more than 15 minutes.
  condition: true
  for: 15m
  labels:
    severity: warning
- alert: KubeDeploymentGenerationMismatch
  expr: {{ printf "kube_deployment_status_observed_generation{namespace=~\"%s\",job=\"kube-state-metrics\"} != kube_deployment_metadata_generation{namespace=~\"%s\",job=\"kube-state-metrics\"}" .targetNamespace .targetNamespace }}
  annotations:
    description: Deployment generation for {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.deployment {{`}}`}} does not match, this indicates that the Deployment has failed but has not been rolled back on cluster {{`{{`}} $labels.{{ $clusterLabel }} {{`}}`}}.
    runbook_url: {{ $runbookUrl }}/kubernetes/kubedeploymentgenerationmismatch
    summary: Deployment generation mismatch due to possible roll-back
  condition: true
  for: 15m
  labels:
    severity: warning
- alert: KubeDeploymentReplicasMismatch
  expr: {{ printf "(kube_deployment_spec_replicas{namespace=~\"%s\",job=\"kube-state-metrics\"} > kube_deployment_status_replicas_available{namespace=~\"%s\",job=\"kube-state-metrics\"}) and (changes(kube_deployment_status_replicas_updated{namespace=~\"%s\",job=\"kube-state-metrics\"}[10m]) == 0)" .targetNamespace .targetNamespace .targetNamespace }}
  annotations:
    description: Deployment {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.deployment {{`}}`}} has not matched the expected number of replicas for longer than 15 minutes on cluster {{`{{`}} $labels.{{ $clusterLabel }} {{`}}`}}.
    runbook_url: {{ $runbookUrl }}/kubernetes/kubedeploymentreplicasmismatch
    summary: Deployment has not matched the expected number of replicas.
  condition: true
  for: 15m
  labels:
    severity: warning
- alert: KubeDeploymentRolloutStuck
  expr: {{ printf "kube_deployment_status_condition{condition=\"Progressing\",status=\"false\",namespace=~\"%s\",job=\"kube-state-metrics\"} != 0" .targetNamespace }}
  annotations:
    description: Rollout of deployment {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.deployment {{`}}`}} is not progressing for longer than 15 minutes on cluster {{`{{`}} $labels.{{ $clusterLabel }} {{`}}`}}.
    runbook_url: {{ $runbookUrl }}/kubernetes/kubedeploymentrolloutstuck
    summary: Deployment rollout is not progressing.
  condition: true
  for: 15m
  labels:
    severity: warning
- alert: KubeStatefulSetReplicasMismatch
  expr: {{ printf "(kube_statefulset_status_replicas_ready{namespace=~\"%s\",job=\"kube-state-metrics\"} != kube_statefulset_replicas{namespace=~\"%s\",job=\"kube-state-metrics\"}) and (changes(kube_statefulset_status_replicas_updated{namespace=~\"%s\",job=\"kube-state-metrics\"}[10m]) == 0)" .targetNamespace .targetNamespace .targetNamespace }}
  annotations:
    description: StatefulSet {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.statefulset {{`}}`}} has not matched the expected number of replicas for longer than 15 minutes on cluster {{`{{`}} $labels.{{ $clusterLabel }} {{`}}`}}.
    runbook_url: {{ $runbookUrl }}/kubernetes/kubestatefulsetreplicasmismatch
    summary: StatefulSet has not matched the expected number of replicas.
  condition: true
  for: 15m
  labels:
    severity: warning
- alert: KubeStatefulSetGenerationMismatch
  expr: {{ printf "kube_statefulset_status_observed_generation{namespace=~\"%s\",job=\"kube-state-metrics\"} != kube_statefulset_metadata_generation{namespace=~\"%s\",job=\"kube-state-metrics\"}" .targetNamespace .targetNamespace }}
  annotations:
    description: StatefulSet generation for {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.statefulset {{`}}`}} does not match, this indicates that the StatefulSet has failed but has not been rolled back on cluster {{`{{`}} $labels.{{ $clusterLabel }} {{`}}`}}.
    runbook_url: {{ $runbookUrl }}/kubernetes/kubestatefulsetgenerationmismatch
    summary: StatefulSet generation mismatch due to possible roll-back
  condition: true
  for: 15m
  labels:
    severity: warning
- alert: KubeStatefulSetUpdateNotRolledOut
  expr: {{ printf "(max(kube_statefulset_status_current_revision{namespace=~\"%s\",job=\"kube-state-metrics\"} unless kube_statefulset_status_update_revision{namespace=~\"%s\",job=\"kube-state-metrics\"}) by(namespace,statefulset,job,%s) * on(namespace,statefulset,job,%s) (kube_statefulset_replicas{namespace=~\"%s\",job=\"kube-state-metrics\"} != kube_statefulset_status_replicas_updated{namespace=~\"%s\",job=\"kube-state-metrics\"})) and on(namespace,statefulset,job,%s) (changes(kube_statefulset_status_replicas_updated{namespace=~\"%s\",job=\"kube-state-metrics\"}[5m]) == 0)" .targetNamespace .targetNamespace $groupLabels $groupLabels .targetNamespace .targetNamespace $groupLabels .targetNamespace }}
  annotations:
    description: StatefulSet {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.statefulset {{`}}`}} update has not been rolled out on cluster {{`{{`}} $labels.{{ $clusterLabel }} {{`}}`}}.
    runbook_url: {{ $runbookUrl }}/kubernetes/kubestatefulsetupdatenotrolledout
    summary: StatefulSet update has not been rolled out.
  condition: true
  for: 15m
  labels:
    severity: warning
- alert: KubeDaemonSetRolloutStuck
  expr: {{ printf "((((kube_daemonset_status_current_number_scheduled{namespace=~\"%s\",job=\"kube-state-metrics\"} != kube_daemonset_status_desired_number_scheduled{namespace=~\"%s\",job=\"kube-state-metrics\"}) or (kube_daemonset_status_number_misscheduled{namespace=~\"%s\",job=\"kube-state-metrics\"} != 0)) or (kube_daemonset_status_updated_number_scheduled{namespace=~\"%s\",job=\"kube-state-metrics\"} != kube_daemonset_status_desired_number_scheduled{namespace=~\"%s\",job=\"kube-state-metrics\"})) or (kube_daemonset_status_number_available{namespace=~\"%s\",job=\"kube-state-metrics\"} != kube_daemonset_status_desired_number_scheduled{namespace=~\"%s\",job=\"kube-state-metrics\"})) and (changes(kube_daemonset_status_updated_number_scheduled{namespace=~\"%s\",job=\"kube-state-metrics\"}[5m]) == 0)" .targetNamespace .targetNamespace .targetNamespace .targetNamespace .targetNamespace .targetNamespace .targetNamespace .targetNamespace }}
  annotations:
    description: DaemonSet {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.daemonset {{`}}`}} has not finished or progressed for at least 15m on cluster {{`{{`}} $labels.{{ $clusterLabel }} {{`}}`}}.
    runbook_url: {{ $runbookUrl }}/kubernetes/kubedaemonsetrolloutstuck
    summary: DaemonSet rollout is stuck.
  condition: true
  for: 15m
  labels:
    severity: warning
- alert: KubeContainerWaiting
  expr: {{ printf "kube_pod_container_status_waiting_reason{reason!=\"CrashLoopBackOff\",namespace=~\"%s\",job=\"kube-state-metrics\"} > 0" .targetNamespace }}
  annotations:
    description: 'pod/{{`{{`}} $labels.pod {{`}}`}} in namespace {{`{{`}} $labels.namespace {{`}}`}} on container {{`{{`}} $labels.container{{`}}`}} has been in waiting state for longer than 1 hour. (reason: "{{`{{`}} $labels.reason {{`}}`}}") on cluster {{`{{`}} $labels.{{ $clusterLabel }} {{`}}`}}.'
    runbook_url: {{ $runbookUrl }}/kubernetes/kubecontainerwaiting
    summary: Pod container waiting longer than 1 hour
  condition: true
  for: 1h
  labels:
    severity: warning
- alert: KubeDaemonSetNotScheduled
  expr: {{ printf "(kube_daemonset_status_desired_number_scheduled{namespace=~\"%s\",job=\"kube-state-metrics\"} - kube_daemonset_status_current_number_scheduled{namespace=~\"%s\",job=\"kube-state-metrics\"}) > 0" .targetNamespace .targetNamespace }}
  annotations:
    description: '{{`{{`}} $value {{`}}`}} Pods of DaemonSet {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.daemonset {{`}}`}} are not scheduled on cluster {{`{{`}} $labels.{{ $clusterLabel }} {{`}}`}}.'
    runbook_url: {{ $runbookUrl }}/kubernetes/kubedaemonsetnotscheduled
    summary: DaemonSet pods are not scheduled.
  condition: true
  for: 10m
  labels:
    severity: warning
- alert: KubeDaemonSetMisScheduled
  expr: {{ printf "kube_daemonset_status_number_misscheduled{namespace=~\"%s\",job=\"kube-state-metrics\"} > 0" .targetNamespace }}
  annotations:
    description: '{{`{{`}} $value {{`}}`}} Pods of DaemonSet {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.daemonset {{`}}`}} are running where they are not supposed to run on cluster {{`{{`}} $labels.{{ $clusterLabel }} {{`}}`}}.'
    runbook_url: {{ $runbookUrl }}/kubernetes/kubedaemonsetmisscheduled
    summary: DaemonSet pods are misscheduled.
  condition: true
  for: 15m
  labels:
    severity: warning
- alert: KubeJobNotCompleted
  expr: {{ printf "(time() - max(kube_job_status_start_time{namespace=~\"%s\",job=\"kube-state-metrics\"} and (kube_job_status_active{namespace=~\"%s\",job=\"kube-state-metrics\"} > 0)) by(namespace,job_name,%s)) > 43200" .targetNamespace .targetNamespace $groupLabels }}
  annotations:
    description: Job {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.job_name {{`}}`}} is taking more than {{`{{`}} "43200" | humanizeDuration {{`}}`}} to complete on cluster {{`{{`}} $labels.{{ $clusterLabel }} {{`}}`}}.
    runbook_url: {{ $runbookUrl }}/kubernetes/kubejobnotcompleted
    summary: Job did not complete in time
  condition: true
  labels:
    severity: warning
- alert: KubeJobFailed
  expr: {{ printf "kube_job_failed{namespace=~\"%s\",job=\"kube-state-metrics\"} > 0" .targetNamespace }}
  annotations:
    description: Job {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.job_name {{`}}`}} failed to complete. Removing failed job after investigation should clear this alert on cluster {{`{{`}} $labels.{{ $clusterLabel }} {{`}}`}}.
    runbook_url: {{ $runbookUrl }}/kubernetes/kubejobfailed
    summary: Job failed to complete.
  condition: true
  for: 15m
  labels:
    severity: warning
- alert: KubeHpaReplicasMismatch
  expr: {{ printf "(((kube_horizontalpodautoscaler_status_desired_replicas{namespace=~\"%s\",job=\"kube-state-metrics\"} != kube_horizontalpodautoscaler_status_current_replicas{namespace=~\"%s\",job=\"kube-state-metrics\"}) and (kube_horizontalpodautoscaler_status_current_replicas{namespace=~\"%s\",job=\"kube-state-metrics\"} > kube_horizontalpodautoscaler_spec_min_replicas{namespace=~\"%s\",job=\"kube-state-metrics\"})) and (kube_horizontalpodautoscaler_status_current_replicas{namespace=~\"%s\",job=\"kube-state-metrics\"} < kube_horizontalpodautoscaler_spec_max_replicas{namespace=~\"%s\",job=\"kube-state-metrics\"})) and (changes(kube_horizontalpodautoscaler_status_current_replicas{namespace=~\"%s\",job=\"kube-state-metrics\"}[15m]) == 0)" .targetNamespace .targetNamespace .targetNamespace .targetNamespace .targetNamespace .targetNamespace .targetNamespace }}
  annotations:
    description: HPA {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.horizontalpodautoscaler  {{`}}`}} has not matched the desired number of replicas for longer than 15 minutes on cluster {{`{{`}} $labels.{{ $clusterLabel }} {{`}}`}}.
    runbook_url: {{ $runbookUrl }}/kubernetes/kubehpareplicasmismatch
    summary: HPA has not matched desired number of replicas.
  condition: true
  for: 15m
  labels:
    severity: warning
- alert: KubeHpaMaxedOut
  expr: {{ printf "(kube_horizontalpodautoscaler_status_current_replicas{namespace=~\"%s\",job=\"kube-state-metrics\"} == kube_horizontalpodautoscaler_spec_max_replicas{namespace=~\"%s\",job=\"kube-state-metrics\"}) and on(namespace,horizontalpodautoscaler,%s) (kube_horizontalpodautoscaler_spec_max_replicas{namespace=~\"%s\",job=\"kube-state-metrics\"} != kube_horizontalpodautoscaler_spec_min_replicas{namespace=~\"%s\",job=\"kube-state-metrics\"})" .targetNamespace .targetNamespace $groupLabels .targetNamespace .targetNamespace }}
  annotations:
    description: HPA {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.horizontalpodautoscaler  {{`}}`}} has been running at max replicas for longer than 15 minutes on cluster {{`{{`}} $labels.{{ $clusterLabel }} {{`}}`}}.
    runbook_url: {{ $runbookUrl }}/kubernetes/kubehpamaxedout
    summary: HPA is running at max replicas
  condition: true
  for: 15m
  labels:
    severity: warning
- alert: KubePdbNotEnoughHealthyPods
  expr: {{ printf "(kube_poddisruptionbudget_status_desired_healthy{namespace=~\"%s\",job=\"kube-state-metrics\"} - kube_poddisruptionbudget_status_current_healthy{namespace=~\"%s\",job=\"kube-state-metrics\"}) > 0" .targetNamespace .targetNamespace }}
  annotations:
    description: PDB {{`{{`}} $labels.{{ $clusterLabel }} {{`}}`}}/{{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.poddisruptionbudget {{`}}`}} expects {{`{{`}} $value {{`}}`}} more healthy pods. The desired number of healthy pods has not been met for at least 15m.
    runbook_url: {{ $runbookUrl }}/kubernetes/kubepdbnotenoughhealthypods
    summary: PDB does not have enough healthy pods.
  condition: true
  for: 15m
  labels:
    severity: warning
condition: true
