
{{- $Values := (.helm).Values | default .Values }}
{{- $runbookUrl := ($Values.defaultRules).runbookUrl | default "https://runbooks.prometheus-operator.dev/runbooks" }}
{{- $clusterLabel := ($Values.global).clusterLabel | default "cluster" }}
{{- $additionalGroupByLabels := append $Values.defaultRules.additionalGroupByLabels $clusterLabel }}
{{- $groupLabels := join "," (uniq $additionalGroupByLabels) }}
{{- $grafanaAddr := include "vm-k8s-stack.grafana.addr" . }}
name: kubernetes-resources
rules:
- alert: KubeCPUOvercommit
  expr: {{ printf "(((sum(namespace_cpu:kube_pod_container_resource_requests:sum) by(%s) - sum(kube_node_status_allocatable{job=\"kube-state-metrics\",resource=\"cpu\"}) by(%s)) > 0) and (count(max(kube_node_role{job=\"kube-state-metrics\",role=\"control-plane\"}) by(%s,node)) by(%s) < 3)) or ((sum(namespace_cpu:kube_pod_container_resource_requests:sum) by(%s) - ((sum(kube_node_status_allocatable{job=\"kube-state-metrics\",resource=\"cpu\"}) by(%s) - max(kube_node_status_allocatable{job=\"kube-state-metrics\",resource=\"cpu\"}) by(%s)) > 0)) > 0)" $groupLabels $groupLabels $groupLabels $groupLabels $groupLabels $groupLabels $groupLabels }}
  annotations:
    description: Cluster {{`{{`}} $labels.{{ $clusterLabel }} {{`}}`}} has overcommitted CPU resource requests for Pods by {{`{{`}} printf "%.2f" $value {{`}}`}} CPU shares and cannot tolerate node failure.
    runbook_url: {{ $runbookUrl }}/kubernetes/kubecpuovercommit
    summary: Cluster has overcommitted CPU resource requests.
  condition: true
  for: 10m
  labels:
    severity: warning
- alert: KubeMemoryOvercommit
  expr: {{ printf "(((sum(namespace_memory:kube_pod_container_resource_requests:sum) by(%s) - sum(kube_node_status_allocatable{job=\"kube-state-metrics\",resource=\"memory\"}) by(%s)) > 0) and (count(max(kube_node_role{job=\"kube-state-metrics\",role=\"control-plane\"}) by(%s,node)) by(%s) < 3)) or ((sum(namespace_memory:kube_pod_container_resource_requests:sum) by(%s) - ((sum(kube_node_status_allocatable{job=\"kube-state-metrics\",resource=\"memory\"}) by(%s) - max(kube_node_status_allocatable{job=\"kube-state-metrics\",resource=\"memory\"}) by(%s)) > 0)) > 0)" $groupLabels $groupLabels $groupLabels $groupLabels $groupLabels $groupLabels $groupLabels }}
  annotations:
    description: Cluster {{`{{`}} $labels.{{ $clusterLabel }} {{`}}`}} has overcommitted memory resource requests for Pods by {{`{{`}} $value | humanize {{`}}`}} bytes and cannot tolerate node failure.
    runbook_url: {{ $runbookUrl }}/kubernetes/kubememoryovercommit
    summary: Cluster has overcommitted memory resource requests.
  condition: true
  for: 10m
  labels:
    severity: warning
- alert: KubeCPUQuotaOvercommit
  expr: {{ printf "(sum(min(kube_resourcequota{job=\"kube-state-metrics\",type=\"hard\",resource=~\"(cpu|requests.cpu)\"}) without(resource)) by(%s) / sum(kube_node_status_allocatable{resource=\"cpu\",job=\"kube-state-metrics\"}) by(%s)) > 1.5" $groupLabels $groupLabels }}
  annotations:
    description: Cluster {{`{{`}} $labels.{{ $clusterLabel }} {{`}}`}} has overcommitted CPU resource requests for Namespaces.
    runbook_url: {{ $runbookUrl }}/kubernetes/kubecpuquotaovercommit
    summary: Cluster has overcommitted CPU resource requests.
  condition: true
  for: 5m
  labels:
    severity: warning
- alert: KubeMemoryQuotaOvercommit
  expr: {{ printf "(sum(min(kube_resourcequota{job=\"kube-state-metrics\",type=\"hard\",resource=~\"(memory|requests.memory)\"}) without(resource)) by(%s) / sum(kube_node_status_allocatable{resource=\"memory\",job=\"kube-state-metrics\"}) by(%s)) > 1.5" $groupLabels $groupLabels }}
  annotations:
    description: Cluster {{`{{`}} $labels.{{ $clusterLabel }} {{`}}`}} has overcommitted memory resource requests for Namespaces.
    runbook_url: {{ $runbookUrl }}/kubernetes/kubememoryquotaovercommit
    summary: Cluster has overcommitted memory resource requests.
  condition: true
  for: 5m
  labels:
    severity: warning
- alert: KubeQuotaAlmostFull
  expr: {{ printf "((max(kube_resourcequota{job=\"kube-state-metrics\",type=\"used\"}) without(instance,job,type) / on(%s,namespace,resource,resourcequota) group_left() (max(kube_resourcequota{job=\"kube-state-metrics\",type=\"hard\"}) without(instance,job,type) > 0)) > 0.9) < 1" $groupLabels }}
  annotations:
    description: Namespace {{`{{`}} $labels.namespace {{`}}`}} is using {{`{{`}} $value | humanizePercentage {{`}}`}} of its {{`{{`}} $labels.resource {{`}}`}} quota on cluster {{`{{`}} $labels.{{ $clusterLabel }} {{`}}`}}.
    runbook_url: {{ $runbookUrl }}/kubernetes/kubequotaalmostfull
    summary: Namespace quota is going to be full.
  condition: true
  for: 15m
  labels:
    severity: info
- alert: KubeQuotaFullyUsed
  expr: {{ printf "(max(kube_resourcequota{job=\"kube-state-metrics\",type=\"used\"}) without(instance,job,type) / on(%s,namespace,resource,resourcequota) group_left() (max(kube_resourcequota{job=\"kube-state-metrics\",type=\"hard\"}) without(instance,job,type) > 0)) == 1" $groupLabels }}
  annotations:
    description: Namespace {{`{{`}} $labels.namespace {{`}}`}} is using {{`{{`}} $value | humanizePercentage {{`}}`}} of its {{`{{`}} $labels.resource {{`}}`}} quota on cluster {{`{{`}} $labels.{{ $clusterLabel }} {{`}}`}}.
    runbook_url: {{ $runbookUrl }}/kubernetes/kubequotafullyused
    summary: Namespace quota is fully used.
  condition: true
  for: 15m
  labels:
    severity: info
- alert: KubeQuotaExceeded
  expr: {{ printf "(max(kube_resourcequota{job=\"kube-state-metrics\",type=\"used\"}) without(instance,job,type) / on(%s,namespace,resource,resourcequota) group_left() (max(kube_resourcequota{job=\"kube-state-metrics\",type=\"hard\"}) without(instance,job,type) > 0)) > 1" $groupLabels }}
  annotations:
    description: Namespace {{`{{`}} $labels.namespace {{`}}`}} is using {{`{{`}} $value | humanizePercentage {{`}}`}} of its {{`{{`}} $labels.resource {{`}}`}} quota on cluster {{`{{`}} $labels.{{ $clusterLabel }} {{`}}`}}.
    runbook_url: {{ $runbookUrl }}/kubernetes/kubequotaexceeded
    summary: Namespace quota has exceeded the limits.
  condition: true
  for: 15m
  labels:
    severity: warning
- alert: CPUThrottlingHigh
  expr: {{ printf "(sum(topk(1, increase(container_cpu_cfs_throttled_periods_total{container!=\"\",job=\"kubelet\",metrics_path=\"/metrics/cadvisor\"}[5m])) by(%s,namespace,pod,container,instance)) without(id,metrics_path,name,image,endpoint,job,node) / on(%s,namespace,pod,container,instance) group_left() sum(topk(1, increase(container_cpu_cfs_periods_total{job=\"kubelet\",metrics_path=\"/metrics/cadvisor\"}[5m])) by(%s,namespace,pod,container,instance)) without(id,metrics_path,name,image,endpoint,job,node)) > 0.25" $groupLabels $groupLabels $groupLabels }}
  annotations:
    description: '{{`{{`}} $value | humanizePercentage {{`}}`}} throttling of CPU in namespace {{`{{`}} $labels.namespace {{`}}`}} for container {{`{{`}} $labels.container {{`}}`}} in pod {{`{{`}} $labels.pod {{`}}`}} on cluster {{`{{`}} $labels.{{ $clusterLabel }} {{`}}`}}.'
    runbook_url: {{ $runbookUrl }}/kubernetes/cputhrottlinghigh
    summary: Processes experience elevated CPU throttling.
  condition: true
  for: 15m
  labels:
    severity: info
condition: true
