
{{- $Values := (.helm).Values | default .Values }}
{{- $runbookUrl := ($Values.defaultRules).runbookUrl | default "https://runbooks.prometheus-operator.dev/runbooks" }}
{{- $clusterLabel := ($Values.global).clusterLabel | default "cluster" }}
{{- $additionalGroupByLabels := append $Values.defaultRules.additionalGroupByLabels $clusterLabel }}
{{- $groupLabels := join "," (uniq $additionalGroupByLabels) }}
{{- $grafanaAddr := include "vm-k8s-stack.grafana.addr" . }}
name: kubernetes-system-apiserver
rules:
- alert: KubeClientCertificateExpiration
  expr: {{ printf "(histogram_quantile(0.01, sum(rate(apiserver_client_certificate_expiration_seconds_bucket{job=\"apiserver\"}[5m])) without(namespace,service,endpoint)) < 604800) and on(job,%s,instance) (apiserver_client_certificate_expiration_seconds_count{job=\"apiserver\"} > 0)" $groupLabels }}
  annotations:
    description: A client certificate used to authenticate to kubernetes apiserver is expiring in less than 7.0 days on cluster {{`{{`}} $labels.{{ $clusterLabel }} {{`}}`}}.
    runbook_url: {{ $runbookUrl }}/kubernetes/kubeclientcertificateexpiration
    summary: Client certificate is about to expire.
  condition: true
  for: 5m
  labels:
    severity: warning
- alert: KubeClientCertificateExpiration
  expr: {{ printf "(histogram_quantile(0.01, sum(rate(apiserver_client_certificate_expiration_seconds_bucket{job=\"apiserver\"}[5m])) without(namespace,service,endpoint)) < 86400) and on(job,%s,instance) (apiserver_client_certificate_expiration_seconds_count{job=\"apiserver\"} > 0)" $groupLabels }}
  annotations:
    description: A client certificate used to authenticate to kubernetes apiserver is expiring in less than 24.0 hours on cluster {{`{{`}} $labels.{{ $clusterLabel }} {{`}}`}}.
    runbook_url: {{ $runbookUrl }}/kubernetes/kubeclientcertificateexpiration
    summary: Client certificate is about to expire.
  condition: true
  for: 5m
  labels:
    severity: critical
- alert: KubeAggregatedAPIErrors
  expr: {{ printf "sum(increase(aggregator_unavailable_apiservice_total{job=\"apiserver\"}[1m])) by(%s,instance,name,reason) > 0" $groupLabels }}
  annotations:
    description: Kubernetes aggregated API {{`{{`}} $labels.instance {{`}}`}}/{{`{{`}} $labels.name {{`}}`}} has reported {{`{{`}} $labels.reason {{`}}`}} errors on cluster {{`{{`}} $labels.{{ $clusterLabel }} {{`}}`}}.
    runbook_url: {{ $runbookUrl }}/kubernetes/kubeaggregatedapierrors
    summary: Kubernetes aggregated API has reported errors.
  condition: true
  for: 10m
  labels:
    severity: warning
- alert: KubeAggregatedAPIDown
  expr: {{ printf "((1 - max(avg_over_time(aggregator_unavailable_apiservice{job=\"apiserver\"}[10m])) by(name,namespace,%s)) * 100) < 85" $groupLabels }}
  annotations:
    description: Kubernetes aggregated API {{`{{`}} $labels.name {{`}}`}}/{{`{{`}} $labels.namespace {{`}}`}} has been only {{`{{`}} $value | humanize {{`}}`}}% available over the last 10m on cluster {{`{{`}} $labels.{{ $clusterLabel }} {{`}}`}}.
    runbook_url: {{ $runbookUrl }}/kubernetes/kubeaggregatedapidown
    summary: Kubernetes aggregated API is down.
  condition: true
  for: 5m
  labels:
    severity: warning
- alert: KubeAPIDown
  expr: absent(up{job="apiserver"})
  annotations:
    description: KubeAPI has disappeared from Prometheus target discovery.
    runbook_url: {{ $runbookUrl }}/kubernetes/kubeapidown
    summary: Target disappeared from Prometheus target discovery.
  condition: {{ ($Values.kubeApiServer).enabled }}
  for: 15m
  labels:
    severity: critical
- alert: KubeAPITerminatedRequests
  expr: {{ printf "(sum(rate(apiserver_request_terminations_total{job=\"apiserver\"}[10m])) by(%s) / (sum(rate(apiserver_request_total{job=\"apiserver\"}[10m])) by(%s) + sum(rate(apiserver_request_terminations_total{job=\"apiserver\"}[10m])) by(%s))) > 0.20" $groupLabels $groupLabels $groupLabels }}
  annotations:
    description: The kubernetes apiserver has terminated {{`{{`}} $value | humanizePercentage {{`}}`}} of its incoming requests on cluster {{`{{`}} $labels.{{ $clusterLabel }} {{`}}`}}.
    runbook_url: {{ $runbookUrl }}/kubernetes/kubeapiterminatedrequests
    summary: The kubernetes apiserver has terminated {{`{{`}} $value | humanizePercentage {{`}}`}} of its incoming requests.
  condition: true
  for: 5m
  labels:
    severity: warning
condition: true
