
{{- $Values := (.helm).Values | default .Values }}
{{- $runbookUrl := ($Values.defaultRules).runbookUrl | default "https://runbooks.prometheus-operator.dev/runbooks" }}
{{- $clusterLabel := ($Values.global).clusterLabel | default "cluster" }}
{{- $additionalGroupByLabels := append $Values.defaultRules.additionalGroupByLabels $clusterLabel }}
{{- $groupLabels := join "," (uniq $additionalGroupByLabels) }}
{{- $grafanaAddr := include "vm-k8s-stack.grafana.addr" . }}
name: kubernetes-storage
rules:
- alert: KubePersistentVolumeFillingUp
  expr: {{ printf "((((kubelet_volume_stats_available_bytes{namespace=~\"%s\",job=\"kubelet\",metrics_path=\"/metrics\"} / kubelet_volume_stats_capacity_bytes{namespace=~\"%s\",job=\"kubelet\",metrics_path=\"/metrics\"}) < 0.03) and (kubelet_volume_stats_used_bytes{namespace=~\"%s\",job=\"kubelet\",metrics_path=\"/metrics\"} > 0)) unless on(%s,namespace,persistentvolumeclaim) (kube_persistentvolumeclaim_access_mode{access_mode=\"ReadOnlyMany\"} == 1)) unless on(%s,namespace,persistentvolumeclaim) (kube_persistentvolumeclaim_labels{label_excluded_from_alerts=\"true\"} == 1)" .targetNamespace .targetNamespace .targetNamespace $groupLabels $groupLabels }}
  annotations:
    description: The PersistentVolume claimed by {{`{{`}} $labels.persistentvolumeclaim {{`}}`}} in Namespace {{`{{`}} $labels.namespace {{`}}`}} {{`{{`}} with $labels.{{ $clusterLabel }} -{{`}}`}} on Cluster {{`{{`}} . {{`}}`}} {{`{{`}}- end {{`}}`}} is only {{`{{`}} $value | humanizePercentage {{`}}`}} free.
    runbook_url: {{ $runbookUrl }}/kubernetes/kubepersistentvolumefillingup
    summary: PersistentVolume is filling up.
  condition: true
  for: 1m
  labels:
    severity: critical
- alert: KubePersistentVolumeFillingUp
  expr: {{ printf "(((((kubelet_volume_stats_available_bytes{namespace=~\"%s\",job=\"kubelet\",metrics_path=\"/metrics\"} / kubelet_volume_stats_capacity_bytes{namespace=~\"%s\",job=\"kubelet\",metrics_path=\"/metrics\"}) < 0.15) and (kubelet_volume_stats_used_bytes{namespace=~\"%s\",job=\"kubelet\",metrics_path=\"/metrics\"} > 0)) and (predict_linear(kubelet_volume_stats_available_bytes{namespace=~\"%s\",job=\"kubelet\",metrics_path=\"/metrics\"}[6h], 345600) < 0)) unless on(%s,namespace,persistentvolumeclaim) (kube_persistentvolumeclaim_access_mode{access_mode=\"ReadOnlyMany\"} == 1)) unless on(%s,namespace,persistentvolumeclaim) (kube_persistentvolumeclaim_labels{label_excluded_from_alerts=\"true\"} == 1)" .targetNamespace .targetNamespace .targetNamespace .targetNamespace $groupLabels $groupLabels }}
  annotations:
    description: Based on recent sampling, the PersistentVolume claimed by {{`{{`}} $labels.persistentvolumeclaim {{`}}`}} in Namespace {{`{{`}} $labels.namespace {{`}}`}} {{`{{`}} with $labels.{{ $clusterLabel }} -{{`}}`}} on Cluster {{`{{`}} . {{`}}`}} {{`{{`}}- end {{`}}`}} is expected to fill up within four days. Currently {{`{{`}} $value | humanizePercentage {{`}}`}} is available.
    runbook_url: {{ $runbookUrl }}/kubernetes/kubepersistentvolumefillingup
    summary: PersistentVolume is filling up.
  condition: true
  for: 1h
  labels:
    severity: warning
- alert: KubePersistentVolumeInodesFillingUp
  expr: {{ printf "((((kubelet_volume_stats_inodes_free{namespace=~\"%s\",job=\"kubelet\",metrics_path=\"/metrics\"} / kubelet_volume_stats_inodes{namespace=~\"%s\",job=\"kubelet\",metrics_path=\"/metrics\"}) < 0.03) and (kubelet_volume_stats_inodes_used{namespace=~\"%s\",job=\"kubelet\",metrics_path=\"/metrics\"} > 0)) unless on(%s,namespace,persistentvolumeclaim) (kube_persistentvolumeclaim_access_mode{access_mode=\"ReadOnlyMany\"} == 1)) unless on(%s,namespace,persistentvolumeclaim) (kube_persistentvolumeclaim_labels{label_excluded_from_alerts=\"true\"} == 1)" .targetNamespace .targetNamespace .targetNamespace $groupLabels $groupLabels }}
  annotations:
    description: The PersistentVolume claimed by {{`{{`}} $labels.persistentvolumeclaim {{`}}`}} in Namespace {{`{{`}} $labels.namespace {{`}}`}} {{`{{`}} with $labels.{{ $clusterLabel }} -{{`}}`}} on Cluster {{`{{`}} . {{`}}`}} {{`{{`}}- end {{`}}`}} only has {{`{{`}} $value | humanizePercentage {{`}}`}} free inodes.
    runbook_url: {{ $runbookUrl }}/kubernetes/kubepersistentvolumeinodesfillingup
    summary: PersistentVolumeInodes are filling up.
  condition: true
  for: 1m
  labels:
    severity: critical
- alert: KubePersistentVolumeInodesFillingUp
  expr: {{ printf "(((((kubelet_volume_stats_inodes_free{namespace=~\"%s\",job=\"kubelet\",metrics_path=\"/metrics\"} / kubelet_volume_stats_inodes{namespace=~\"%s\",job=\"kubelet\",metrics_path=\"/metrics\"}) < 0.15) and (kubelet_volume_stats_inodes_used{namespace=~\"%s\",job=\"kubelet\",metrics_path=\"/metrics\"} > 0)) and (predict_linear(kubelet_volume_stats_inodes_free{namespace=~\"%s\",job=\"kubelet\",metrics_path=\"/metrics\"}[6h], 345600) < 0)) unless on(%s,namespace,persistentvolumeclaim) (kube_persistentvolumeclaim_access_mode{access_mode=\"ReadOnlyMany\"} == 1)) unless on(%s,namespace,persistentvolumeclaim) (kube_persistentvolumeclaim_labels{label_excluded_from_alerts=\"true\"} == 1)" .targetNamespace .targetNamespace .targetNamespace .targetNamespace $groupLabels $groupLabels }}
  annotations:
    description: Based on recent sampling, the PersistentVolume claimed by {{`{{`}} $labels.persistentvolumeclaim {{`}}`}} in Namespace {{`{{`}} $labels.namespace {{`}}`}} {{`{{`}} with $labels.{{ $clusterLabel }} -{{`}}`}} on Cluster {{`{{`}} . {{`}}`}} {{`{{`}}- end {{`}}`}} is expected to run out of inodes within four days. Currently {{`{{`}} $value | humanizePercentage {{`}}`}} of its inodes are free.
    runbook_url: {{ $runbookUrl }}/kubernetes/kubepersistentvolumeinodesfillingup
    summary: PersistentVolumeInodes are filling up.
  condition: true
  for: 1h
  labels:
    severity: warning
- alert: KubePersistentVolumeErrors
  expr: kube_persistentvolume_status_phase{phase=~"Failed|Pending",job="kube-state-metrics"} > 0
  annotations:
    description: The persistent volume {{`{{`}} $labels.persistentvolume {{`}}`}} {{`{{`}} with $labels.{{ $clusterLabel }} -{{`}}`}} on Cluster {{`{{`}} . {{`}}`}} {{`{{`}}- end {{`}}`}} has status {{`{{`}} $labels.phase {{`}}`}}.
    runbook_url: {{ $runbookUrl }}/kubernetes/kubepersistentvolumeerrors
    summary: PersistentVolume is having issues with provisioning.
  condition: true
  for: 5m
  labels:
    severity: critical
condition: true
