
{{- $Values := (.helm).Values | default .Values }}
{{- $runbookUrl := ($Values.defaultRules).runbookUrl | default "https://runbooks.prometheus-operator.dev/runbooks" }}
{{- $clusterLabel := ($Values.global).clusterLabel | default "cluster" }}
{{- $additionalGroupByLabels := append $Values.defaultRules.additionalGroupByLabels $clusterLabel }}
{{- $groupLabels := join "," (uniq $additionalGroupByLabels) }}
{{- $grafanaAddr := include "vm-k8s-stack.grafana.addr" . }}
name: node-exporter
rules:
- alert: NodeFilesystemSpaceFillingUp
  expr: {{ printf "((((node_filesystem_avail_bytes{job=\"%s\",fstype!=\"\",mountpoint!=\"\"} / node_filesystem_size_bytes{job=\"%s\",fstype!=\"\",mountpoint!=\"\"}) * 100) < 15) and (predict_linear(node_filesystem_avail_bytes{job=\"%s\",fstype!=\"\",mountpoint!=\"\"}[6h], 86400) < 0)) and (node_filesystem_readonly{job=\"%s\",fstype!=\"\",mountpoint!=\"\"} == 0)" (include "vm-k8s-stack.nodeExporter.name" .) (include "vm-k8s-stack.nodeExporter.name" .) (include "vm-k8s-stack.nodeExporter.name" .) (include "vm-k8s-stack.nodeExporter.name" .) }}
  annotations:
    description: Filesystem on {{`{{`}} $labels.device {{`}}`}}, mounted on {{`{{`}} $labels.mountpoint {{`}}`}}, at {{`{{`}} $labels.instance {{`}}`}} has only {{`{{`}} printf "%.2f" $value {{`}}`}}% available space left and is filling up.
    runbook_url: {{ $runbookUrl }}/node/nodefilesystemspacefillingup
    summary: Filesystem is predicted to run out of space within the next 24 hours.
  condition: true
  for: 1h
  labels:
    severity: warning
- alert: NodeFilesystemSpaceFillingUp
  expr: {{ printf "((((node_filesystem_avail_bytes{job=\"%s\",fstype!=\"\",mountpoint!=\"\"} / node_filesystem_size_bytes{job=\"%s\",fstype!=\"\",mountpoint!=\"\"}) * 100) < 10) and (predict_linear(node_filesystem_avail_bytes{job=\"%s\",fstype!=\"\",mountpoint!=\"\"}[6h], 14400) < 0)) and (node_filesystem_readonly{job=\"%s\",fstype!=\"\",mountpoint!=\"\"} == 0)" (include "vm-k8s-stack.nodeExporter.name" .) (include "vm-k8s-stack.nodeExporter.name" .) (include "vm-k8s-stack.nodeExporter.name" .) (include "vm-k8s-stack.nodeExporter.name" .) }}
  annotations:
    description: Filesystem on {{`{{`}} $labels.device {{`}}`}}, mounted on {{`{{`}} $labels.mountpoint {{`}}`}}, at {{`{{`}} $labels.instance {{`}}`}} has only {{`{{`}} printf "%.2f" $value {{`}}`}}% available space left and is filling up fast.
    runbook_url: {{ $runbookUrl }}/node/nodefilesystemspacefillingup
    summary: Filesystem is predicted to run out of space within the next 4 hours.
  condition: true
  for: 1h
  labels:
    severity: critical
- alert: NodeFilesystemAlmostOutOfSpace
  expr: {{ printf "(((node_filesystem_avail_bytes{job=\"%s\",fstype!=\"\",mountpoint!=\"\"} / node_filesystem_size_bytes{job=\"%s\",fstype!=\"\",mountpoint!=\"\"}) * 100) < 5) and (node_filesystem_readonly{job=\"%s\",fstype!=\"\",mountpoint!=\"\"} == 0)" (include "vm-k8s-stack.nodeExporter.name" .) (include "vm-k8s-stack.nodeExporter.name" .) (include "vm-k8s-stack.nodeExporter.name" .) }}
  annotations:
    description: Filesystem on {{`{{`}} $labels.device {{`}}`}}, mounted on {{`{{`}} $labels.mountpoint {{`}}`}}, at {{`{{`}} $labels.instance {{`}}`}} has only {{`{{`}} printf "%.2f" $value {{`}}`}}% available space left.
    runbook_url: {{ $runbookUrl }}/node/nodefilesystemalmostoutofspace
    summary: Filesystem has less than 5% space left.
  condition: true
  for: 30m
  labels:
    severity: warning
- alert: NodeFilesystemAlmostOutOfSpace
  expr: {{ printf "(((node_filesystem_avail_bytes{job=\"%s\",fstype!=\"\",mountpoint!=\"\"} / node_filesystem_size_bytes{job=\"%s\",fstype!=\"\",mountpoint!=\"\"}) * 100) < 3) and (node_filesystem_readonly{job=\"%s\",fstype!=\"\",mountpoint!=\"\"} == 0)" (include "vm-k8s-stack.nodeExporter.name" .) (include "vm-k8s-stack.nodeExporter.name" .) (include "vm-k8s-stack.nodeExporter.name" .) }}
  annotations:
    description: Filesystem on {{`{{`}} $labels.device {{`}}`}}, mounted on {{`{{`}} $labels.mountpoint {{`}}`}}, at {{`{{`}} $labels.instance {{`}}`}} has only {{`{{`}} printf "%.2f" $value {{`}}`}}% available space left.
    runbook_url: {{ $runbookUrl }}/node/nodefilesystemalmostoutofspace
    summary: Filesystem has less than 3% space left.
  condition: true
  for: 30m
  labels:
    severity: critical
- alert: NodeFilesystemFilesFillingUp
  expr: {{ printf "((((node_filesystem_files_free{job=\"%s\",fstype!=\"\",mountpoint!=\"\"} / node_filesystem_files{job=\"%s\",fstype!=\"\",mountpoint!=\"\"}) * 100) < 40) and (predict_linear(node_filesystem_files_free{job=\"%s\",fstype!=\"\",mountpoint!=\"\"}[6h], 86400) < 0)) and (node_filesystem_readonly{job=\"%s\",fstype!=\"\",mountpoint!=\"\"} == 0)" (include "vm-k8s-stack.nodeExporter.name" .) (include "vm-k8s-stack.nodeExporter.name" .) (include "vm-k8s-stack.nodeExporter.name" .) (include "vm-k8s-stack.nodeExporter.name" .) }}
  annotations:
    description: Filesystem on {{`{{`}} $labels.device {{`}}`}}, mounted on {{`{{`}} $labels.mountpoint {{`}}`}}, at {{`{{`}} $labels.instance {{`}}`}} has only {{`{{`}} printf "%.2f" $value {{`}}`}}% available inodes left and is filling up.
    runbook_url: {{ $runbookUrl }}/node/nodefilesystemfilesfillingup
    summary: Filesystem is predicted to run out of inodes within the next 24 hours.
  condition: true
  for: 1h
  labels:
    severity: warning
- alert: NodeFilesystemFilesFillingUp
  expr: {{ printf "((((node_filesystem_files_free{job=\"%s\",fstype!=\"\",mountpoint!=\"\"} / node_filesystem_files{job=\"%s\",fstype!=\"\",mountpoint!=\"\"}) * 100) < 20) and (predict_linear(node_filesystem_files_free{job=\"%s\",fstype!=\"\",mountpoint!=\"\"}[6h], 14400) < 0)) and (node_filesystem_readonly{job=\"%s\",fstype!=\"\",mountpoint!=\"\"} == 0)" (include "vm-k8s-stack.nodeExporter.name" .) (include "vm-k8s-stack.nodeExporter.name" .) (include "vm-k8s-stack.nodeExporter.name" .) (include "vm-k8s-stack.nodeExporter.name" .) }}
  annotations:
    description: Filesystem on {{`{{`}} $labels.device {{`}}`}}, mounted on {{`{{`}} $labels.mountpoint {{`}}`}}, at {{`{{`}} $labels.instance {{`}}`}} has only {{`{{`}} printf "%.2f" $value {{`}}`}}% available inodes left and is filling up fast.
    runbook_url: {{ $runbookUrl }}/node/nodefilesystemfilesfillingup
    summary: Filesystem is predicted to run out of inodes within the next 4 hours.
  condition: true
  for: 1h
  labels:
    severity: critical
- alert: NodeFilesystemAlmostOutOfFiles
  expr: {{ printf "(((node_filesystem_files_free{job=\"%s\",fstype!=\"\",mountpoint!=\"\"} / node_filesystem_files{job=\"%s\",fstype!=\"\",mountpoint!=\"\"}) * 100) < 5) and (node_filesystem_readonly{job=\"%s\",fstype!=\"\",mountpoint!=\"\"} == 0)" (include "vm-k8s-stack.nodeExporter.name" .) (include "vm-k8s-stack.nodeExporter.name" .) (include "vm-k8s-stack.nodeExporter.name" .) }}
  annotations:
    description: Filesystem on {{`{{`}} $labels.device {{`}}`}}, mounted on {{`{{`}} $labels.mountpoint {{`}}`}}, at {{`{{`}} $labels.instance {{`}}`}} has only {{`{{`}} printf "%.2f" $value {{`}}`}}% available inodes left.
    runbook_url: {{ $runbookUrl }}/node/nodefilesystemalmostoutoffiles
    summary: Filesystem has less than 5% inodes left.
  condition: true
  for: 1h
  labels:
    severity: warning
- alert: NodeFilesystemAlmostOutOfFiles
  expr: {{ printf "(((node_filesystem_files_free{job=\"%s\",fstype!=\"\",mountpoint!=\"\"} / node_filesystem_files{job=\"%s\",fstype!=\"\",mountpoint!=\"\"}) * 100) < 3) and (node_filesystem_readonly{job=\"%s\",fstype!=\"\",mountpoint!=\"\"} == 0)" (include "vm-k8s-stack.nodeExporter.name" .) (include "vm-k8s-stack.nodeExporter.name" .) (include "vm-k8s-stack.nodeExporter.name" .) }}
  annotations:
    description: Filesystem on {{`{{`}} $labels.device {{`}}`}}, mounted on {{`{{`}} $labels.mountpoint {{`}}`}}, at {{`{{`}} $labels.instance {{`}}`}} has only {{`{{`}} printf "%.2f" $value {{`}}`}}% available inodes left.
    runbook_url: {{ $runbookUrl }}/node/nodefilesystemalmostoutoffiles
    summary: Filesystem has less than 3% inodes left.
  condition: true
  for: 1h
  labels:
    severity: critical
- alert: NodeNetworkReceiveErrs
  expr: {{ printf "(rate(node_network_receive_errs_total{job=\"%s\"}[2m]) / rate(node_network_receive_packets_total{job=\"%s\"}[2m])) > 0.01" (include "vm-k8s-stack.nodeExporter.name" .) (include "vm-k8s-stack.nodeExporter.name" .) }}
  annotations:
    description: '{{`{{`}} $labels.instance {{`}}`}} interface {{`{{`}} $labels.device {{`}}`}} has encountered {{`{{`}} printf "%.0f" $value {{`}}`}} receive errors in the last two minutes.'
    runbook_url: {{ $runbookUrl }}/node/nodenetworkreceiveerrs
    summary: Network interface is reporting many receive errors.
  condition: true
  for: 1h
  labels:
    severity: warning
- alert: NodeNetworkTransmitErrs
  expr: {{ printf "(rate(node_network_transmit_errs_total{job=\"%s\"}[2m]) / rate(node_network_transmit_packets_total{job=\"%s\"}[2m])) > 0.01" (include "vm-k8s-stack.nodeExporter.name" .) (include "vm-k8s-stack.nodeExporter.name" .) }}
  annotations:
    description: '{{`{{`}} $labels.instance {{`}}`}} interface {{`{{`}} $labels.device {{`}}`}} has encountered {{`{{`}} printf "%.0f" $value {{`}}`}} transmit errors in the last two minutes.'
    runbook_url: {{ $runbookUrl }}/node/nodenetworktransmiterrs
    summary: Network interface is reporting many transmit errors.
  condition: true
  for: 1h
  labels:
    severity: warning
- alert: NodeHighNumberConntrackEntriesUsed
  expr: {{ printf "(node_nf_conntrack_entries{job=\"%s\"} / node_nf_conntrack_entries_limit) > 0.75" (include "vm-k8s-stack.nodeExporter.name" .) }}
  annotations:
    description: '{{`{{`}} $labels.instance {{`}}`}} {{`{{`}} $value | humanizePercentage {{`}}`}} of conntrack entries are used.'
    runbook_url: {{ $runbookUrl }}/node/nodehighnumberconntrackentriesused
    summary: Number of conntrack are getting close to the limit.
  condition: true
  labels:
    severity: warning
- alert: NodeTextFileCollectorScrapeError
  expr: {{ printf "node_textfile_scrape_error{job=\"%s\"} == 1" (include "vm-k8s-stack.nodeExporter.name" .) }}
  annotations:
    description: Node Exporter text file collector on {{`{{`}} $labels.instance {{`}}`}} failed to scrape.
    runbook_url: {{ $runbookUrl }}/node/nodetextfilecollectorscrapeerror
    summary: Node Exporter text file collector failed to scrape.
  condition: true
  labels:
    severity: warning
- alert: NodeClockSkewDetected
  expr: {{ printf "((node_timex_offset_seconds{job=\"%s\"} > 0.05) and (deriv(node_timex_offset_seconds{job=\"%s\"}[5m]) >= 0)) or ((node_timex_offset_seconds{job=\"%s\"} < -0.05) and (deriv(node_timex_offset_seconds{job=\"%s\"}[5m]) <= 0))" (include "vm-k8s-stack.nodeExporter.name" .) (include "vm-k8s-stack.nodeExporter.name" .) (include "vm-k8s-stack.nodeExporter.name" .) (include "vm-k8s-stack.nodeExporter.name" .) }}
  annotations:
    description: Clock at {{`{{`}} $labels.instance {{`}}`}} is out of sync by more than 0.05s. Ensure NTP is configured correctly on this host.
    runbook_url: {{ $runbookUrl }}/node/nodeclockskewdetected
    summary: Clock skew detected.
  condition: true
  for: 10m
  labels:
    severity: warning
- alert: NodeClockNotSynchronising
  expr: {{ printf "(min_over_time(node_timex_sync_status{job=\"%s\"}[5m]) == 0) and (node_timex_maxerror_seconds{job=\"%s\"} >= 16)" (include "vm-k8s-stack.nodeExporter.name" .) (include "vm-k8s-stack.nodeExporter.name" .) }}
  annotations:
    description: Clock at {{`{{`}} $labels.instance {{`}}`}} is not synchronising. Ensure NTP is configured on this host.
    runbook_url: {{ $runbookUrl }}/node/nodeclocknotsynchronising
    summary: Clock not synchronising.
  condition: true
  for: 10m
  labels:
    severity: warning
- alert: NodeRAIDDegraded
  expr: {{ printf "(node_md_disks_required{job=\"%s\",device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\"} - ignoring(state) node_md_disks{state=\"active\",job=\"%s\",device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\"}) > 0" (include "vm-k8s-stack.nodeExporter.name" .) (include "vm-k8s-stack.nodeExporter.name" .) }}
  annotations:
    description: RAID array '{{`{{`}} $labels.device {{`}}`}}' at {{`{{`}} $labels.instance {{`}}`}} is in degraded state due to one or more disks failures. Number of spare drives is insufficient to fix issue automatically.
    runbook_url: {{ $runbookUrl }}/node/noderaiddegraded
    summary: RAID Array is degraded.
  condition: true
  for: 15m
  labels:
    severity: critical
- alert: NodeRAIDDiskFailure
  expr: {{ printf "node_md_disks{state=\"failed\",job=\"%s\",device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\"} > 0" (include "vm-k8s-stack.nodeExporter.name" .) }}
  annotations:
    description: At least one device in RAID array at {{`{{`}} $labels.instance {{`}}`}} failed. Array '{{`{{`}} $labels.device {{`}}`}}' needs attention and possibly a disk swap.
    runbook_url: {{ $runbookUrl }}/node/noderaiddiskfailure
    summary: Failed device in RAID array.
  condition: true
  labels:
    severity: warning
- alert: NodeFileDescriptorLimit
  expr: {{ printf "((node_filefd_allocated{job=\"%s\"} * 100) / node_filefd_maximum{job=\"%s\"}) > 70" (include "vm-k8s-stack.nodeExporter.name" .) (include "vm-k8s-stack.nodeExporter.name" .) }}
  annotations:
    description: File descriptors limit at {{`{{`}} $labels.instance {{`}}`}} is currently at {{`{{`}} printf "%.2f" $value {{`}}`}}%.
    runbook_url: {{ $runbookUrl }}/node/nodefiledescriptorlimit
    summary: Kernel is predicted to exhaust file descriptors limit soon.
  condition: true
  for: 15m
  labels:
    severity: warning
- alert: NodeFileDescriptorLimit
  expr: {{ printf "((node_filefd_allocated{job=\"%s\"} * 100) / node_filefd_maximum{job=\"%s\"}) > 90" (include "vm-k8s-stack.nodeExporter.name" .) (include "vm-k8s-stack.nodeExporter.name" .) }}
  annotations:
    description: File descriptors limit at {{`{{`}} $labels.instance {{`}}`}} is currently at {{`{{`}} printf "%.2f" $value {{`}}`}}%.
    runbook_url: {{ $runbookUrl }}/node/nodefiledescriptorlimit
    summary: Kernel is predicted to exhaust file descriptors limit soon.
  condition: true
  for: 15m
  labels:
    severity: critical
- alert: NodeCPUHighUsage
  expr: {{ printf "(sum(avg(rate(node_cpu_seconds_total{job=\"%s\",mode!~\"idle|iowait\"}[2m])) without(cpu)) without(mode) * 100) > 90" (include "vm-k8s-stack.nodeExporter.name" .) }}
  annotations:
    description: |
      CPU usage at {{`{{`}} $labels.instance {{`}}`}} has been above 90% for the last 15 minutes, is currently at {{`{{`}} printf "%.2f" $value {{`}}`}}%.
    runbook_url: {{ $runbookUrl }}/node/nodecpuhighusage
    summary: High CPU usage.
  condition: true
  for: 15m
  labels:
    severity: info
- alert: NodeSystemSaturation
  expr: {{ printf "(node_load1{job=\"%s\"} / count(node_cpu_seconds_total{job=\"%s\",mode=\"idle\"}) without(cpu,mode)) > 2" (include "vm-k8s-stack.nodeExporter.name" .) (include "vm-k8s-stack.nodeExporter.name" .) }}
  annotations:
    description: |
      System load per core at {{`{{`}} $labels.instance {{`}}`}} has been above 2 for the last 15 minutes, is currently at {{`{{`}} printf "%.2f" $value {{`}}`}}.
      This might indicate this instance resources saturation and can cause it becoming unresponsive.
    runbook_url: {{ $runbookUrl }}/node/nodesystemsaturation
    summary: System saturated, load per core is very high.
  condition: true
  for: 15m
  labels:
    severity: warning
- alert: NodeMemoryMajorPagesFaults
  expr: {{ printf "rate(node_vmstat_pgmajfault{job=\"%s\"}[5m]) > 500" (include "vm-k8s-stack.nodeExporter.name" .) }}
  annotations:
    description: |
      Memory major pages are occurring at very high rate at {{`{{`}} $labels.instance {{`}}`}}, 500 major page faults per second for the last 15 minutes, is currently at {{`{{`}} printf "%.2f" $value {{`}}`}}.
      Please check that there is enough memory available at this instance.
    runbook_url: {{ $runbookUrl }}/node/nodememorymajorpagesfaults
    summary: Memory major page faults are occurring at very high rate.
  condition: true
  for: 15m
  labels:
    severity: warning
- alert: NodeMemoryHighUtilization
  expr: {{ printf "(100 - ((node_memory_MemAvailable_bytes{job=\"%s\"} / node_memory_MemTotal_bytes{job=\"%s\"}) * 100)) > 90" (include "vm-k8s-stack.nodeExporter.name" .) (include "vm-k8s-stack.nodeExporter.name" .) }}
  annotations:
    description: |
      Memory is filling up at {{`{{`}} $labels.instance {{`}}`}}, has been above 90% for the last 15 minutes, is currently at {{`{{`}} printf "%.2f" $value {{`}}`}}%.
    runbook_url: {{ $runbookUrl }}/node/nodememoryhighutilization
    summary: Host is running out of memory.
  condition: true
  for: 15m
  labels:
    severity: warning
- alert: NodeDiskIOSaturation
  expr: {{ printf "rate(node_disk_io_time_weighted_seconds_total{job=\"%s\",device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\"}[5m]) > 10" (include "vm-k8s-stack.nodeExporter.name" .) }}
  annotations:
    description: |
      Disk IO queue (aqu-sq) is high on {{`{{`}} $labels.device {{`}}`}} at {{`{{`}} $labels.instance {{`}}`}}, has been above 10 for the last 30 minutes, is currently at {{`{{`}} printf "%.2f" $value {{`}}`}}.
      This symptom might indicate disk saturation.
    runbook_url: {{ $runbookUrl }}/node/nodediskiosaturation
    summary: Disk IO queue is high.
  condition: true
  for: 30m
  labels:
    severity: warning
- alert: NodeSystemdServiceFailed
  expr: {{ printf "node_systemd_unit_state{job=\"%s\",state=\"failed\"} == 1" (include "vm-k8s-stack.nodeExporter.name" .) }}
  annotations:
    description: Systemd service {{`{{`}} $labels.name {{`}}`}} has entered failed state at {{`{{`}} $labels.instance {{`}}`}}
    runbook_url: {{ $runbookUrl }}/node/nodesystemdservicefailed
    summary: Systemd service has entered failed state.
  condition: true
  for: 5m
  labels:
    severity: warning
- alert: NodeSystemdServiceCrashlooping
  expr: {{ printf "increase(node_systemd_service_restart_total{job=\"%s\"}[5m]) > 2" (include "vm-k8s-stack.nodeExporter.name" .) }}
  annotations:
    description: Systemd service {{`{{`}} $labels.name {{`}}`}} has being restarted too many times at {{`{{`}} $labels.instance {{`}}`}} for the last 15 minutes. Please check if service is crash looping.
    runbook_url: {{ $runbookUrl }}/node/nodesystemdservicecrashlooping
    summary: Systemd service keeps restaring, possibly crash looping.
  condition: true
  for: 15m
  labels:
    severity: warning
- alert: NodeBondingDegraded
  expr: {{ printf "(node_bonding_slaves{job=\"%s\"} - node_bonding_active{job=\"%s\"}) != 0" (include "vm-k8s-stack.nodeExporter.name" .) (include "vm-k8s-stack.nodeExporter.name" .) }}
  annotations:
    description: Bonding interface {{`{{`}} $labels.master {{`}}`}} on {{`{{`}} $labels.instance {{`}}`}} is in degraded state due to one or more slave failures.
    runbook_url: {{ $runbookUrl }}/node/nodebondingdegraded
    summary: Bonding interface is degraded.
  condition: true
  for: 5m
  labels:
    severity: warning
condition: true
