
{{- $Values := (.helm).Values | default .Values }}
{{- $runbookUrl := ($Values.defaultRules).runbookUrl | default "https://runbooks.prometheus-operator.dev/runbooks" }}
{{- $clusterLabel := ($Values.global).clusterLabel | default "cluster" }}
{{- $additionalGroupByLabels := append $Values.defaultRules.additionalGroupByLabels $clusterLabel }}
{{- $groupLabels := join "," (uniq $additionalGroupByLabels) }}
{{- $grafanaAddr := include "vm-k8s-stack.grafana.addr" . }}
name: vmalert
rules:
- alert: ConfigurationReloadFailure
  expr: vmalert_config_last_reload_successful != 1
  annotations:
    description: Configuration hot-reload failed for vmalert on instance {{`{{`}} $labels.instance {{`}}`}}. Check vmalert's logs for detailed error message.
    summary: Configuration reload failed for vmalert instance {{`{{`}} $labels.instance {{`}}`}}
  condition: true
  labels:
    severity: warning
- alert: AlertingRulesError
  expr: sum(increase(vmalert_alerting_rules_errors_total[5m])) without(id) > 0
  annotations:
    dashboard: '{{`{{`}} $externalURL {{`}}`}}/d/LzldHAVnz?viewPanel=13&var-instance={{`{{`}} $labels.instance {{`}}`}}&var-file={{`{{`}} $labels.file {{`}}`}}&var-group={{`{{`}} $labels.group {{`}}`}}'
    description: Alerting rules execution is failing for "{{`{{`}} $labels.alertname {{`}}`}}" from group "{{`{{`}} $labels.group {{`}}`}}" in file "{{`{{`}} $labels.file {{`}}`}}". Check vmalert's logs for detailed error message.
    summary: Alerting rules are failing for vmalert instance {{`{{`}} $labels.instance {{`}}`}}
  condition: true
  for: 5m
  labels:
    severity: warning
- alert: RecordingRulesError
  expr: sum(increase(vmalert_recording_rules_errors_total[5m])) without(id) > 0
  annotations:
    dashboard: '{{`{{`}} $externalURL {{`}}`}}/d/LzldHAVnz?viewPanel=30&var-instance={{`{{`}} $labels.instance {{`}}`}}&var-file={{`{{`}} $labels.file {{`}}`}}&var-group={{`{{`}} $labels.group {{`}}`}}'
    description: Recording rules execution is failing for "{{`{{`}} $labels.recording {{`}}`}}" from group "{{`{{`}} $labels.group {{`}}`}}" in file "{{`{{`}} $labels.file {{`}}`}}". Check vmalert's logs for detailed error message.
    summary: Recording rules are failing for vmalert instance {{`{{`}} $labels.instance {{`}}`}}
  condition: true
  for: 5m
  labels:
    severity: warning
- alert: RecordingRulesNoData
  expr: sum(vmalert_recording_rules_last_evaluation_samples) without(id) < 1
  annotations:
    dashboard: '{{`{{`}} $externalURL {{`}}`}}/d/LzldHAVnz?viewPanel=33&var-file={{`{{`}} $labels.file {{`}}`}}&var-group={{`{{`}} $labels.group {{`}}`}}'
    description: Recording rule "{{`{{`}} $labels.recording {{`}}`}}" from group "{{`{{`}} $labels.group {{`}}`}} in file "{{`{{`}} $labels.file {{`}}`}}" produces 0 samples over the last 30min. It might be caused by a misconfiguration or incorrect query expression.
    summary: Recording rule {{`{{`}} $labels.recording {{`}}`}} ({{`{{`}} $labels.group {{`}}`}}) produces no data
  condition: true
  for: 30m
  labels:
    severity: info
- alert: TooManyMissedIterations
  expr: increase(vmalert_iteration_missed_total[5m]) > 0
  annotations:
    description: vmalert instance {{`{{`}} $labels.instance {{`}}`}} is missing rules evaluations for group "{{`{{`}} $labels.group {{`}}`}}" in file "{{`{{`}} $labels.file {{`}}`}}". The group evaluation time takes longer than the configured evaluation interval. This may result in missed alerting notifications or recording rules samples. Try increasing evaluation interval or concurrency of group "{{`{{`}} $labels.group {{`}}`}}". See https://docs.victoriametrics.com/victoriametrics/vmalert/#groups. If rule expressions are taking longer than expected, please see https://docs.victoriametrics.com/victoriametrics/troubleshooting/#slow-queries.
    summary: vmalert instance {{`{{`}} $labels.instance {{`}}`}} is missing rules evaluations
  condition: true
  for: 15m
  labels:
    severity: warning
- alert: RemoteWriteErrors
  expr: increase(vmalert_remotewrite_errors_total[5m]) > 0
  annotations:
    description: vmalert instance {{`{{`}} $labels.instance {{`}}`}} is failing to push metrics generated via alerting or recording rules to the configured remote write URL. Check vmalert's logs for detailed error message.
    summary: vmalert instance {{`{{`}} $labels.instance {{`}}`}} is failing to push metrics to remote write URL
  condition: true
  for: 15m
  labels:
    severity: warning
- alert: RemoteWriteDroppingData
  expr: increase(vmalert_remotewrite_dropped_rows_total[5m]) > 0
  annotations:
    description: vmalert instance {{`{{`}} $labels.instance {{`}}`}} is failing to send results of alerting or recording rules to the configured remote write URL. This may result into gaps in recording rules or alerts state. Check vmalert's logs for detailed error message.
    summary: vmalert instance {{`{{`}} $labels.instance {{`}}`}} is dropping data sent to remote write URL
  condition: true
  for: 5m
  labels:
    severity: critical
- alert: AlertmanagerErrors
  expr: increase(vmalert_alerts_send_errors_total[5m]) > 0
  annotations:
    description: vmalert instance {{`{{`}} $labels.instance {{`}}`}} is failing to send alert notifications to "{{`{{`}} $labels.addr {{`}}`}}". Check vmalert's logs for detailed error message.
    summary: vmalert instance {{`{{`}} $labels.instance {{`}}`}} is failing to send notifications to Alertmanager
  condition: true
  for: 15m
  labels:
    severity: warning
condition: true
interval: 30s
