
{{- $Values := (.helm).Values | default .Values }}
{{- $runbookUrl := ($Values.defaultRules).runbookUrl | default "https://runbooks.prometheus-operator.dev/runbooks" }}
{{- $clusterLabel := ($Values.global).clusterLabel | default "cluster" }}
{{- $additionalGroupByLabels := append $Values.defaultRules.additionalGroupByLabels $clusterLabel }}
{{- $groupLabels := join "," (uniq $additionalGroupByLabels) }}
{{- $grafanaAddr := include "vm-k8s-stack.grafana.addr" . }}
name: vm-health
rules:
- alert: TooManyRestarts
  expr: changes(process_start_time_seconds{job=~".*(victoriametrics|vmselect|vminsert|vmstorage|vmagent|vmalert|vmsingle|vmalertmanager|vmauth).*"}[15m]) > 2
  annotations:
    description: |
      Job {{`{{`}} $labels.job {{`}}`}} (instance {{`{{`}} $labels.instance {{`}}`}}) has restarted more than twice in the last 15 minutes. It might be crashlooping.
    summary: '{{`{{`}} $labels.job {{`}}`}} too many restarts (instance {{`{{`}} $labels.instance {{`}}`}})'
  condition: true
  labels:
    severity: critical
- alert: ServiceDown
  expr: up{job=~".*(victoriametrics|vmselect|vminsert|vmstorage|vmagent|vmalert|vmsingle|vmalertmanager|vmauth).*"} == 0
  annotations:
    description: '{{`{{`}} $labels.instance {{`}}`}} of job {{`{{`}} $labels.job {{`}}`}} has been down for more than 2 minutes.'
    summary: Service {{`{{`}} $labels.job {{`}}`}} is down on {{`{{`}} $labels.instance {{`}}`}}
  condition: true
  for: 2m
  labels:
    severity: critical
- alert: ProcessNearFDLimits
  expr: (process_max_fds - process_open_fds) < 100
  annotations:
    description: |
      Exhausting OS file descriptors limit can cause severe degradation of the process.
      Consider to increase the limit as fast as possible.
    summary: Number of free file descriptors is less than 100 for "{{`{{`}} $labels.job {{`}}`}}"("{{`{{`}} $labels.instance {{`}}`}}") for the last 5m
  condition: true
  for: 5m
  labels:
    severity: critical
- alert: TooHighMemoryUsage
  expr: (min_over_time(process_resident_memory_anon_bytes[10m]) / vm_available_memory_bytes) > 0.8
  annotations:
    description: |
      Too high memory usage may result into multiple issues such as OOMs or degraded performance.
      Consider to either increase available memory or decrease the load on the process.
    summary: It is more than 80% of memory used by "{{`{{`}} $labels.job {{`}}`}}"("{{`{{`}} $labels.instance {{`}}`}}")
  condition: true
  for: 5m
  labels:
    severity: critical
- alert: TooHighCPUUsage
  expr: (rate(process_cpu_seconds_total[5m]) / process_cpu_cores_available) > 0.9
  annotations:
    description: |
      Too high CPU usage may be a sign of insufficient resources and make process unstable. Consider to either increase available CPU resources or decrease the load on the process.
    summary: More than 90% of CPU is used by "{{`{{`}} $labels.job {{`}}`}}"("{{`{{`}} $labels.instance {{`}}`}}") during the last 5m
  condition: true
  for: 5m
  labels:
    severity: critical
- alert: TooHighGoroutineSchedulingLatency
  expr: {{ printf "histogram_quantile(0.99, sum(rate(go_sched_latencies_seconds_bucket{job=~\".*(victoriametrics|vmselect|vminsert|vmstorage|vmagent|vmalert|vmsingle|vmalertmanager|vmauth).*\"}[5m])) by(le,job,instance,%s)) > 0.1" $groupLabels }}
  annotations:
    description: |
      Go runtime is unable to schedule goroutines execution in acceptable time. This is usually a sign of insufficient CPU resources or CPU throttling. Verify that service has enough CPU resources. Otherwise, the service could work unreliably with delays in processing.
    summary: '"{{`{{`}} $labels.job {{`}}`}}"("{{`{{`}} $labels.instance {{`}}`}}") has insufficient CPU resources for >15m'
  condition: true
  for: 15m
  labels:
    severity: critical
- alert: TooManyLogs
  expr: sum(increase(vm_log_messages_total{level!="info"}[5m])) without(app_version,location,is_printed) > 0
  annotations:
    description: |
      The job \"{{`{{`}} $labels.job {{`}}`}}\" ({{`{{`}} $labels.instance {{`}}`}}) generated {{`{{`}} $value {{`}}`}} log messages with the level higher than info for the last 5 minutes. Check the logs for the given target. Check also the \"location\" label at the vm_log_messages_total metric if -loggerLevel command-line flag is set to value other than INFO. This label contains code locations responsible for generating log messages suppressed by -loggerLevel.
    summary: Too many logs are generated for job "{{`{{`}} $labels.job {{`}}`}}" ({{`{{`}} $labels.instance {{`}}`}})
  condition: true
  for: 15m
  labels:
    severity: warning
- alert: TooManyTSIDMisses
  expr: increase(vm_missing_tsids_for_metric_id_total[5m]) > 0
  annotations:
    description: |
      Unexpected TSID misses for \"{{`{{`}} $labels.job {{`}}`}}\" ({{`{{`}} $labels.instance {{`}}`}}) for the last 15 minutes.
      If this happens after unclean shutdown of VictoriaMetrics process (via \"kill -9\", OOM or power off),
      then this is OK - the alert must go away in a few minutes after the restart.
      Otherwise this may point to the corruption of index data.
    summary: Unexpected TSID misses for job "{{`{{`}} $labels.job {{`}}`}}" ({{`{{`}} $labels.instance {{`}}`}}) for the last 15 minutes
  condition: true
  for: 15m
  labels:
    severity: critical
- alert: ConcurrentInsertsHitTheLimit
  expr: avg_over_time(vm_concurrent_insert_current[1m]) >= vm_concurrent_insert_capacity
  annotations:
    description: "The limit of concurrent inserts on instance {{`{{`}} $labels.instance {{`}}`}} depends on the number of CPUs.\nUsually, when component constantly hits the limit it is likely the component is overloaded and requires more CPU.\nIn some cases for components like vmagent or vminsert the alert might trigger if there are too many clients\nmaking write attempts. If vmagent's or vminsert's CPU usage and network saturation are at normal level, then \nit might be worth adjusting `-maxConcurrentInserts` cmd-line flag.\n"
    summary: '{{`{{`}} $labels.job {{`}}`}} on instance {{`{{`}} $labels.instance {{`}}`}} is constantly hitting concurrent inserts limit'
  condition: true
  for: 15m
  labels:
    severity: warning
- alert: IndexDBRecordsDrop
  expr: increase(vm_indexdb_items_dropped_total[5m]) > 0
  annotations:
    description: "VictoriaMetrics could skip registering new timeseries during ingestion if they fail the validation process. \nFor example, `reason=too_long_item` means that time series cannot exceed 64KB. Please, reduce the number \nof labels or label values for such series. Or enforce these limits via `-maxLabelsPerTimeseries` and \n`-maxLabelValueLen` command-line flags.\n"
    summary: IndexDB skipped registering items during data ingestion with reason={{`{{`}} $labels.reason {{`}}`}}.
  condition: true
  labels:
    severity: critical
- alert: RowsRejectedOnIngestion
  expr: rate(vm_rows_ignored_total[5m]) > 0
  annotations:
    description: 'Ingested rows on instance "{{`{{`}} $labels.instance {{`}}`}}" are rejected due to the following reason: "{{`{{`}} $labels.reason {{`}}`}}"'
    summary: Some rows are rejected on "{{`{{`}} $labels.instance {{`}}`}}" on ingestion attempt
  condition: true
  for: 15m
  labels:
    severity: warning
- alert: TooHighQueryLoad
  expr: increase(vm_concurrent_select_limit_timeout_total[5m]) > 0
  annotations:
    description: |
      Instance {{`{{`}} $labels.instance {{`}}`}} ({{`{{`}} $labels.job {{`}}`}}) is failing to serve read queries during last 15m.
      Concurrency limit `-search.maxConcurrentRequests` was reached on this instance and extra queries were
      put into the queue for `-search.maxQueueDuration` interval. But even after waiting in the queue these queries weren't served.
      This happens if instance is overloaded with the current workload, or datasource is too slow to respond.
      Possible solutions are the following:
      * reduce the query load;
      * increase compute resources or number of replicas;
      * adjust limits `-search.maxConcurrentRequests` and `-search.maxQueueDuration`.
      See more at https://docs.victoriametrics.com/victoriametrics/troubleshooting/#slow-queries.
    summary: Read queries fail with timeout for {{`{{`}} $labels.job {{`}}`}} on instance {{`{{`}} $labels.instance {{`}}`}}
  condition: true
  for: 15m
  labels:
    severity: warning
condition: true
