
{{- $Values := (.helm).Values | default .Values }}
{{- $runbookUrl := ($Values.defaultRules).runbookUrl | default "https://runbooks.prometheus-operator.dev/runbooks" }}
{{- $clusterLabel := ($Values.global).clusterLabel | default "cluster" }}
{{- $additionalGroupByLabels := append $Values.defaultRules.additionalGroupByLabels $clusterLabel }}
{{- $groupLabels := join "," (uniq $additionalGroupByLabels) }}
{{- $grafanaAddr := include "vm-k8s-stack.grafana.addr" . }}
name: vmsingle
rules:
- alert: DiskRunsOutOfSpaceIn3Days
  expr: ((sum(vm_free_disk_space_bytes) without(path) / (((rate(vm_rows_added_to_storage_total[1d]) - sum(rate(vm_deduplicated_samples_total[1d])) without(type)) * (sum(vm_data_size_bytes{type!~"indexdb.*"}) without(type) / sum(vm_rows{type!~"indexdb.*"}) without(type))) + (rate(vm_new_timeseries_created_total[1d]) * (sum(vm_data_size_bytes{type="indexdb/file"}) without(type) / sum(vm_rows{type="indexdb/file"}) without(type))))) < 259200) > 0
  annotations:
    dashboard: '{{`{{`}} $externalURL {{`}}`}}/d/wNf0q_kZk?viewPanel=53&var-instance={{`{{`}} $labels.instance {{`}}`}}'
    description: |-
      Taking into account current ingestion rate, free disk space will be enough only for {{`{{`}} $value | humanizeDuration {{`}}`}} on instance {{`{{`}} $labels.instance {{`}}`}}.
       Consider to limit the ingestion rate, decrease retention or scale the disk space if possible.
    summary: Instance {{`{{`}} $labels.instance {{`}}`}} will run out of disk space soon
  condition: true
  for: 30m
  labels:
    severity: critical
- alert: NodeBecomesReadonlyIn3Days
  expr: ((sum(vm_free_disk_space_bytes - vm_free_disk_space_limit_bytes) without(path) / (((rate(vm_rows_added_to_storage_total[1d]) - sum(rate(vm_deduplicated_samples_total[1d])) without(type)) * (sum(vm_data_size_bytes{type!~"indexdb.*"}) without(type) / sum(vm_rows{type!~"indexdb.*"}) without(type))) + (rate(vm_new_timeseries_created_total[1d]) * (sum(vm_data_size_bytes{type="indexdb/file"}) without(type) / sum(vm_rows{type="indexdb/file"}) without(type))))) < 259200) > 0
  annotations:
    dashboard: '{{`{{`}} $externalURL {{`}}`}}/d/oS7Bi_0Wz?viewPanel=53&var-instance={{`{{`}} $labels.instance {{`}}`}}'
    description: |-
      Taking into account current ingestion rate and free disk space instance {{`{{`}} $labels.instance {{`}}`}} is writable for {{`{{`}} $value | humanizeDuration {{`}}`}}.
       Consider to limit the ingestion rate, decrease retention or scale the disk space up if possible.
    summary: Instance {{`{{`}} $labels.instance {{`}}`}} will become read-only in 3 days
  condition: true
  for: 30m
  labels:
    severity: warning
- alert: DiskRunsOutOfSpace
  expr: {{ printf "(sum(vm_data_size_bytes) by(job,instance,%s) / (sum(vm_free_disk_space_bytes) by(job,instance,%s) + sum(vm_data_size_bytes) by(job,instance,%s))) > 0.8" $groupLabels $groupLabels $groupLabels }}
  annotations:
    dashboard: '{{`{{`}} $externalURL {{`}}`}}/d/wNf0q_kZk?viewPanel=53&var-instance={{`{{`}} $labels.instance {{`}}`}}'
    description: |-
      Disk utilisation on instance {{`{{`}} $labels.instance {{`}}`}} is more than 80%.
       Having less than 20% of free disk space could cripple merge processes and overall performance. Consider to limit the ingestion rate, decrease retention or scale the disk space if possible.
    summary: Instance {{`{{`}} $labels.instance {{`}}`}} (job={{`{{`}} $labels.job {{`}}`}}) will run out of disk space soon
  condition: true
  for: 30m
  labels:
    severity: critical
- alert: RequestErrorsToAPI
  expr: increase(vm_http_request_errors_total[5m]) > 0
  annotations:
    dashboard: '{{`{{`}} $externalURL {{`}}`}}/d/wNf0q_kZk?viewPanel=35&var-instance={{`{{`}} $labels.instance {{`}}`}}'
    description: Requests to path {{`{{`}} $labels.path {{`}}`}} are receiving errors. Please verify if clients are sending correct requests.
    summary: Too many errors served for path {{`{{`}} $labels.path {{`}}`}} (instance {{`{{`}} $labels.instance {{`}}`}})
  condition: true
  for: 15m
  labels:
    severity: warning
- alert: TooHighChurnRate
  expr: {{ printf "(sum(rate(vm_new_timeseries_created_total[5m])) by(instance,%s) / sum(rate(vm_rows_inserted_total[5m])) by(instance,%s)) > 0.1" $groupLabels $groupLabels }}
  annotations:
    dashboard: '{{`{{`}} $externalURL {{`}}`}}/d/wNf0q_kZk?viewPanel=66&var-instance={{`{{`}} $labels.instance {{`}}`}}'
    description: |-
      VM constantly creates new time series on "{{`{{`}} $labels.instance {{`}}`}}".
       This effect is known as Churn Rate.
       High Churn Rate is tightly connected with database performance and may result in unexpected OOM's or slow queries.
    summary: Churn rate is more than 10% on "{{`{{`}} $labels.instance {{`}}`}}" for the last 15m
  condition: true
  for: 15m
  labels:
    severity: warning
- alert: TooHighChurnRate24h
  expr: {{ printf "sum(increase(vm_new_timeseries_created_total[24h])) by(instance,%s) > (sum(vm_cache_entries{type=\"storage/hour_metric_ids\"}) by(instance,%s) * 3)" $groupLabels $groupLabels }}
  annotations:
    dashboard: '{{`{{`}} $externalURL {{`}}`}}/d/wNf0q_kZk?viewPanel=66&var-instance={{`{{`}} $labels.instance {{`}}`}}'
    description: |-
      The number of created new time series over last 24h is 3x times higher than current number of active series on "{{`{{`}} $labels.instance {{`}}`}}".
       This effect is known as Churn Rate.
       High Churn Rate is tightly connected with database performance and may result in unexpected OOM's or slow queries.
    summary: Too high number of new series on "{{`{{`}} $labels.instance {{`}}`}}" created over last 24h
  condition: true
  for: 15m
  labels:
    severity: warning
- alert: TooHighSlowInsertsRate
  expr: {{ printf "(sum(rate(vm_slow_row_inserts_total[5m])) by(instance,%s) / sum(rate(vm_rows_inserted_total[5m])) by(instance,%s)) > 0.05" $groupLabels $groupLabels }}
  annotations:
    dashboard: '{{`{{`}} $externalURL {{`}}`}}/d/wNf0q_kZk?viewPanel=68&var-instance={{`{{`}} $labels.instance {{`}}`}}'
    description: High rate of slow inserts on "{{`{{`}} $labels.instance {{`}}`}}" may be a sign of resource exhaustion for the current load. It is likely more RAM is needed for optimal handling of the current number of active time series. See also https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3976#issuecomment-1476883183
    summary: Percentage of slow inserts is more than 5% on "{{`{{`}} $labels.instance {{`}}`}}" for the last 15m
  condition: true
  for: 15m
  labels:
    severity: warning
- alert: MetadataCacheUtilizationIsTooHigh
  expr: (vm_metrics_metadata_storage_size_bytes / vm_metrics_metadata_storage_max_size_bytes) > 0.95
  annotations:
    description: Metadata cache stores meta information about ingested time series - see https://docs.victoriametrics.com/victoriametrics/#metrics-metadata. When cache is overutilized, the oldest entries will be dropped out automatically. It may result into incomplete response for /api/v1/metadata API calls. It doesn't impact regular queries or alerts. Cache size is controlled via -storage.maxMetadataStorageSize cmd-line flag.
    summary: Metadata cache capacity on {{`{{`}} $labels.instance {{`}}`}} (job={{`{{`}} $labels.job {{`}}`}}) is utilized for more than 95% for the last 15min
  condition: true
  for: 15m
  labels:
    severity: warning
concurrency: 2
condition: true
interval: 30s
