
{{- $Values := (.helm).Values | default .Values }}
{{- $runbookUrl := ($Values.defaultRules).runbookUrl | default "https://runbooks.prometheus-operator.dev/runbooks" }}
{{- $clusterLabel := ($Values.global).clusterLabel | default "cluster" }}
{{- $additionalGroupByLabels := append $Values.defaultRules.additionalGroupByLabels $clusterLabel }}
{{- $groupLabels := join "," (uniq $additionalGroupByLabels) }}
{{- $grafanaAddr := include "vm-k8s-stack.grafana.addr" . }}
name: vmoperator
rules:
- alert: LogErrors
  expr: {{ printf "sum(rate(operator_log_messages_total{level=\"error\",job=~\".*((victoria.*)|vm)-?operator\"}[5m])) by(%s) > 0" $groupLabels }}
  annotations:
    dashboard: '{{`{{`}} $externalURL {{`}}`}}/d/1H179hunk/victoriametrics-operator?ds={{`{{`}} $labels.dc {{`}}`}}&orgId=1&viewPanel=16'
    description: 'Operator has too many errors at logs: {{`{{`}} $value{{`}}`}}, check operator logs'
    summary: 'Too many errors at logs of operator: {{`{{`}} $value{{`}}`}}'
  condition: true
  for: 15m
  labels:
    severity: warning
    show_at: dashboard
- alert: ReconcileErrors
  expr: {{ printf "sum(rate(controller_runtime_reconcile_errors_total{job=~\".*((victoria.*)|vm)-?operator\"}[5m])) by(%s) > 0" $groupLabels }}
  annotations:
    dashboard: '{{`{{`}} $externalURL {{`}}`}}/d/1H179hunk/victoriametrics-operator?ds={{`{{`}} $labels.dc {{`}}`}}&orgId=1&viewPanel=10'
    description: 'Operator cannot parse response from k8s api server, possible bug: {{`{{`}} $value {{`}}`}}, check operator logs'
    summary: 'Too many errors at reconcile loop of operator: {{`{{`}} $value{{`}}`}}'
  condition: true
  for: 10m
  labels:
    severity: warning
    show_at: dashboard
- alert: HighQueueDepth
  expr: {{ printf "sum(workqueue_depth{job=~\".*((victoria.*)|vm)-?operator\",name=~\"(vmagent|vmalert|vmalertmanager|vmauth|vmcluster|vmnodescrape|vmpodscrape|vmprobe|vmrule|vmservicescrape|vmsingle|vmstaticscrape)\"}) by(name,%s) > 10" $groupLabels }}
  annotations:
    dashboard: '{{`{{`}} $externalURL {{`}}`}}/d/1H179hunk/victoriametrics-operator?ds={{`{{`}} $labels.dc {{`}}`}}&orgId=1&viewPanel=20'
    description: 'Operator cannot handle reconciliation load for controller: `{{`{{`}}- $labels.name {{`}}`}}`, current depth: {{`{{`}} $value {{`}}`}}'
    summary: 'Too many `{{`{{`}}- $labels.name {{`}}`}}` in queue: {{`{{`}} $value {{`}}`}}'
  condition: true
  for: 15m
  labels:
    severity: warning
    show_at: dashboard
condition: true
