{{- /*
Copyright Broadcom, Inc. All Rights Reserved.
SPDX-License-Identifier: APACHE-2.0
*/}}

{{- /*
Generated from https://github.com/thanos-io/thanos/blob/main/examples/alerts/alerts.md
*/ -}}
{{- if and .Values.metrics.enabled (or .Values.metrics.prometheusRule.default.create .Values.metrics.prometheusRule.default.ruler ) }}
apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  name: {{ include "thanos.ruler.fullname" . }}
  namespace: {{ default .Release.Namespace .Values.metrics.prometheusRule.namespace | quote }}
  labels: {{- include "common.labels.standard" ( dict "customLabels" .Values.commonLabels "context" $ ) | nindent 4 }}
    {{- if .Values.metrics.prometheusRule.additionalLabels }}
    {{- include "common.tplvalues.render" (dict "value" .Values.metrics.prometheusRule.additionalLabels "context" $) | nindent 4 }}
    {{- end }}
  {{- if .Values.commonAnnotations }}
  annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }}
  {{- end }}
spec:
  groups:
  - name: thanos-rule
    rules:
    {{- if not (.Values.metrics.prometheusRule.default.disabled.ThanosRuleQueueIsDroppingAlerts | default false) }}
    - alert: ThanosRuleQueueIsDroppingAlerts
      annotations:
        {{- if .Values.commonAnnotations }}
        {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 8 }}
        {{- end }}
        description: Thanos Rule {{`{{`}} $labels.instance {{`}}`}} is failing to queue alerts.
        runbook_url: {{ .Values.metrics.prometheusRule.runbookUrl }}thanosrulequeueisdroppingalerts
        summary: Thanos Rule is failing to queue alerts.
      expr: |
        sum by (job, instance) (rate(thanos_alert_queue_alerts_dropped_total{job=~".*{{ include "thanos.ruler.fullname" . }}.*"}[5m])) > 0
      for: 5m
      labels:
        severity: critical
        {{- if .Values.metrics.prometheusRule.additionalLabels }}
        {{- include "common.tplvalues.render" (dict "value" .Values.metrics.prometheusRule.additionalLabels "context" $) | nindent 8 }}
        {{- end }}
    {{- end }}
    {{- if not (.Values.metrics.prometheusRule.default.disabled.ThanosRuleSenderIsFailingAlerts | default false) }}
    - alert: ThanosRuleSenderIsFailingAlerts
      annotations:
        {{- if .Values.commonAnnotations }}
        {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 8 }}
        {{- end }}
        description: Thanos Rule {{`{{`}} $labels.instance {{`}}`}} is failing to send alerts to alertmanager.
        runbook_url: {{ .Values.metrics.prometheusRule.runbookUrl }}thanosrulesenderisfailingalerts
        summary: Thanos Rule is failing to send alerts to alertmanager.
      expr: |
        sum by (job, instance) (rate(thanos_alert_sender_alerts_dropped_total{job=~".*{{ include "thanos.ruler.fullname" . }}.*"}[5m])) > 0
      for: 5m
      labels:
        severity: critical
        {{- if .Values.metrics.prometheusRule.additionalLabels }}
        {{- include "common.tplvalues.render" (dict "value" .Values.metrics.prometheusRule.additionalLabels "context" $) | nindent 8 }}
        {{- end }}
    {{- end }}
    {{- if not (.Values.metrics.prometheusRule.default.disabled.ThanosRuleHighRuleEvaluationFailures | default false) }}
    - alert: ThanosRuleHighRuleEvaluationFailures
      annotations:
        {{- if .Values.commonAnnotations }}
        {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 8 }}
        {{- end }}
        description: Thanos Rule {{`{{`}} $labels.instance {{`}}`}} is failing to evaluate rules.
        runbook_url: {{ .Values.metrics.prometheusRule.runbookUrl }}thanosrulehighruleevaluationfailures
        summary: Thanos Rule is failing to evaluate rules.
      expr: |
        (
          sum by (job, instance) (rate(prometheus_rule_evaluation_failures_total{job=~".*{{ include "thanos.ruler.fullname" . }}.*"}[5m]))
        /
          sum by (job, instance) (rate(prometheus_rule_evaluations_total{job=~".*{{ include "thanos.ruler.fullname" . }}.*"}[5m]))
        * 100 > 5
        )
      for: 5m
      labels:
        severity: critical
        {{- if .Values.metrics.prometheusRule.additionalLabels }}
        {{- include "common.tplvalues.render" (dict "value" .Values.metrics.prometheusRule.additionalLabels "context" $) | nindent 8 }}
        {{- end }}
    {{- end }}
    {{- if not (.Values.metrics.prometheusRule.default.disabled.ThanosRuleHighRuleEvaluationWarnings | default false) }}
    - alert: ThanosRuleHighRuleEvaluationWarnings
      annotations:
        {{- if .Values.commonAnnotations }}
        {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 8 }}
        {{- end }}
        description: Thanos Rule {{`{{`}} $labels.instance {{`}}`}} has high number of evaluation warnings.
        runbook_url: {{ .Values.metrics.prometheusRule.runbookUrl }}thanosrulehighruleevaluationwarnings
        summary: Thanos Rule has high number of evaluation warnings.
      expr: |
        sum by (job, instance) (rate(thanos_rule_evaluation_with_warnings_total{job=~".*{{ include "thanos.ruler.fullname" . }}.*"}[5m])) > 0
      for: 15m
      labels:
        severity: info
        {{- if .Values.metrics.prometheusRule.additionalLabels }}
        {{- include "common.tplvalues.render" (dict "value" .Values.metrics.prometheusRule.additionalLabels "context" $) | nindent 8 }}
        {{- end }}
    {{- end }}
    {{- if not (.Values.metrics.prometheusRule.default.disabled.ThanosRuleRuleEvaluationLatencyHigh | default false) }}
    - alert: ThanosRuleRuleEvaluationLatencyHigh
      annotations:
        {{- if .Values.commonAnnotations }}
        {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 8 }}
        {{- end }}
        description: Thanos Rule {{`{{`}} $labels.instance {{`}}`}} has higher evaluation latency than interval for {{`{{`}} $labels.rule_group {{`}}`}}.
        runbook_url: {{ .Values.metrics.prometheusRule.runbookUrl }}thanosruleruleevaluationlatencyhigh
        summary: Thanos Rule has high rule evaluation latency.
      expr: |
        (
          sum by (job, instance, rule_group) (prometheus_rule_group_last_duration_seconds{job=~".*{{ include "thanos.ruler.fullname" . }}.*"})
        >
          sum by (job, instance, rule_group) (prometheus_rule_group_interval_seconds{job=~".*{{ include "thanos.ruler.fullname" . }}.*"})
        )
      for: 5m
      labels:
        severity: warning
        {{- if .Values.metrics.prometheusRule.additionalLabels }}
        {{- include "common.tplvalues.render" (dict "value" .Values.metrics.prometheusRule.additionalLabels "context" $) | nindent 8 }}
        {{- end }}
    {{- end }}
    {{- if not (.Values.metrics.prometheusRule.default.disabled.ThanosRuleGrpcErrorRate | default false) }}
    - alert: ThanosRuleGrpcErrorRate
      annotations:
        {{- if .Values.commonAnnotations }}
        {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 8 }}
        {{- end }}
        description: Thanos Rule {{`{{`}} $labels.job {{`}}`}} is failing to handle {{`{{`}} $value | humanize {{`}}`}}% of requests.
        runbook_url: {{ .Values.metrics.prometheusRule.runbookUrl }}thanosrulegrpcerrorrate
        summary: Thanos Rule is failing to handle grpc requests.
      expr: |
        (
          sum by (job, instance) (rate(grpc_server_handled_total{grpc_code=~"Unknown|ResourceExhausted|Internal|Unavailable|DataLoss|DeadlineExceeded", job=~".*{{ include "thanos.ruler.fullname" . }}.*"}[5m]))
        /
          sum by (job, instance) (rate(grpc_server_started_total{job=~".*{{ include "thanos.ruler.fullname" . }}.*"}[5m]))
        * 100 > 5
        )
      for: 5m
      labels:
        severity: warning
        {{- if .Values.metrics.prometheusRule.additionalLabels }}
        {{- include "common.tplvalues.render" (dict "value" .Values.metrics.prometheusRule.additionalLabels "context" $) | nindent 8 }}
        {{- end }}
    {{- end }}
    {{- if not (.Values.metrics.prometheusRule.default.disabled.ThanosRuleConfigReloadFailure | default false) }}
    - alert: ThanosRuleConfigReloadFailure
      annotations:
        {{- if .Values.commonAnnotations }}
        {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 8 }}
        {{- end }}
        description: Thanos Rule {{`{{`}} $labels.job {{`}}`}} has not been able to reload its configuration.
        runbook_url: {{ .Values.metrics.prometheusRule.runbookUrl }}thanosruleconfigreloadfailure
        summary: Thanos Rule has not been able to reload configuration.
      expr: avg by (job, instance) (thanos_rule_config_last_reload_successful{job=~".*{{ include "thanos.ruler.fullname" . }}.*"}) != 1
      for: 5m
      labels:
        severity: info
        {{- if .Values.metrics.prometheusRule.additionalLabels }}
        {{- include "common.tplvalues.render" (dict "value" .Values.metrics.prometheusRule.additionalLabels "context" $) | nindent 8 }}
        {{- end }}
    {{- end }}
    {{- if not (.Values.metrics.prometheusRule.default.disabled.ThanosRuleQueryHighDNSFailures | default false) }}
    - alert: ThanosRuleQueryHighDNSFailures
      annotations:
        {{- if .Values.commonAnnotations }}
        {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 8 }}
        {{- end }}
        description: Thanos Rule {{`{{`}} $labels.job {{`}}`}} has {{`{{`}} $value | humanize{{`}}`}}% of failing DNS queries for query endpoints.
        runbook_url: {{ .Values.metrics.prometheusRule.runbookUrl }}thanosrulequeryhighdnsfailures
        summary: Thanos Rule is having high number of DNS failures.
      expr: |
        (
          sum by (job, instance) (rate(thanos_rule_query_apis_dns_failures_total{job=~".*{{ include "thanos.ruler.fullname" . }}.*"}[5m]))
        /
          sum by (job, instance) (rate(thanos_rule_query_apis_dns_lookups_total{job=~".*{{ include "thanos.ruler.fullname" . }}.*"}[5m]))
        * 100 > 1
        )
      for: 15m
      labels:
        severity: warning
        {{- if .Values.metrics.prometheusRule.additionalLabels }}
        {{- include "common.tplvalues.render" (dict "value" .Values.metrics.prometheusRule.additionalLabels "context" $) | nindent 8 }}
        {{- end }}
    {{- end }}
    {{- if not (.Values.metrics.prometheusRule.default.disabled.ThanosRuleAlertmanagerHighDNSFailures | default false) }}
    - alert: ThanosRuleAlertmanagerHighDNSFailures
      annotations:
        {{- if .Values.commonAnnotations }}
        {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 8 }}
        {{- end }}
        description: Thanos Rule {{`{{`}} $labels.instance{{`}}`}} has {{`{{`}} $value | humanize {{`}}`}}% of failing DNS queries for Alertmanager endpoints.
        runbook_url: {{ .Values.metrics.prometheusRule.runbookUrl }}thanosrulealertmanagerhighdnsfailures
        summary: Thanos Rule is having high number of DNS failures.
      expr: |
        (
          sum by (job, instance) (rate(thanos_rule_alertmanagers_dns_failures_total{job=~".*{{ include "thanos.ruler.fullname" . }}.*"}[5m]))
        /
          sum by (job, instance) (rate(thanos_rule_alertmanagers_dns_lookups_total{job=~".*{{ include "thanos.ruler.fullname" . }}.*"}[5m]))
        * 100 > 1
        )
      for: 15m
      labels:
        severity: warning
        {{- if .Values.metrics.prometheusRule.additionalLabels }}
        {{- include "common.tplvalues.render" (dict "value" .Values.metrics.prometheusRule.additionalLabels "context" $) | nindent 8 }}
        {{- end }}
    {{- end }}
    {{- if not (.Values.metrics.prometheusRule.default.disabled.ThanosRuleNoEvaluationFor10Intervals | default false) }}
    - alert: ThanosRuleNoEvaluationFor10Intervals
      annotations:
        {{- if .Values.commonAnnotations }}
        {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 8 }}
        {{- end }}
        description: Thanos Rule {{`{{`}} $labels.job {{`}}`}} has rule groups that did not evaluate for at least 10x of their expected interval.
        runbook_url: {{ .Values.metrics.prometheusRule.runbookUrl }}thanosrulenoevaluationfor10intervals
        summary: Thanos Rule has rule groups that did not evaluate for 10 intervals.
      expr: |
        time() -  max by (job, instance, group) (prometheus_rule_group_last_evaluation_timestamp_seconds{job=~".*{{ include "thanos.ruler.fullname" . }}.*"})
        >
        10 * max by (job, instance, group) (prometheus_rule_group_interval_seconds{job=~".*{{ include "thanos.ruler.fullname" . }}.*"})
      for: 5m
      labels:
        severity: info
        {{- if .Values.metrics.prometheusRule.additionalLabels }}
        {{- include "common.tplvalues.render" (dict "value" .Values.metrics.prometheusRule.additionalLabels "context" $) | nindent 8 }}
        {{- end }}
    {{- end }}
    {{- if not (.Values.metrics.prometheusRule.default.disabled.ThanosNoRuleEvaluations | default false) }}
    - alert: ThanosNoRuleEvaluations
      annotations:
        {{- if .Values.commonAnnotations }}
        {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 8 }}
        {{- end }}
        description: Thanos Rule {{`{{`}} $labels.instance {{`}}`}} did not perform any rule evaluations in the past 10 minutes.
        runbook_url: {{ .Values.metrics.prometheusRule.runbookUrl }}thanosnoruleevaluations
        summary: Thanos Rule did not perform any rule evaluations.
      expr: |
        sum by (job, instance) (rate(prometheus_rule_evaluations_total{job=~".*{{ include "thanos.ruler.fullname" . }}.*"}[5m])) <= 0
          and
        sum by (job, instance) (thanos_rule_loaded_rules{job=~".*{{ include "thanos.ruler.fullname" . }}.*"}) > 0
      for: 5m
      labels:
        severity: critical
        {{- if .Values.metrics.prometheusRule.additionalLabels }}
        {{- include "common.tplvalues.render" (dict "value" .Values.metrics.prometheusRule.additionalLabels "context" $) | nindent 8 }}
        {{- end }}
    {{- end }}
{{- end }}
