
{{- $Values := (.helm).Values | default .Values }}
{{- $multicluster := ((($Values.grafana).sidecar).dashboards).multicluster | default false }}
{{- $defaultDatasource := "prometheus" -}}
{{- $clusterLabel := ($Values.global).clusterLabel | default "cluster" }}
{{- range (($Values.defaultDatasources).victoriametrics).datasources | default list }}
  {{- if and .isDefault .type }}{{ $defaultDatasource = .type }}{{- end }}
{{- end }}
title: Alertmanager / Overview
templating:
  list:
  - name: datasource
    label: Data Source
    type: datasource
    query: {{ $defaultDatasource }}
    hide: 0
    current:
      selected: false
      text: Prometheus
      value: Prometheus
  - name: namespace
    label: namespace
    datasource:
      type: {{ $defaultDatasource }}
      uid: ${datasource}
    type: query
    query: label_values(alertmanager_alerts, namespace)
    current:
      selected: false
      text: ""
      value: ""
    refresh: 2
    sort: 1
  - name: service
    label: service
    datasource:
      type: {{ $defaultDatasource }}
      uid: ${datasource}
    type: query
    query: label_values(alertmanager_alerts, service)
    current:
      selected: false
      text: ""
      value: ""
    refresh: 2
    sort: 1
  - name: integration
    includeAll: true
    datasource:
      type: {{ $defaultDatasource }}
      uid: ${datasource}
    type: query
    query: {{ printf "label_values(alertmanager_notifications_total{integration=~\".*\",%s=~\"$cluster\"}, integration)" $clusterLabel }}
    hide: 2
    current:
      selected: false
      text: $__all
      value: $__all
    refresh: 2
    sort: 1
  - name: cluster
    label: cluster
    multi: {{ ternary true false $multicluster }}
    includeAll: {{ ternary true false $multicluster }}
    datasource:
      type: {{ $defaultDatasource }}
    type: {{ ternary "query" "constant" $multicluster }}
    query: {{ ternary (printf "label_values(alertmanager_alerts, %s)" $clusterLabel) ".*" $multicluster }}
    hide: {{ ternary 0 2 $multicluster }}
    allValue: .*
editable: false
timezone: {{ default "utc" ($Values.defaultDashboards).defaultTimezone }}
tags:
- alertmanager-mixin
- vm-k8s-stack
panels:
- title: Alerts
  collapsed: false
  gridPos:
    h: 1
    w: 24
    x: 0
    "y": 0
  id: 1
  type: row
- title: Alerts
  targets:
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $datasource
    expr: {{ printf "sum(alertmanager_alerts{namespace=~\"$namespace\",service=~\"$service\",%s=~\"$cluster\"}) by(namespace,service,instance,%s)" $clusterLabel $clusterLabel }}
    intervalFactor: 2
    legendFormat: '{{`{{`}}instance{{`}}`}}'
  datasource:
    type: {{ $defaultDatasource }}
    uid: $datasource
  description: current set of alerts stored in the Alertmanager
  fieldConfig:
    defaults:
      custom:
        fillOpacity: 10
        showPoints: never
        stacking:
          mode: normal
      unit: none
  gridPos:
    h: 7
    w: 12
    x: 0
    "y": 1
  id: 2
  options:
    legend:
      showLegend: false
    tooltip:
      mode: multi
  pluginVersion: v11.4.0
  type: timeseries
- title: Alerts receive rate
  targets:
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $datasource
    expr: {{ printf "sum(rate(alertmanager_alerts_received_total{namespace=~\"$namespace\",service=~\"$service\",%s=~\"$cluster\"}[$__rate_interval])) by(namespace,service,instance,%s)" $clusterLabel $clusterLabel }}
    intervalFactor: 2
    legendFormat: '{{`{{`}}instance{{`}}`}} Received'
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $datasource
    expr: {{ printf "sum(rate(alertmanager_alerts_invalid_total{namespace=~\"$namespace\",service=~\"$service\",%s=~\"$cluster\"}[$__rate_interval])) by(namespace,service,instance,%s)" $clusterLabel $clusterLabel }}
    intervalFactor: 2
    legendFormat: '{{`{{`}}instance{{`}}`}} Invalid'
  datasource:
    type: {{ $defaultDatasource }}
    uid: $datasource
  description: rate of successful and invalid alerts received by the Alertmanager
  fieldConfig:
    defaults:
      custom:
        fillOpacity: 10
        showPoints: never
        stacking:
          mode: normal
      unit: ops
  gridPos:
    h: 7
    w: 12
    x: 12
    "y": 1
  id: 3
  options:
    legend:
      showLegend: false
    tooltip:
      mode: multi
  pluginVersion: v11.4.0
  type: timeseries
- title: Notifications
  collapsed: false
  gridPos:
    h: 1
    w: 24
    x: 0
    "y": 8
  id: 4
  type: row
- title: '$integration: Notifications Send Rate'
  targets:
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $datasource
    expr: {{ printf "sum(rate(alertmanager_notifications_total{namespace=~\"$namespace\",service=~\"$service\",integration=\"$integration\",%s=~\"$cluster\"}[$__rate_interval])) by(integration,namespace,service,instance,%s)" $clusterLabel $clusterLabel }}
    intervalFactor: 2
    legendFormat: '{{`{{`}}instance{{`}}`}} Total'
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $datasource
    expr: {{ printf "sum(rate(alertmanager_notifications_failed_total{namespace=~\"$namespace\",service=~\"$service\",integration=\"$integration\",%s=~\"$cluster\"}[$__rate_interval])) by(integration,namespace,service,instance,%s)" $clusterLabel $clusterLabel }}
    intervalFactor: 2
    legendFormat: '{{`{{`}}instance{{`}}`}} Failed'
  datasource:
    type: {{ $defaultDatasource }}
    uid: $datasource
  description: rate of successful and invalid notifications sent by the Alertmanager
  fieldConfig:
    defaults:
      custom:
        fillOpacity: 10
        showPoints: never
        stacking:
          mode: normal
      unit: ops
  gridPos:
    h: 7
    w: 12
    x: 0
    "y": 9
  id: 5
  options:
    legend:
      showLegend: false
    tooltip:
      mode: multi
  pluginVersion: v11.4.0
  repeat: integration
  type: timeseries
- title: '$integration: Notification Duration'
  targets:
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $datasource
    expr: {{ printf "histogram_quantile(0.99, sum(rate(alertmanager_notification_latency_seconds_bucket{namespace=~\"$namespace\",service=~\"$service\",integration=\"$integration\",%s=~\"$cluster\"}[$__rate_interval])) by(le,namespace,service,instance,%s))" $clusterLabel $clusterLabel }}
    intervalFactor: 2
    legendFormat: '{{`{{`}}instance{{`}}`}} 99th Percentile'
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $datasource
    expr: {{ printf "histogram_quantile(0.50, sum(rate(alertmanager_notification_latency_seconds_bucket{namespace=~\"$namespace\",service=~\"$service\",integration=\"$integration\",%s=~\"$cluster\"}[$__rate_interval])) by(le,namespace,service,instance,%s))" $clusterLabel $clusterLabel }}
    intervalFactor: 2
    legendFormat: '{{`{{`}}instance{{`}}`}} Median'
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $datasource
    expr: {{ printf "sum(rate(alertmanager_notification_latency_seconds_sum{namespace=~\"$namespace\",service=~\"$service\",integration=\"$integration\",%s=~\"$cluster\"}[$__rate_interval])) by(namespace,service,instance,%s) / sum(rate(alertmanager_notification_latency_seconds_count{namespace=~\"$namespace\",service=~\"$service\",integration=\"$integration\",%s=~\"$cluster\"}[$__rate_interval])) by(namespace,service,instance,%s)" $clusterLabel $clusterLabel $clusterLabel $clusterLabel }}
    intervalFactor: 2
    legendFormat: '{{`{{`}}instance{{`}}`}} Average'
  datasource:
    type: {{ $defaultDatasource }}
    uid: $datasource
  description: latency of notifications sent by the Alertmanager
  fieldConfig:
    defaults:
      custom:
        fillOpacity: 10
        showPoints: never
        stacking:
          mode: normal
      unit: s
  gridPos:
    h: 7
    w: 12
    x: 12
    "y": 9
  id: 6
  options:
    legend:
      showLegend: false
    tooltip:
      mode: multi
  pluginVersion: v11.4.0
  repeat: integration
  type: timeseries
condition: {{ ($Values.alertmanager).enabled }}
graphTooltip: 1
schemaVersion: 39
time:
  from: now-1h
  to: now
timepicker:
  refresh_intervals:
  - 30s
uid: alertmanager-overview
