
{{- $Values := (.helm).Values | default .Values }}
{{- $multicluster := ((($Values.grafana).sidecar).dashboards).multicluster | default false }}
{{- $defaultDatasource := "prometheus" -}}
{{- $clusterLabel := ($Values.global).clusterLabel | default "cluster" }}
{{- range (($Values.defaultDatasources).victoriametrics).datasources | default list }}
  {{- if and .isDefault .type }}{{ $defaultDatasource = .type }}{{- end }}
{{- end }}
title: VictoriaMetrics - vmalert
templating:
  list:
  - name: ds
    type: datasource
    query: {{ $defaultDatasource }}
    current:
      text: VictoriaMetrics - cluster
      value: PAF93674D0B4E9963
    options: []
    refresh: 1
    regex: ""
  - name: job
    multi: true
    includeAll: true
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    type: query
    query:
      query: {{ printf "label_values(vm_app_version{version=~\"^vmalert.*\",%s=~\"$cluster\"}, job)" $clusterLabel }}
      refId: StandardVariableQuery
    definition: {{ printf "label_values(vm_app_version{version=~\"^vmalert.*\",%s=~\"$cluster\"}, job)" $clusterLabel }}
    current: {}
    options: []
    refresh: 1
    regex: ""
  - name: instance
    multi: true
    includeAll: true
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    type: query
    query:
      query: {{ printf "label_values(vm_app_version{job=~\"$job\",%s=~\"$cluster\"}, instance)" $clusterLabel }}
      refId: StandardVariableQuery
    definition: {{ printf "label_values(vm_app_version{job=~\"$job\",%s=~\"$cluster\"}, instance)" $clusterLabel }}
    allValue: .*
    current: {}
    options: []
    refresh: 1
    regex: ""
  - name: file
    multi: true
    includeAll: true
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    type: query
    query:
      query: {{ printf "label_values(vmalert_iteration_total{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}, file)" $clusterLabel }}
      refId: PrometheusVariableQueryEditor-VariableQuery
    definition: {{ printf "label_values(vmalert_iteration_total{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}, file)" $clusterLabel }}
    allValue: .*
    current: {}
    options: []
    refresh: 1
    regex: ""
  - name: group
    multi: true
    includeAll: true
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    type: query
    query:
      query: {{ printf "label_values(vmalert_iteration_total{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}, group)" $clusterLabel }}
      refId: StandardVariableQuery
    definition: {{ printf "label_values(vmalert_iteration_total{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}, group)" $clusterLabel }}
    allValue: .*
    current: {}
    options: []
    refresh: 1
    regex: ""
  - name: topk
    type: custom
    query: 5, 10, 20, 30, 40, 50
    current:
      text: "5"
      value: "5"
    options:
    - selected: true
      text: "5"
      value: "5"
    - selected: false
      text: "10"
      value: "10"
    - selected: false
      text: "20"
      value: "20"
    - selected: false
      text: "30"
      value: "30"
    - selected: false
      text: "40"
      value: "40"
    - selected: false
      text: "50"
      value: "50"
  - name: filter
    datasource:
      type: {{ $defaultDatasource }}
      uid: ${ds}
    type: adhoc
    baseFilters: []
    filters: []
  - name: cluster
    label: cluster
    multi: {{ ternary true false $multicluster }}
    includeAll: {{ ternary true false $multicluster }}
    datasource:
      type: {{ $defaultDatasource }}
    type: {{ ternary "query" "constant" $multicluster }}
    query: {{ ternary (printf "label_values(vm_app_version, %s)" $clusterLabel) ".*" $multicluster }}
    hide: {{ ternary 0 2 $multicluster }}
    allValue: .*
editable: false
timezone: {{ default "utc" ($Values.defaultDashboards).defaultTimezone }}
tags:
- victoriametrics
- vm-k8s-stack
annotations:
  list:
  - name: Annotations & Alerts
    datasource:
      type: datasource
      uid: grafana
    builtIn: 1
    enable: true
    hide: true
    iconColor: rgba(0, 211, 255, 1)
    target:
      limit: 100
      matchAny: false
      tags: []
      type: dashboard
    type: dashboard
  - name: version
    expr: {{ printf "sum(vm_app_version{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}) by(short_version,%s) unless sum(vm_app_version{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"} offset $__interval) by(short_version,%s)" $clusterLabel $clusterLabel $clusterLabel $clusterLabel }}
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    enable: true
    hide: true
    iconColor: dark-blue
    textFormat: '{{`{{`}}short_version{{`}}`}}'
    titleFormat: Version change
  - name: restarts
    expr: {{ printf "sum(changes(vm_app_start_timestamp{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}[$__interval])) by(job,instance,%s)" $clusterLabel $clusterLabel }}
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    enable: true
    hide: false
    iconColor: dark-yellow
    textFormat: '{{`{{`}}job{{`}}`}}:{{`{{`}}instance{{`}}`}} restarted'
panels:
- title: Stats
  collapsed: false
  gridPos:
    h: 1
    w: 24
    x: 0
    "y": 0
  id: 47
  type: row
- title: Config update
  targets:
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    expr: {{ printf "count(vmalert_config_last_reload_successful{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"} < 1) by(%s)" $clusterLabel $clusterLabel }}
    editorMode: code
    exemplar: false
    interval: ""
    legendFormat: ""
    range: true
    refId: A
  datasource:
    type: {{ $defaultDatasource }}
    uid: $ds
  description: Shows if the last configuration update was successful. "Not Ok" means there was an unsuccessful attempt to update the configuration due to some error. Check the log for details.
  fieldConfig:
    defaults:
      mappings:
      - options:
          match: "null"
          result:
            color: green
            index: 0
            text: Ok
        type: special
      - options:
          from: 1
          result:
            color: red
            index: 1
            text: Not Ok
          to: 999999
        type: range
      thresholds:
        mode: absolute
        steps:
        - color: green
    overrides: []
  gridPos:
    h: 3
    w: 4
    x: 0
    "y": 1
  id: 6
  options:
    colorMode: value
    graphMode: area
    justifyMode: auto
    orientation: auto
    percentChangeColorMode: standard
    reduceOptions:
      calcs:
      - last
      fields: ""
      values: false
    showPercentChange: false
    text: {}
    textMode: auto
    wideLayout: true
  pluginVersion: 12.0.2
  type: stat
- title: Alerting rules
  targets:
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    expr: {{ printf "count(vmalert_alerting_rules_last_evaluation_samples{job=~\"$job\",instance=~\"$instance\",group=~\"$group\",file=~\"$file\",%s=~\"$cluster\"}) by(%s)" $clusterLabel $clusterLabel }}
    exemplar: false
    interval: ""
    legendFormat: ""
    refId: A
  datasource:
    type: {{ $defaultDatasource }}
    uid: $ds
  description: Shows the total number of loaded alerting rules across selected instances and groups.
  fieldConfig:
    defaults:
      mappings: []
      min: 0
      thresholds:
        mode: absolute
        steps:
        - color: green
    overrides: []
  gridPos:
    h: 3
    w: 5
    x: 4
    "y": 1
  id: 9
  options:
    colorMode: value
    graphMode: area
    justifyMode: auto
    orientation: auto
    percentChangeColorMode: standard
    reduceOptions:
      calcs:
      - last
      fields: ""
      values: false
    showPercentChange: false
    text: {}
    textMode: auto
    wideLayout: true
  pluginVersion: 12.0.2
  type: stat
- title: Recording rules
  targets:
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    expr: {{ printf "count(vmalert_recording_rules_last_evaluation_samples{job=~\"$job\",instance=~\"$instance\",group=~\"$group\",file=~\"$file\",%s=~\"$cluster\"}) by(%s)" $clusterLabel $clusterLabel }}
    exemplar: false
    interval: ""
    legendFormat: ""
    refId: A
  datasource:
    type: {{ $defaultDatasource }}
    uid: $ds
  description: Shows the total number of loaded recording rules across selected instances and groups.
  fieldConfig:
    defaults:
      mappings: []
      min: 0
      thresholds:
        mode: absolute
        steps:
        - color: green
    overrides: []
  gridPos:
    h: 3
    w: 5
    x: 9
    "y": 1
  id: 7
  options:
    colorMode: value
    graphMode: area
    justifyMode: auto
    orientation: auto
    percentChangeColorMode: standard
    reduceOptions:
      calcs:
      - last
      fields: ""
      values: false
    showPercentChange: false
    text: {}
    textMode: auto
    wideLayout: true
  pluginVersion: 12.0.2
  type: stat
- title: Errors
  targets:
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    expr: {{ printf "sum(increase(vmalert_alerting_rules_errors_total{job=~\"$job\",instance=~\"$instance\",group=~\"$group\",file=~\"$file\",%s=~\"$cluster\"}[$__rate_interval])) by(%s) + sum(increase(vmalert_recording_rules_errors_total{job=~\"$job\",instance=~\"$instance\",group=~\"$group\",file=~\"$file\",%s=~\"$cluster\"}[$__rate_interval])) by(%s)" $clusterLabel $clusterLabel $clusterLabel $clusterLabel }}
    editorMode: code
    exemplar: false
    interval: ""
    legendFormat: ""
    range: true
    refId: A
  datasource:
    type: {{ $defaultDatasource }}
    uid: $ds
  description: Shows the total number of errors generated by recording/alerting rules for selected instances and groups.
  fieldConfig:
    defaults:
      mappings: []
      min: 0
      thresholds:
        mode: absolute
        steps:
        - color: green
        - color: red
          value: 1
    overrides: []
  gridPos:
    h: 3
    w: 5
    x: 14
    "y": 1
  id: 8
  options:
    colorMode: value
    graphMode: area
    justifyMode: auto
    orientation: auto
    percentChangeColorMode: standard
    reduceOptions:
      calcs:
      - last
      fields: ""
      values: false
    showPercentChange: false
    text: {}
    textMode: auto
    wideLayout: true
  pluginVersion: 12.0.2
  type: stat
- title: No data errors
  targets:
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    expr: {{ printf "count(vmalert_recording_rules_last_evaluation_samples{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"} < 1) by(%s)" $clusterLabel $clusterLabel }}
    editorMode: code
    exemplar: false
    interval: ""
    legendFormat: ""
    range: true
    refId: A
  datasource:
    type: {{ $defaultDatasource }}
    uid: $ds
  description: |-
    Shows number of Recording Rules which produce no data.

     Usually it means that such rules are misconfigured, since they give no output during the evaluation.
    Please check if rule's expression is correct and it is working as expected.
  fieldConfig:
    defaults:
      mappings:
      - options:
          match: "null"
          result:
            index: 1
            text: "0"
        type: special
      min: 0
      thresholds:
        mode: absolute
        steps:
        - color: green
        - color: red
          value: 1
    overrides: []
  gridPos:
    h: 3
    w: 5
    x: 19
    "y": 1
  id: 48
  options:
    colorMode: value
    graphMode: area
    justifyMode: auto
    orientation: auto
    percentChangeColorMode: standard
    reduceOptions:
      calcs:
      - last
      fields: ""
      values: false
    showPercentChange: false
    text: {}
    textMode: auto
    wideLayout: true
  pluginVersion: 12.0.2
  type: stat
- targets:
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    expr: {{ printf "sum(vm_app_version{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}) by(job,short_version,%s)" $clusterLabel $clusterLabel }}
    editorMode: code
    exemplar: false
    format: table
    instant: true
    range: false
    refId: A
  datasource:
    type: {{ $defaultDatasource }}
    uid: $ds
  fieldConfig:
    defaults:
      color:
        mode: thresholds
      custom:
        align: auto
        cellOptions:
          type: auto
        inspect: false
        minWidth: 50
      mappings: []
      thresholds:
        mode: absolute
        steps:
        - color: green
        - color: red
          value: 80
    overrides:
    - matcher:
        id: byName
        options: Time
      properties:
      - id: custom.hidden
        value: true
    - matcher:
        id: byName
        options: Value
      properties:
      - id: displayName
        value: Count
  gridPos:
    h: 4
    w: 9
    x: 0
    "y": 4
  id: 45
  options:
    cellHeight: sm
    footer:
      countRows: false
      fields: ""
      reducer:
      - sum
      show: false
    showHeader: true
  pluginVersion: 12.0.2
  type: table
- title: Uptime
  targets:
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    expr: {{ printf "sum(min_over_time(up{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}[$__rate_interval])) by(job,%s)" $clusterLabel $clusterLabel }}
    editorMode: code
    exemplar: false
    format: time_series
    instant: false
    interval: ""
    legendFormat: '{{`{{`}}job{{`}}`}}'
    refId: A
  datasource:
    type: {{ $defaultDatasource }}
    uid: $ds
  fieldConfig:
    defaults:
      color:
        mode: palette-classic
      custom:
        axisBorderShow: false
        axisCenteredZero: false
        axisColorMode: text
        axisLabel: ""
        axisPlacement: auto
        barAlignment: 0
        barWidthFactor: 0.6
        drawStyle: line
        fillOpacity: 0
        gradientMode: none
        hideFrom:
          legend: false
          tooltip: false
          viz: false
        insertNulls: false
        lineInterpolation: stepAfter
        lineWidth: 1
        pointSize: 5
        scaleDistribution:
          type: linear
        showPoints: never
        spanNulls: false
        stacking:
          group: A
          mode: none
        thresholdsStyle:
          mode: "off"
      decimals: 0
      links: []
      mappings: []
      min: 0
      thresholds:
        mode: absolute
        steps:
        - color: green
        - color: red
          value: 80
      unit: short
    overrides: []
  gridPos:
    h: 4
    w: 15
    x: 9
    "y": 4
  id: 4
  options:
    legend:
      calcs:
      - lastNotNull
      displayMode: table
      placement: right
      showLegend: true
      sortBy: Last *
      sortDesc: true
    tooltip:
      hideZeros: false
      mode: multi
      sort: asc
  pluginVersion: 12.0.2
  type: timeseries
- title: Overview ($instance)
  collapsed: false
  gridPos:
    h: 1
    w: 24
    x: 0
    "y": 8
  id: 11
  type: row
- title: Alerts fired total ($instance)
  targets:
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    expr: {{ printf "sum(increase(vmalert_alerts_fired_total{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}[$__rate_interval])) by(job,%s)" $clusterLabel $clusterLabel }}
    editorMode: code
    exemplar: false
    interval: ""
    legendFormat: '{{`{{`}}job{{`}}`}}'
    range: true
    refId: A
  datasource:
    type: {{ $defaultDatasource }}
    uid: $ds
  description: Shows the number of fired alerts by job.
  fieldConfig:
    defaults:
      color:
        mode: palette-classic
      custom:
        axisBorderShow: false
        axisCenteredZero: false
        axisColorMode: text
        axisLabel: ""
        axisPlacement: auto
        barAlignment: 0
        barWidthFactor: 0.6
        drawStyle: line
        fillOpacity: 0
        gradientMode: none
        hideFrom:
          legend: false
          tooltip: false
          viz: false
        insertNulls: false
        lineInterpolation: linear
        lineWidth: 1
        pointSize: 5
        scaleDistribution:
          type: linear
        showPoints: never
        spanNulls: false
        stacking:
          group: A
          mode: none
        thresholdsStyle:
          mode: "off"
      mappings: []
      thresholds:
        mode: absolute
        steps:
        - color: green
        - color: red
          value: 80
      unit: short
    overrides: []
  gridPos:
    h: 8
    w: 12
    x: 0
    "y": 9
  id: 15
  options:
    legend:
      calcs:
      - mean
      - lastNotNull
      - max
      displayMode: table
      placement: bottom
      showLegend: true
      sortBy: Last *
      sortDesc: true
    tooltip:
      hideZeros: false
      mode: multi
      sort: desc
  pluginVersion: 12.0.2
  type: timeseries
- title: Top $topk groups avg evaluation duration ($group)
  targets:
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    expr: {{ printf "topk($topk, max(sum(rate(vmalert_iteration_duration_seconds_sum{job=~\"$job\",instance=~\"$instance\",group=~\"$group\",file=~\"$file\",%s=~\"$cluster\"}[$__rate_interval]) / rate(vmalert_iteration_duration_seconds_count{job=~\"$job\",instance=~\"$instance\",group=~\"$group\",file=~\"$file\",%s=~\"$cluster\"}[$__rate_interval])) by(job,instance,group,file,%s)) by(job,group,file,%s)) by(%s)" $clusterLabel $clusterLabel $clusterLabel $clusterLabel $clusterLabel }}
    editorMode: code
    exemplar: false
    interval: ""
    legendFormat: ({{`{{`}}job{{`}}`}}) {{`{{`}}group{{`}}`}}({{`{{`}}file{{`}}`}})
    range: true
    refId: A
  datasource:
    type: {{ $defaultDatasource }}
    uid: $ds
  description: Top $topk groups by evaluation duration. Shows groups that take the most of time during the evaluation across all instances.
  fieldConfig:
    defaults:
      color:
        mode: palette-classic
      custom:
        axisBorderShow: false
        axisCenteredZero: false
        axisColorMode: text
        axisLabel: ""
        axisPlacement: auto
        barAlignment: 0
        barWidthFactor: 0.6
        drawStyle: line
        fillOpacity: 0
        gradientMode: none
        hideFrom:
          legend: false
          tooltip: false
          viz: false
        insertNulls: false
        lineInterpolation: linear
        lineWidth: 1
        pointSize: 5
        scaleDistribution:
          type: linear
        showPoints: never
        spanNulls: false
        stacking:
          group: A
          mode: none
        thresholdsStyle:
          mode: "off"
      mappings: []
      thresholds:
        mode: absolute
        steps:
        - color: green
        - color: red
          value: 80
      unit: s
    overrides: []
  gridPos:
    h: 8
    w: 12
    x: 12
    "y": 9
  id: 23
  options:
    legend:
      calcs:
      - mean
      - lastNotNull
      - max
      displayMode: table
      placement: bottom
      showLegend: true
      sortBy: Last *
      sortDesc: true
    tooltip:
      hideZeros: false
      mode: multi
      sort: desc
  pluginVersion: 12.0.2
  type: timeseries
- title: Rules execution rate ($instance)
  targets:
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    expr: {{ printf "sum(rate(vmalert_execution_total{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}[$__rate_interval])) by(job,%s)" $clusterLabel $clusterLabel }}
    editorMode: code
    exemplar: false
    interval: ""
    legendFormat: '{{`{{`}}job{{`}}`}}'
    range: true
    refId: A
  datasource:
    type: {{ $defaultDatasource }}
    uid: $ds
  description: Shows how many requests (executions) per second vmalert sends to the configured datasource.
  fieldConfig:
    defaults:
      color:
        mode: palette-classic
      custom:
        axisBorderShow: false
        axisCenteredZero: false
        axisColorMode: text
        axisLabel: ""
        axisPlacement: auto
        barAlignment: 0
        barWidthFactor: 0.6
        drawStyle: line
        fillOpacity: 0
        gradientMode: none
        hideFrom:
          legend: false
          tooltip: false
          viz: false
        insertNulls: false
        lineInterpolation: linear
        lineWidth: 1
        pointSize: 5
        scaleDistribution:
          type: linear
        showPoints: never
        spanNulls: false
        stacking:
          group: A
          mode: none
        thresholdsStyle:
          mode: "off"
      mappings: []
      thresholds:
        mode: absolute
        steps:
        - color: green
        - color: red
          value: 80
      unit: short
    overrides: []
  gridPos:
    h: 8
    w: 12
    x: 0
    "y": 17
  id: 24
  options:
    legend:
      calcs:
      - mean
      - lastNotNull
      - max
      displayMode: table
      placement: bottom
      showLegend: true
    tooltip:
      hideZeros: false
      mode: multi
      sort: desc
  pluginVersion: 12.0.2
  type: timeseries
- title: Rules execution errors ($instance)
  targets:
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    expr: {{ printf "sum(rate(vmalert_execution_errors_total{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}[$__rate_interval])) by(job,%s) > 0" $clusterLabel $clusterLabel }}
    editorMode: code
    exemplar: false
    interval: ""
    legendFormat: __auto
    range: true
    refId: A
  datasource:
    type: {{ $defaultDatasource }}
    uid: $ds
  description: Shows the error rate while executing configured rules. Non-zero value means there are some issues with existing rules. Check the logs to get more details.
  fieldConfig:
    defaults:
      color:
        mode: palette-classic
      custom:
        axisBorderShow: false
        axisCenteredZero: false
        axisColorMode: text
        axisLabel: ""
        axisPlacement: auto
        barAlignment: 0
        barWidthFactor: 0.6
        drawStyle: line
        fillOpacity: 10
        gradientMode: none
        hideFrom:
          legend: false
          tooltip: false
          viz: false
        insertNulls: false
        lineInterpolation: linear
        lineWidth: 1
        pointSize: 5
        scaleDistribution:
          type: linear
        showPoints: never
        spanNulls: false
        stacking:
          group: A
          mode: none
        thresholdsStyle:
          mode: "off"
      mappings: []
      thresholds:
        mode: absolute
        steps:
        - color: green
        - color: red
          value: 80
      unit: short
    overrides: []
  gridPos:
    h: 8
    w: 12
    x: 12
    "y": 17
  id: 25
  options:
    legend:
      calcs:
      - mean
      - lastNotNull
      - max
      displayMode: table
      placement: bottom
      showLegend: true
    tooltip:
      hideZeros: false
      mode: multi
      sort: none
  pluginVersion: 12.0.2
  type: timeseries
- title: Resource usage
  panels:
  - title: Memory usage % ($instance)
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "max(max_over_time(process_resident_memory_bytes{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}[$__rate_interval]) / vm_available_memory_bytes{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}) by(job,%s)" $clusterLabel $clusterLabel $clusterLabel }}
      editorMode: code
      exemplar: false
      interval: ""
      legendFormat: __auto
      range: true
      refId: A
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: |-
      How much resident memory (RAM) the process is actually using, compared to its allowed container or system limit.

      - Good: Below 70% most of the time, maybe spiking a bit under load.
      - Bad: Above 90% for more than 5 minutes = risk of out-of-memory (OOM) kill.
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ""
          axisPlacement: auto
          barAlignment: 0
          barWidthFactor: 0.6
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: "off"
        links: []
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
          - color: red
            value: 80
        unit: percentunit
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 0
      "y": 162
    id: 37
    links:
    - targetBlank: true
      title: Profiling
      url: https://docs.victoriametrics.com/victoriametrics/vmagent/#profiling
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        hideZeros: false
        mode: multi
        sort: desc
    pluginVersion: 11.5.0
    type: timeseries
  - title: CPU usage %($instance)
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "max(rate(process_cpu_seconds_total{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}[$__rate_interval]) / process_cpu_cores_available{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}) by(job,%s)" $clusterLabel $clusterLabel $clusterLabel }}
      editorMode: code
      exemplar: false
      format: time_series
      interval: ""
      intervalFactor: 1
      legendFormat: '{{`{{`}}job{{`}}`}}'
      range: true
      refId: A
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: "Shows the CPU usage percentage per vmalert instance. \nIf you think that usage is abnormal or unexpected pls file an issue and attach CPU profile if possible."
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ""
          axisPlacement: auto
          barAlignment: 0
          barWidthFactor: 0.6
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: "off"
        links: []
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
          - color: red
            value: 80
        unit: percentunit
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 12
      "y": 162
    id: 35
    links:
    - targetBlank: true
      title: Profiling
      url: https://docs.victoriametrics.com/victoriametrics/vmagent/#profiling
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        hideZeros: false
        mode: multi
        sort: desc
    pluginVersion: 11.5.0
    type: timeseries
  - title: RSS anonymous memory % usage
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "max(max_over_time(process_resident_memory_anon_bytes{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}[$__rate_interval]) / vm_available_memory_bytes{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}) by(instance,%s)" $clusterLabel $clusterLabel $clusterLabel }}
      editorMode: code
      exemplar: false
      interval: ""
      legendFormat: __auto
      range: true
      refId: A
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: |-
      Share for memory allocated by the process itself. When memory usage reaches 100% it will be likely OOM-killed.
      Safe memory usage % considered to be below 80%
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ""
          axisPlacement: auto
          barAlignment: 0
          barWidthFactor: 0.6
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: "off"
        links: []
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
          - color: red
            value: 80
        unit: percentunit
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 0
      "y": 170
    id: 65
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        hideZeros: false
        mode: multi
        sort: desc
    pluginVersion: 11.5.0
    type: timeseries
  - title: CPU usage ($instance)
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "max(rate(process_cpu_seconds_total{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}[$__rate_interval])) by(job,%s)" $clusterLabel $clusterLabel }}
      editorMode: code
      exemplar: false
      format: time_series
      interval: ""
      intervalFactor: 1
      legendFormat: '{{`{{`}}job{{`}}`}}'
      range: true
      refId: A
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "min(process_cpu_cores_available{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}) by(job,%s)" $clusterLabel $clusterLabel }}
      editorMode: code
      exemplar: false
      format: time_series
      hide: false
      interval: ""
      intervalFactor: 1
      legendFormat: limit ({{`{{`}}job{{`}}`}})
      range: true
      refId: B
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: Shows the max number of CPU cores used  by a `job` and the corresponding limit.
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ""
          axisPlacement: auto
          barAlignment: 0
          barWidthFactor: 0.6
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: "off"
        links: []
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
          - color: red
            value: 80
        unit: short
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 12
      "y": 170
    id: 56
    links:
    - targetBlank: true
      title: Profiling
      url: https://docs.victoriametrics.com/victoriametrics/vmagent/#profiling
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        hideZeros: false
        mode: multi
        sort: desc
    pluginVersion: 11.5.0
    type: timeseries
  - title: Memory usage ($instance)
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "max(max_over_time(process_resident_memory_bytes{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}[$__rate_interval])) by(job,%s)" $clusterLabel $clusterLabel }}
      editorMode: code
      exemplar: false
      interval: ""
      legendFormat: '{{`{{`}}job{{`}}`}}'
      range: true
      refId: A
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: |-
      Amount of used RSS memory

      If you think that usage is abnormal or unexpected, please file an issue and attach memory profile if possible.
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ""
          axisPlacement: auto
          barAlignment: 0
          barWidthFactor: 0.6
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: "off"
        links: []
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
          - color: red
            value: 80
        unit: bytes
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 0
      "y": 178
    id: 57
    links:
    - targetBlank: true
      title: Profiling
      url: https://docs.victoriametrics.com/victoriametrics/vmagent/#profiling
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        hideZeros: false
        mode: multi
        sort: desc
    pluginVersion: 11.5.0
    type: timeseries
  - title: CPU pressure
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "sum(rate(process_pressure_cpu_waiting_seconds_total{job=~\"$job\",%s=~\"$cluster\"}[$__rate_interval])) by(job,instance,%s)" $clusterLabel $clusterLabel }}
      editorMode: code
      format: time_series
      interval: ""
      intervalFactor: 2
      legendFormat: '{{`{{`}}instance{{`}}`}} - waiting'
      range: true
      refId: A
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "sum(rate(process_pressure_cpu_stalled_seconds_total{job=~\"$job\",%s=~\"$cluster\"}[$__rate_interval])) by(job,instance,%s)" $clusterLabel $clusterLabel }}
      editorMode: code
      format: time_series
      hide: false
      interval: ""
      intervalFactor: 2
      legendFormat: '{{`{{`}}instance{{`}}`}} - stalled'
      range: true
      refId: B
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: |-
      CPU pressure based on [Pressure Stall Information](https://docs.kernel.org/accounting/psi.html). Helps troubleshoot high CPU usage or throttling.
      Lower is better.

      - waiting: The percentage of time at least one task in the process was ready to run (runnable) but couldn't get scheduled on the CPU.
      - stalled: The percentage of time all tasks in the process (except idle ones) were unable to get CPU time — a full CPU stall.

      If there's a CPU burst, it's normal to see waiting or stalled > 1%. It only becomes a concern if it consistently climbs above 5–10% and aligns with latency spikes or GC slowdowns.
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ""
          axisPlacement: auto
          barAlignment: 0
          barWidthFactor: 0.6
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: line
        decimals: 0
        links: []
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
        unit: s
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 12
      "y": 178
    id: 66
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        hideZeros: false
        mode: multi
        sort: desc
    pluginVersion: 11.5.0
    type: timeseries
  - title: Memory pressure
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "sum(rate(process_pressure_memory_waiting_seconds_total{job=~\"$job\",%s=~\"$cluster\"}[$__rate_interval])) by(job,instance,%s)" $clusterLabel $clusterLabel }}
      editorMode: code
      format: time_series
      interval: ""
      intervalFactor: 2
      legendFormat: '{{`{{`}}instance{{`}}`}} - waiting'
      range: true
      refId: A
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "sum(rate(process_pressure_memory_stalled_seconds_total{job=~\"$job\",%s=~\"$cluster\"}[$__rate_interval])) by(job,instance,%s)" $clusterLabel $clusterLabel }}
      editorMode: code
      format: time_series
      hide: false
      interval: ""
      intervalFactor: 2
      legendFormat: '{{`{{`}}instance{{`}}`}} - stalled'
      range: true
      refId: B
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: |-
      Shows memory pressure based on [Pressure Stall Information](https://docs.kernel.org/accounting/psi.html).
      Lower is better.

      - waiting: Time fraction where at least one thread was blocked on memory.
      - stalled: Time fraction where every thread was blocked on memory (severe pressure).

      If queries slow down and both series spike, the host is likely limited by RAM or I/O throughput.
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ""
          axisPlacement: auto
          barAlignment: 0
          barWidthFactor: 0.6
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: line
        decimals: 0
        links: []
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
        unit: s
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 0
      "y": 186
    id: 67
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        hideZeros: false
        mode: multi
        sort: desc
    pluginVersion: 11.5.0
    type: timeseries
  - title: Goroutines ($instance)
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "sum(go_goroutines{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}) by(job,%s)" $clusterLabel $clusterLabel }}
      editorMode: code
      format: time_series
      interval: ""
      intervalFactor: 2
      legendFormat: __auto
      range: true
      refId: A
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ""
          axisPlacement: auto
          barAlignment: 0
          barWidthFactor: 0.6
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: "off"
        decimals: 0
        links: []
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
          - color: red
            value: 80
        unit: short
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 12
      "y": 186
    id: 41
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
      tooltip:
        hideZeros: false
        mode: multi
        sort: none
    pluginVersion: 11.5.0
    type: timeseries
  - title: Open FDs usage % ($instance)
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "max(max_over_time(process_open_fds{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}[$__rate_interval]) / process_max_fds{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}) by(job,%s)" $clusterLabel $clusterLabel $clusterLabel }}
      editorMode: code
      exemplar: false
      format: time_series
      interval: ""
      intervalFactor: 2
      legendFormat: __auto
      range: true
      refId: A
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: |-
      Panel shows the percentage of open file descriptors in the OS.
      Reaching the limit of open files can cause various issues and must be prevented.

      See how to change limits here https://medium.com/@muhammadtriwibowo/set-permanently-ulimit-n-open-files-in-ubuntu-4d61064429a
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ""
          axisPlacement: auto
          barAlignment: 0
          barWidthFactor: 0.6
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: "off"
        decimals: 3
        links: []
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
          - color: red
            value: 80
        unit: percentunit
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 0
      "y": 194
    id: 39
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
      tooltip:
        hideZeros: false
        mode: multi
        sort: desc
    pluginVersion: 11.5.0
    type: timeseries
  - title: Go scheduling latency
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "max(histogram_quantile(0.99, sum(rate(go_sched_latencies_seconds_bucket{job=~\"$job\",%s=~\"$cluster\"}[$__rate_interval])) by(job,instance,le,%s))) by(job,%s)" $clusterLabel $clusterLabel $clusterLabel }}
      editorMode: code
      format: time_series
      interval: ""
      intervalFactor: 2
      legendFormat: __auto
      range: true
      refId: A
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: "Shows the time goroutines have spent in runnable state before actually running. The lower is better.\n\nHigh values or values exceeding the threshold is usually a sign of            insufficient CPU resources or CPU throttling. \n\nVerify that service has enough CPU resources. Otherwise, the service could work unreliably with delays in processing."
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ""
          axisPlacement: auto
          barAlignment: 0
          barWidthFactor: 0.6
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: line
        decimals: 0
        links: []
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
          - color: red
            value: 0.1
        unit: s
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 12
      "y": 194
    id: 61
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
      tooltip:
        hideZeros: false
        mode: multi
        sort: desc
    pluginVersion: 11.5.0
    type: timeseries
  - title: CPU spent on GC ($instance)
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "max(rate(go_gc_cpu_seconds_total{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}[$__rate_interval]) / rate(process_cpu_seconds_total{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}[$__rate_interval])) by(job,%s)" $clusterLabel $clusterLabel $clusterLabel }}
      editorMode: code
      format: time_series
      interval: ""
      intervalFactor: 2
      legendFormat: __auto
      range: true
      refId: A
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: "Shows the percent of CPU spent on garbage collection.\n\nIf % is high, then CPU usage can be decreased by changing GOGC to higher values. Increasing GOGC value will increase memory usage, and decrease CPU usage.\n\nTry searching for keyword `GOGC` at https://docs.victoriametrics.com/victoriametrics/troubleshooting/ "
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ""
          axisPlacement: auto
          barAlignment: 0
          barWidthFactor: 0.6
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: "off"
        decimals: 0
        links: []
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
          - color: red
            value: 80
        unit: percentunit
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 0
      "y": 202
    id: 59
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
      tooltip:
        hideZeros: false
        mode: multi
        sort: desc
    pluginVersion: 11.5.0
    type: timeseries
  - title: Memory allocations rate
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "sum(rate(go_memstats_alloc_bytes_total{job=~\"$job\",%s=~\"$cluster\"}[$__rate_interval])) by(job,instance,%s)" $clusterLabel $clusterLabel }}
      editorMode: code
      format: time_series
      interval: ""
      intervalFactor: 2
      legendFormat: __auto
      range: true
      refId: A
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: Shows the rate of allocations in memory. Sudden increase in allocations would mean increased pressure on Go Garbage Collector and can saturate CPU resources of the application.
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ""
          axisPlacement: auto
          barAlignment: 0
          barWidthFactor: 0.6
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: line
        decimals: 0
        links: []
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
        unit: bytes
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 0
      "y": 210
    id: 64
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
      tooltip:
        hideZeros: false
        mode: multi
        sort: desc
    pluginVersion: 11.5.0
    type: timeseries
  collapsed: true
  gridPos:
    h: 1
    w: 24
    x: 0
    "y": 25
  id: 43
  type: row
- title: Troubleshooting
  panels:
  - title: Non-default flags
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "sum(flag{is_set=\"true\",job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}) by(job,instance,name,value,%s)" $clusterLabel $clusterLabel }}
      editorMode: code
      exemplar: false
      format: table
      instant: true
      legendFormat: __auto
      range: false
      refId: A
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    fieldConfig:
      defaults:
        color:
          mode: thresholds
        custom:
          align: auto
          cellOptions:
            type: auto
          filterable: true
          inspect: false
        mappings: []
        thresholds:
          mode: absolute
          steps:
          - color: green
          - color: red
            value: 80
      overrides:
      - matcher:
          id: byName
          options: Value
        properties:
        - id: custom.hidden
          value: true
      - matcher:
          id: byName
          options: Time
        properties:
        - id: custom.hidden
          value: true
    gridPos:
      h: 7
      w: 12
      x: 0
      "y": 345
    id: 50
    options:
      cellHeight: sm
      filterable: true
      footer:
        countRows: false
        fields: ""
        reducer:
        - sum
        show: false
      showHeader: true
      showSearch: true
      sortBy:
      - desc: true
        displayName: job
    pluginVersion: 10.4.2
    transformations:
    - id: groupBy
      options:
        fields:
          instance:
            aggregations:
            - uniqueValues
            operation: aggregate
          job:
            aggregations: []
            operation: groupby
          name:
            aggregations: []
            operation: groupby
          value:
            aggregations: []
            operation: groupby
    type: table
  - title: Missed evaluations ($instance)
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "sum(increase(vmalert_iteration_missed_total{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}[$__rate_interval])) by(job,group,file,%s) > 0" $clusterLabel $clusterLabel }}
      editorMode: code
      exemplar: false
      interval: 1m
      legendFormat: ({{`{{`}}job{{`}}`}}) {{`{{`}}group{{`}}`}}({{`{{`}}file{{`}}`}})
      range: true
      refId: A
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: "Missed evaluation means that group evaluation time takes longer than the configured evaluation interval. \nThis may result in missed alerting notifications or recording rules samples. Try increasing evaluation interval or concurrency for such groups. See https://docs.victoriametrics.com/victoriametrics/vmalert/#groups\n\nIf rule expressions are taking longer than expected, please see https://docs.victoriametrics.com/victoriametrics/troubleshooting/#slow-queries.\""
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ""
          axisPlacement: auto
          barAlignment: 0
          drawStyle: bars
          fillOpacity: 10
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: "off"
        mappings: []
        thresholds:
          mode: absolute
          steps:
          - color: green
          - color: red
            value: 80
        unit: short
      overrides: []
    gridPos:
      h: 7
      w: 12
      x: 12
      "y": 345
    id: 58
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
      tooltip:
        mode: multi
        sort: none
    pluginVersion: 9.2.6
    type: timeseries
  - title: Restarts ($job)
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "sum(changes(vm_app_start_timestamp{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}[$__rate_interval]) > 0) by(job,%s)" $clusterLabel $clusterLabel }}
      editorMode: code
      format: time_series
      instant: false
      legendFormat: '{{`{{`}}job{{`}}`}}'
      refId: A
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: 'Shows the number of restarts per job. The chart can be useful to identify periodic process restarts and correlate them with potential issues or anomalies. Normally, processes shouldn''t restart unless restart was inited by user. The reason of restarts should be figured out by checking the logs of each specific service. '
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ""
          axisPlacement: auto
          axisSoftMin: 0
          barAlignment: 0
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: stepAfter
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: "off"
        decimals: 0
        links: []
        mappings: []
        thresholds:
          mode: absolute
          steps:
          - color: green
          - color: red
            value: 80
        unit: none
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 0
      "y": 352
    id: 63
    options:
      legend:
        calcs:
        - lastNotNull
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        mode: multi
        sort: desc
    pluginVersion: 9.1.0
    type: timeseries
  collapsed: true
  gridPos:
    h: 1
    w: 24
    x: 0
    "y": 26
  id: 62
  type: row
- title: Alerting rules ($instance)
  panels:
  - title: Top $topk active alerts ($group)
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "topk($topk, sum(vmalert_alerts_firing{job=~\"$job\",instance=~\"$instance\",group=~\"$group\",file=~\"$file\",%s=~\"$cluster\"}) by(job,group,file,alertname,%s) > 0) by(%s)" $clusterLabel $clusterLabel $clusterLabel }}
      editorMode: code
      exemplar: false
      interval: ""
      legendFormat: ({{`{{`}}job{{`}}`}}) {{`{{`}}group{{`}}`}}.{{`{{`}}alertname{{`}}`}}({{`{{`}}file{{`}}`}})
      range: true
      refId: A
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: Shows top $topk current active (firing) alerting rules.
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ""
          axisPlacement: auto
          barAlignment: 0
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: "off"
        mappings: []
        thresholds:
          mode: absolute
          steps:
          - color: green
          - color: red
            value: 80
        unit: short
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 0
      "y": 370
    id: 14
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        mode: multi
        sort: desc
    pluginVersion: 9.2.6
    type: timeseries
  - title: Errors ($group)
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "sum(increase(vmalert_alerting_rules_errors_total{job=~\"$job\",instance=~\"$instance\",group=~\"$group\",file=~\"$file\",%s=~\"$cluster\"}[$__rate_interval])) by(job,group,file,alertname,%s) > 0" $clusterLabel $clusterLabel }}
      editorMode: code
      exemplar: false
      interval: ""
      legendFormat: ({{`{{`}}job{{`}}`}}) {{`{{`}}group{{`}}`}}.{{`{{`}}alertname{{`}}`}}({{`{{`}}file{{`}}`}})
      range: true
      refId: A
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: Shows the events when rule execution resulted into an error. Check the logs for more details.
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ""
          axisPlacement: auto
          barAlignment: 0
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: "off"
        mappings: []
        thresholds:
          mode: absolute
          steps:
          - color: green
          - color: red
            value: 80
        unit: short
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 12
      "y": 370
    id: 13
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        mode: multi
        sort: desc
    pluginVersion: 9.2.6
    type: timeseries
  - title: Pending ($group)
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "sum(vmalert_alerts_pending{job=~\"$job\",instance=~\"$instance\",group=~\"$group\",file=~\"$file\",%s=~\"$cluster\"}) by(job,group,file,alertname,%s) > 0" $clusterLabel $clusterLabel }}
      editorMode: code
      exemplar: false
      interval: ""
      legendFormat: ({{`{{`}}job{{`}}`}}) {{`{{`}}group{{`}}`}}.{{`{{`}}alertname{{`}}`}}({{`{{`}}file{{`}}`}})
      range: true
      refId: A
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: |-
      Shows the current pending alerting rules per group.
      By pending means the rule which remains active less than configured `for` parameter.
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ""
          axisPlacement: auto
          barAlignment: 0
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: "off"
        mappings: []
        thresholds:
          mode: absolute
          steps:
          - color: green
          - color: red
            value: 80
        unit: short
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 0
      "y": 378
    id: 20
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Mean
        sortDesc: true
      tooltip:
        mode: multi
        sort: desc
    pluginVersion: 9.2.6
    type: timeseries
  - title: Errors rate to Alertmanager
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "sum(rate(vmalert_alerts_send_errors_total{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}[$__rate_interval])) by(instance,addr,%s) > 0" $clusterLabel $clusterLabel }}
      exemplar: false
      interval: ""
      legendFormat: '{{`{{`}}instance{{`}}`}} => {{`{{`}}addr{{`}}`}}'
      refId: A
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: Shows the error rate for the attempts to send alerts to Alertmanager. If not zero it means there issues on attempt to send notification to Alertmanager and some alerts may be not delivered properly. Check the logs for more details.
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ""
          axisPlacement: auto
          barAlignment: 0
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: "off"
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
          - color: red
            value: 80
        unit: short
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 12
      "y": 378
    id: 32
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
      tooltip:
        mode: multi
        sort: desc
    pluginVersion: 9.2.6
    type: timeseries
  - title: Requests rate to Alertmanager by job ($group)
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "sum(rate(vmalert_alerts_sent_total{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}[$__rate_interval])) by(job,addr,%s) > 0" $clusterLabel $clusterLabel }}
      editorMode: code
      exemplar: false
      interval: ""
      legendFormat: '{{`{{`}}job{{`}}`}} => {{`{{`}}addr{{`}}`}}'
      range: true
      refId: A
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: Shows how many alerts are sent to Alertmanager per second. Only active alerts are sent.
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ""
          axisPlacement: auto
          barAlignment: 0
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: "off"
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
          - color: red
            value: 80
        unit: short
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 0
      "y": 386
    id: 26
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
      tooltip:
        mode: multi
        sort: desc
    pluginVersion: 9.2.6
    type: timeseries
  collapsed: true
  gridPos:
    h: 1
    w: 24
    x: 0
    "y": 27
  id: 17
  type: row
- title: Recording rules ($instance)
  panels:
  - title: Top $topk rules by produced samples ($group)
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "topk($topk, max(sum(vmalert_recording_rules_last_evaluation_samples{job=~\"$job\",instance=~\"$instance\",group=~\"$group\",file=~\"$file\",%s=~\"$cluster\"}) by(job,instance,group,file,recording,%s) > 0) by(job,group,file,recording,%s)) by(%s)" $clusterLabel $clusterLabel $clusterLabel $clusterLabel }}
      editorMode: code
      exemplar: false
      interval: ""
      legendFormat: ({{`{{`}}job{{`}}`}}) {{`{{`}}group{{`}}`}}.{{`{{`}}recording{{`}}`}}({{`{{`}}file{{`}}`}})
      range: true
      refId: A
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: Shows the top $topk recording rules which generate the most of [samples](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#raw-samples). Each generated sample is basically a time series which then ingested into configured remote storage. Rules with high numbers may cause the most pressure on the remote database and become a source of too high cardinality.
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ""
          axisPlacement: auto
          barAlignment: 0
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: "off"
        mappings: []
        thresholds:
          mode: absolute
          steps:
          - color: green
          - color: red
            value: 80
        unit: short
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 0
      "y": 385
    id: 31
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        mode: multi
        sort: desc
    pluginVersion: 9.2.6
    type: timeseries
  - title: Rules with 0 produced samples ($group)
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "count(vmalert_recording_rules_last_evaluation_samples{job=~\"$job\",instance=~\"$instance\",group=~\"$group\",file=~\"$file\",%s=~\"$cluster\"} < 1) by(job,group,file,recording,%s)" $clusterLabel $clusterLabel }}
      editorMode: code
      exemplar: false
      interval: ""
      legendFormat: ({{`{{`}}job{{`}}`}}) {{`{{`}}group{{`}}`}}.{{`{{`}}recording{{`}}`}}({{`{{`}}file{{`}}`}})
      range: true
      refId: A
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: |-
      Shows the rules which do not produce any [samples](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#raw-samples) during the evaluation. Usually it means that such rules are misconfigured, since they give no output during the evaluation.
      Please check if rule's expression is correct and it is working as expected.
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ""
          axisPlacement: auto
          barAlignment: 0
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          spanNulls: true
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: "off"
        mappings: []
        thresholds:
          mode: absolute
          steps:
          - color: green
          - color: red
            value: 80
        unit: short
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 12
      "y": 385
    id: 33
    options:
      legend:
        calcs:
        - lastNotNull
        - max
        - mean
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        mode: multi
        sort: desc
    pluginVersion: 8.0.3
    type: timeseries
  - title: Errors ($group)
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "sum(increase(vmalert_recording_rules_errors_total{job=~\"$job\",instance=~\"$instance\",group=~\"$group\",file=~\"$file\",%s=~\"$cluster\"}[$__rate_interval])) by(job,group,file,recording,%s) > 0" $clusterLabel $clusterLabel }}
      editorMode: code
      exemplar: false
      interval: ""
      legendFormat: ({{`{{`}}job{{`}}`}}) {{`{{`}}group{{`}}`}}.{{`{{`}}recording{{`}}`}}({{`{{`}}file{{`}}`}})
      range: true
      refId: A
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ""
          axisPlacement: auto
          barAlignment: 0
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: "off"
        mappings: []
        thresholds:
          mode: absolute
          steps:
          - color: green
          - color: red
            value: 80
        unit: short
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 0
      "y": 393
    id: 30
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
      tooltip:
        mode: multi
        sort: none
    pluginVersion: 9.2.6
    type: timeseries
  collapsed: true
  gridPos:
    h: 1
    w: 24
    x: 0
    "y": 28
  id: 28
  type: row
- title: Remote write
  panels:
  - title: Datapoints send rate ($instance)
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "sum(rate(vmalert_remotewrite_sent_rows_total{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}[$__rate_interval])) by(job,%s)" $clusterLabel $clusterLabel }}
      editorMode: code
      legendFormat: __auto
      range: true
      refId: A
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ""
          axisPlacement: auto
          barAlignment: 0
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: auto
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: "off"
        mappings: []
        thresholds:
          mode: absolute
          steps:
          - color: green
          - color: red
            value: 80
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 0
      "y": 351
    id: 52
    options:
      legend:
        calcs: []
        displayMode: list
        placement: bottom
        showLegend: true
      tooltip:
        mode: single
        sort: desc
    type: timeseries
  - title: Datapoints drop rate ($instance)
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "sum(rate(vmalert_remotewrite_dropped_rows_total{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}[$__rate_interval])) by(job,%s) > 0" $clusterLabel $clusterLabel }}
      editorMode: code
      legendFormat: __auto
      range: true
      refId: A
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: Shows the number of datapoints dropped by vmalert while sending to the configured remote write URL. vmalert performs up to 5 retries before dropping the data. Check vmalert's error logs for the specific error message.
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ""
          axisPlacement: auto
          barAlignment: 0
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: auto
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: "off"
        mappings: []
        thresholds:
          mode: absolute
          steps:
          - color: green
          - color: red
            value: 80
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 12
      "y": 351
    id: 53
    options:
      legend:
        calcs: []
        displayMode: list
        placement: bottom
        showLegend: true
      tooltip:
        mode: single
        sort: desc
    type: timeseries
  - title: Connections ($instance)
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "sum(max_over_time(vmalert_remotewrite_conns{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}[$__rate_interval])) by(job,%s)" $clusterLabel $clusterLabel }}
      editorMode: code
      exemplar: true
      interval: ""
      legendFormat: __auto
      range: true
      refId: A
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: |+
      Shows current number of established connections to remote write endpoints.

    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ""
          axisPlacement: auto
          barAlignment: 0
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: "off"
        links: []
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
          - color: red
            value: 80
        unit: short
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 0
      "y": 378
    id: 54
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
      tooltip:
        mode: multi
        sort: desc
    type: timeseries
  - title: Bytes write rate ($instance)
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "sum(rate(vmalert_remotewrite_conn_bytes_written_total{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}[$__rate_interval])) by(job,%s) > 0" $clusterLabel $clusterLabel }}
      editorMode: code
      exemplar: true
      interval: ""
      legendFormat: __auto
      range: true
      refId: A
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: Shows the global rate for number of written bytes via remote write connections.
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ""
          axisPlacement: auto
          barAlignment: 0
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: "off"
        links: []
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
          - color: red
            value: 80
        unit: decbytes
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 12
      "y": 378
    id: 60
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
      tooltip:
        mode: multi
        sort: desc
    type: timeseries
  collapsed: true
  gridPos:
    h: 1
    w: 24
    x: 0
    "y": 29
  id: 55
  type: row
condition: {{ ($Values.vmalert).enabled }}
description: Overview for VictoriaMetrics vmalert v1.117.0 or higher
fiscalYearStartMonth: 0
graphTooltip: 1
id: 3
links:
- icon: doc
  tags: []
  targetBlank: true
  title: Docs
  type: link
  url: https://docs.victoriametrics.com/victoriametrics/vmalert/
- asDropdown: false
  icon: question
  includeVars: false
  keepTime: false
  tags: []
  targetBlank: true
  title: Troubleshooting
  tooltip: ""
  type: link
  url: https://docs.victoriametrics.com/victoriametrics/vmalert/#troubleshooting
- icon: info
  tags: []
  targetBlank: true
  title: Community support
  type: link
  url: https://victoriametrics.com/support/community-support/
- asDropdown: false
  icon: bolt
  includeVars: false
  keepTime: false
  tags: []
  targetBlank: true
  title: Enterprise support
  tooltip: ""
  type: link
  url: https://victoriametrics.com/support/enterprise-support/
- icon: info
  tags: []
  targetBlank: true
  title: Releases
  type: link
  url: https://github.com/VictoriaMetrics/VictoriaMetrics/releases
preload: false
refresh: ""
schemaVersion: 41
time:
  from: now-3h
  to: now
timepicker: {}
uid: LzldHAVnz
version: 1
