
{{- $Values := (.helm).Values | default .Values }}
{{- $multicluster := ((($Values.grafana).sidecar).dashboards).multicluster | default false }}
{{- $defaultDatasource := "prometheus" -}}
{{- $clusterLabel := ($Values.global).clusterLabel | default "cluster" }}
{{- range (($Values.defaultDatasources).victoriametrics).datasources | default list }}
  {{- if and .isDefault .type }}{{ $defaultDatasource = .type }}{{- end }}
{{- end }}
title: VictoriaMetrics - cluster
templating:
  list:
  - name: ds
    type: datasource
    query: {{ $defaultDatasource }}
    current:
      text: VictoriaMetrics - cluster
      value: PAF93674D0B4E9963
    options: []
    refresh: 1
    regex: ""
  - name: job
    multi: true
    includeAll: true
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    type: query
    query:
      query: {{ printf "label_values(vm_app_version{version=~\"^vm(insert|select|storage).*\",%s=~\"$cluster\"}, job)" $clusterLabel }}
      refId: VictoriaMetrics-job-Variable-Query
    definition: {{ printf "label_values(vm_app_version{version=~\"^vm(insert|select|storage).*\",%s=~\"$cluster\"}, job)" $clusterLabel }}
    current:
      text: All
      value:
      - $__all
    options: []
    refresh: 1
    regex: ""
  - name: job_insert
    multi: true
    includeAll: true
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    type: query
    query:
      query: {{ printf "label_values(vm_app_version{job=~\"$job\",version=~\"^vminsert.*\",%s=~\"$cluster\"}, job)" $clusterLabel }}
      refId: StandardVariableQuery
    definition: {{ printf "label_values(vm_app_version{job=~\"$job\",version=~\"^vminsert.*\",%s=~\"$cluster\"}, job)" $clusterLabel }}
    hide: 2
    current:
      text: All
      value:
      - $__all
    options: []
    refresh: 1
    regex: ""
  - name: job_select
    multi: true
    includeAll: true
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    type: query
    query:
      query: {{ printf "label_values(vm_app_version{job=~\"$job\",version=~\"^vmselect.*\",%s=~\"$cluster\"}, job)" $clusterLabel }}
      refId: StandardVariableQuery
    definition: {{ printf "label_values(vm_app_version{job=~\"$job\",version=~\"^vmselect.*\",%s=~\"$cluster\"}, job)" $clusterLabel }}
    hide: 2
    current:
      text: All
      value:
      - $__all
    options: []
    refresh: 1
    regex: ""
  - name: job_storage
    multi: true
    includeAll: true
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    type: query
    query:
      query: {{ printf "label_values(vm_app_version{job=~\"$job\",version=~\"^vmstorage.*\",%s=~\"$cluster\"}, job)" $clusterLabel }}
      refId: StandardVariableQuery
    definition: {{ printf "label_values(vm_app_version{job=~\"$job\",version=~\"^vmstorage.*\",%s=~\"$cluster\"}, job)" $clusterLabel }}
    hide: 2
    current:
      text: All
      value:
      - $__all
    options: []
    refresh: 1
    regex: ""
  - name: instance
    multi: true
    includeAll: true
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    type: query
    query:
      query: {{ printf "label_values(vm_app_version{job=~\"$job\",%s=~\"$cluster\"}, instance)" $clusterLabel }}
      refId: VictoriaMetrics-instance-Variable-Query
    definition: {{ printf "label_values(vm_app_version{job=~\"$job\",%s=~\"$cluster\"}, instance)" $clusterLabel }}
    allValue: .*
    current:
      text: All
      value:
      - $__all
    options: []
    refresh: 1
    regex: ""
  - name: adhoc
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    type: adhoc
    baseFilters: []
    filters: []
  - name: cluster
    label: cluster
    multi: {{ ternary true false $multicluster }}
    includeAll: {{ ternary true false $multicluster }}
    datasource:
      type: {{ $defaultDatasource }}
    type: {{ ternary "query" "constant" $multicluster }}
    query: {{ ternary (printf "label_values(vm_app_version, %s)" $clusterLabel) ".*" $multicluster }}
    hide: {{ ternary 0 2 $multicluster }}
    allValue: .*
editable: false
timezone: {{ default "utc" ($Values.defaultDashboards).defaultTimezone }}
tags:
- victoriametrics
- vm-k8s-stack
annotations:
  list:
  - name: Annotations & Alerts
    datasource:
      type: datasource
      uid: grafana
    builtIn: 1
    enable: true
    hide: true
    iconColor: rgba(0, 211, 255, 1)
    target:
      limit: 100
      matchAny: false
      tags: []
      type: dashboard
    type: dashboard
  - name: alerts
    expr: {{ printf "sum(ALERTS{job=~\"$job\",instance=~\"$instance\",alertgroup=\"vmcluster\",alertstate=\"firing\",show_at=\"dashboard\",%s=~\"$cluster\"}) by(alertname,%s)" $clusterLabel $clusterLabel }}
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    enable: true
    hide: false
    iconColor: red
    titleFormat: '{{`{{`}}alertname{{`}}`}}'
  - name: version change
    expr: {{ printf "sum(vm_app_version{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}) by(version,%s) unless sum(vm_app_version{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"} offset $__interval) by(version,%s)" $clusterLabel $clusterLabel $clusterLabel $clusterLabel }}
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    enable: true
    hide: true
    iconColor: dark-blue
    textFormat: '{{`{{`}}version{{`}}`}}'
    titleFormat: Version change
  - name: restarts
    expr: {{ printf "sum(changes(vm_app_start_timestamp{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}[$__interval])) by(job,%s)" $clusterLabel $clusterLabel }}
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    enable: true
    hide: false
    iconColor: dark-yellow
    textFormat: '{{`{{`}}job{{`}}`}} restarted'
panels:
- title: Stats
  collapsed: false
  gridPos:
    h: 1
    w: 24
    x: 0
    "y": 0
  id: 137
  type: row
- title: Total datapoints
  targets:
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    expr: {{ printf "sum(vm_rows{job=~\"$job_storage\",type!~\"indexdb.*\",%s=~\"$cluster\"}) by(%s)" $clusterLabel $clusterLabel }}
    exemplar: true
    format: time_series
    instant: true
    interval: ""
    intervalFactor: 1
    legendFormat: ""
    refId: A
  datasource:
    type: {{ $defaultDatasource }}
    uid: $ds
  description: How many [data samples](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#raw-samples) are in storage
  fieldConfig:
    defaults:
      color:
        mode: thresholds
      mappings: []
      thresholds:
        mode: absolute
        steps:
        - color: green
          value: 0
      unit: short
    overrides: []
  gridPos:
    h: 3
    w: 6
    x: 0
    "y": 1
  id: 131
  maxDataPoints: 100
  options:
    colorMode: value
    graphMode: area
    justifyMode: auto
    orientation: horizontal
    percentChangeColorMode: standard
    reduceOptions:
      calcs:
      - lastNotNull
      fields: ""
      values: false
    showPercentChange: false
    text: {}
    textMode: auto
    wideLayout: true
  pluginVersion: 12.3.0
  type: stat
- title: Ingestion rate
  targets:
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    expr: {{ printf "sum(rate(vm_vminsert_metrics_read_total{job=~\"$job_storage\",instance=~\"$instance\",%s=~\"$cluster\"}[$__rate_interval])) by(%s)" $clusterLabel $clusterLabel }}
    editorMode: code
    exemplar: false
    format: time_series
    instant: true
    interval: ""
    intervalFactor: 1
    legendFormat: ""
    range: false
    refId: A
  datasource:
    type: {{ $defaultDatasource }}
    uid: $ds
  description: Shows the datapoints ingestion rate, including replication factor.
  fieldConfig:
    defaults:
      color:
        mode: thresholds
      mappings: []
      thresholds:
        mode: absolute
        steps:
        - color: green
          value: 0
      unit: short
    overrides: []
  gridPos:
    h: 3
    w: 6
    x: 6
    "y": 1
  id: 124
  maxDataPoints: 100
  options:
    colorMode: value
    graphMode: area
    justifyMode: auto
    orientation: horizontal
    percentChangeColorMode: standard
    reduceOptions:
      calcs:
      - lastNotNull
      fields: ""
      values: false
    showPercentChange: false
    text: {}
    textMode: auto
    wideLayout: true
  pluginVersion: 12.3.0
  type: stat
- title: Read requests
  targets:
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    expr: {{ printf "sum(rate(vm_http_requests_total{job=~\"$job\",instance=~\"$instance\",path=~\"/select/.*\",%s=~\"$cluster\"}[$__rate_interval])) by(%s)" $clusterLabel $clusterLabel }}
    editorMode: code
    exemplar: true
    format: time_series
    instant: true
    interval: ""
    intervalFactor: 1
    legendFormat: ""
    refId: A
  datasource:
    type: {{ $defaultDatasource }}
    uid: $ds
  description: Shows the rate of HTTP read requests.
  fieldConfig:
    defaults:
      color:
        mode: thresholds
      mappings: []
      thresholds:
        mode: absolute
        steps:
        - color: green
          value: 0
      unit: req/s
    overrides: []
  gridPos:
    h: 3
    w: 6
    x: 12
    "y": 1
  id: 130
  maxDataPoints: 100
  options:
    colorMode: value
    graphMode: area
    justifyMode: auto
    orientation: horizontal
    percentChangeColorMode: standard
    reduceOptions:
      calcs:
      - lastNotNull
      fields: ""
      values: false
    showPercentChange: false
    text: {}
    textMode: auto
    wideLayout: true
  pluginVersion: 12.3.0
  type: stat
- title: Available CPU
  targets:
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    expr: {{ printf "sum(vm_available_cpu_cores{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}) by(%s)" $clusterLabel $clusterLabel }}
    editorMode: code
    exemplar: true
    format: time_series
    instant: true
    interval: ""
    intervalFactor: 1
    legendFormat: ""
    refId: A
  datasource:
    type: {{ $defaultDatasource }}
    uid: $ds
  description: 'Total number of available CPUs for all VM components. '
  fieldConfig:
    defaults:
      color:
        mode: thresholds
      mappings: []
      thresholds:
        mode: absolute
        steps:
        - color: green
          value: 0
      unit: short
    overrides: []
  gridPos:
    h: 3
    w: 6
    x: 18
    "y": 1
  id: 126
  maxDataPoints: 100
  options:
    colorMode: value
    graphMode: area
    justifyMode: auto
    orientation: horizontal
    percentChangeColorMode: standard
    reduceOptions:
      calcs:
      - lastNotNull
      fields: ""
      values: false
    showPercentChange: false
    text: {}
    textMode: auto
    wideLayout: true
  pluginVersion: 12.3.0
  type: stat
- title: Active series
  targets:
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    expr: {{ printf "sum(max_over_time(vm_cache_entries{job=~\"$job\",instance=~\"$instance\",type=\"storage/hour_metric_ids\",%s=~\"$cluster\"}[1h])) by(%s)" $clusterLabel $clusterLabel }}
    editorMode: code
    exemplar: true
    format: time_series
    instant: true
    interval: ""
    intervalFactor: 1
    legendFormat: ""
    refId: A
  datasource:
    type: {{ $defaultDatasource }}
    uid: $ds
  description: Shows the number of [active time series](https://docs.victoriametrics.com/victoriametrics/faq/#what-is-an-active-time-series) with new data points inserted during the last hour. High value may result in ingestion slowdown.
  fieldConfig:
    defaults:
      color:
        mode: thresholds
      mappings: []
      thresholds:
        mode: absolute
        steps:
        - color: green
          value: 0
      unit: short
    overrides: []
  gridPos:
    h: 3
    w: 6
    x: 0
    "y": 4
  id: 34
  maxDataPoints: 100
  options:
    colorMode: value
    graphMode: area
    justifyMode: auto
    orientation: horizontal
    percentChangeColorMode: standard
    reduceOptions:
      calcs:
      - lastNotNull
      fields: ""
      values: false
    showPercentChange: false
    text: {}
    textMode: auto
    wideLayout: true
  pluginVersion: 12.3.0
  type: stat
- title: Disk space usage
  targets:
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    expr: {{ printf "sum(vm_data_size_bytes{job=~\"$job_storage\",%s=~\"$cluster\"}) by(%s)" $clusterLabel $clusterLabel }}
    exemplar: false
    format: time_series
    instant: true
    interval: ""
    intervalFactor: 1
    legendFormat: ""
    refId: A
  datasource:
    type: {{ $defaultDatasource }}
    uid: $ds
  description: Total amount of used disk space
  fieldConfig:
    defaults:
      color:
        mode: thresholds
      mappings: []
      thresholds:
        mode: absolute
        steps:
        - color: green
          value: 0
      unit: bytes
    overrides: []
  gridPos:
    h: 3
    w: 6
    x: 6
    "y": 4
  id: 35
  maxDataPoints: 100
  options:
    colorMode: value
    graphMode: area
    justifyMode: auto
    orientation: horizontal
    percentChangeColorMode: standard
    reduceOptions:
      calcs:
      - lastNotNull
      fields: ""
      values: false
    showPercentChange: false
    text: {}
    textMode: auto
    wideLayout: true
  pluginVersion: 12.3.0
  type: stat
- title: Bytes per point
  targets:
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    expr: {{ printf "sum(vm_data_size_bytes{job=~\"$job_storage\",%s=~\"$cluster\"}) by(%s) / sum(vm_rows{job=~\"$job_storage\",type!~\"indexdb.*\",%s=~\"$cluster\"}) by(%s)" $clusterLabel $clusterLabel $clusterLabel $clusterLabel }}
    exemplar: true
    format: time_series
    instant: true
    interval: ""
    intervalFactor: 1
    legendFormat: ""
    refId: A
  datasource:
    type: {{ $defaultDatasource }}
    uid: $ds
  description: Average disk usage per datapoint.
  fieldConfig:
    defaults:
      color:
        mode: thresholds
      decimals: 2
      mappings: []
      thresholds:
        mode: absolute
        steps:
        - color: green
          value: 0
      unit: bytes
    overrides: []
  gridPos:
    h: 3
    w: 6
    x: 12
    "y": 4
  id: 112
  maxDataPoints: 100
  options:
    colorMode: value
    graphMode: area
    justifyMode: auto
    orientation: horizontal
    percentChangeColorMode: standard
    reduceOptions:
      calcs:
      - lastNotNull
      fields: ""
      values: false
    showPercentChange: false
    text: {}
    textMode: auto
    wideLayout: true
  pluginVersion: 12.3.0
  type: stat
- title: Available memory
  targets:
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    expr: {{ printf "sum(vm_available_memory_bytes{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}) by(%s)" $clusterLabel $clusterLabel }}
    editorMode: code
    exemplar: true
    format: time_series
    instant: true
    interval: ""
    intervalFactor: 1
    legendFormat: ""
    refId: A
  datasource:
    type: {{ $defaultDatasource }}
    uid: $ds
  description: Total size of available memory for all VM components.
  fieldConfig:
    defaults:
      color:
        mode: thresholds
      mappings: []
      thresholds:
        mode: absolute
        steps:
        - color: green
          value: 0
      unit: bytes
    overrides: []
  gridPos:
    h: 3
    w: 6
    x: 18
    "y": 4
  id: 128
  maxDataPoints: 100
  options:
    colorMode: value
    graphMode: area
    justifyMode: auto
    orientation: horizontal
    percentChangeColorMode: standard
    reduceOptions:
      calcs:
      - lastNotNull
      fields: ""
      values: false
    showPercentChange: false
    text: {}
    textMode: auto
    wideLayout: true
  pluginVersion: 12.3.0
  type: stat
- targets:
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    expr: {{ printf "sum(vm_app_version{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}) by(job,short_version,%s)" $clusterLabel $clusterLabel }}
    editorMode: code
    exemplar: false
    format: table
    instant: true
    range: false
    refId: A
  datasource:
    type: {{ $defaultDatasource }}
    uid: $ds
  fieldConfig:
    defaults:
      color:
        mode: thresholds
      custom:
        align: auto
        cellOptions:
          type: auto
        footer:
          reducers: []
        inspect: false
        minWidth: 50
      mappings: []
      thresholds:
        mode: absolute
        steps:
        - color: green
          value: 0
        - color: red
          value: 80
    overrides:
    - matcher:
        id: byName
        options: Time
      properties:
      - id: custom.hideFrom.viz
        value: true
    - matcher:
        id: byName
        options: Value
      properties:
      - id: displayName
        value: Count
  gridPos:
    h: 5
    w: 8
    x: 0
    "y": 7
  id: 149
  options:
    cellHeight: sm
    showHeader: true
    sortBy:
    - desc: true
      displayName: Count
  pluginVersion: 12.3.0
  type: table
- title: Uptime ($job)
  targets:
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    expr: {{ printf "sum(min_over_time(up{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}[$__rate_interval])) by(job,%s)" $clusterLabel $clusterLabel }}
    editorMode: code
    format: time_series
    instant: false
    legendFormat: '{{`{{`}}job{{`}}`}}'
    refId: A
  datasource:
    type: {{ $defaultDatasource }}
    uid: $ds
  fieldConfig:
    defaults:
      color:
        mode: palette-classic
      custom:
        axisBorderShow: false
        axisCenteredZero: false
        axisColorMode: text
        axisLabel: ""
        axisPlacement: auto
        axisSoftMin: 0
        barAlignment: 0
        barWidthFactor: 0.6
        drawStyle: line
        fillOpacity: 0
        gradientMode: none
        hideFrom:
          legend: false
          tooltip: false
          viz: false
        insertNulls: false
        lineInterpolation: stepAfter
        lineWidth: 1
        pointSize: 5
        scaleDistribution:
          type: linear
        showPoints: never
        showValues: false
        spanNulls: false
        stacking:
          group: A
          mode: none
        thresholdsStyle:
          mode: "off"
      decimals: 0
      links: []
      mappings: []
      thresholds:
        mode: absolute
        steps:
        - color: green
          value: 0
        - color: red
          value: 80
      unit: none
    overrides: []
  gridPos:
    h: 5
    w: 16
    x: 8
    "y": 7
  id: 62
  options:
    legend:
      calcs:
      - lastNotNull
      displayMode: table
      placement: right
      showLegend: true
      sortBy: Last *
      sortDesc: true
    tooltip:
      hideZeros: false
      mode: multi
      sort: desc
  pluginVersion: 12.3.0
  type: timeseries
- title: Overview
  collapsed: false
  gridPos:
    h: 1
    w: 24
    x: 0
    "y": 12
  id: 10
  type: row
- title: Datapoints ingestion rate ($instance)
  targets:
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    expr: {{ printf "sum(rate(vm_rows_inserted_total{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}[$__rate_interval])) by(type,%s) > 0" $clusterLabel $clusterLabel }}
    editorMode: code
    exemplar: true
    interval: ""
    legendFormat: '{{`{{`}}type{{`}}`}}'
    range: true
    refId: A
  datasource:
    type: {{ $defaultDatasource }}
    uid: $ds
  description: How many [data samples](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#raw-samples) are inserted into cluster per second by protocol before the replication. Check vminsert metrics if there are any issues with ingestion.
  fieldConfig:
    defaults:
      color:
        mode: palette-classic
      custom:
        axisBorderShow: false
        axisCenteredZero: false
        axisColorMode: text
        axisLabel: ""
        axisPlacement: auto
        barAlignment: 0
        barWidthFactor: 0.6
        drawStyle: line
        fillOpacity: 10
        gradientMode: none
        hideFrom:
          legend: false
          tooltip: false
          viz: false
        insertNulls: false
        lineInterpolation: linear
        lineStyle:
          fill: solid
        lineWidth: 1
        pointSize: 5
        scaleDistribution:
          type: linear
        showPoints: never
        showValues: false
        spanNulls: false
        stacking:
          group: A
          mode: normal
        thresholdsStyle:
          mode: "off"
      links: []
      mappings: []
      min: 0
      thresholds:
        mode: absolute
        steps:
        - color: green
          value: 0
        - color: red
          value: 80
      unit: short
    overrides: []
  gridPos:
    h: 8
    w: 12
    x: 0
    "y": 13
  id: 2
  options:
    legend:
      calcs:
      - mean
      - lastNotNull
      - max
      displayMode: table
      placement: bottom
      showLegend: true
      sortBy: Last *
      sortDesc: true
    tooltip:
      hideZeros: false
      mode: multi
      sort: none
  pluginVersion: 12.3.0
  type: timeseries
- title: Requests rate ($instance)
  targets:
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    expr: {{ printf "sum(rate(vm_http_requests_total{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}[$__rate_interval])) by(path,%s) > 0" $clusterLabel $clusterLabel }}
    editorMode: code
    format: time_series
    intervalFactor: 1
    legendFormat: '{{`{{`}}path{{`}}`}}'
    range: true
    refId: A
  datasource:
    type: {{ $defaultDatasource }}
    uid: $ds
  description: |-
    * `*` - unsupported query path
    * `/write` - insert into VM
    * `/metrics` - query VM system metrics
    * `/query` - query instant values
    * `/query_range` - query over a range of time
    * `/series` - match a certain label set
    * `/label/{}/values` - query a list of label values (variables mostly)
  fieldConfig:
    defaults:
      color:
        mode: palette-classic
      custom:
        axisBorderShow: false
        axisCenteredZero: false
        axisColorMode: text
        axisLabel: ""
        axisPlacement: auto
        barAlignment: 0
        barWidthFactor: 0.6
        drawStyle: line
        fillOpacity: 0
        gradientMode: none
        hideFrom:
          legend: false
          tooltip: false
          viz: false
        insertNulls: false
        lineInterpolation: linear
        lineWidth: 1
        pointSize: 5
        scaleDistribution:
          type: linear
        showPoints: never
        showValues: false
        spanNulls: false
        stacking:
          group: A
          mode: none
        thresholdsStyle:
          mode: "off"
      links: []
      mappings: []
      min: 0
      thresholds:
        mode: absolute
        steps:
        - color: green
          value: 0
        - color: red
          value: 80
      unit: short
    overrides: []
  gridPos:
    h: 8
    w: 12
    x: 12
    "y": 13
  id: 6
  options:
    legend:
      calcs:
      - mean
      - lastNotNull
      - max
      displayMode: table
      placement: bottom
      showLegend: true
      sortBy: Last *
      sortDesc: true
    tooltip:
      hideZeros: false
      mode: multi
      sort: desc
  pluginVersion: 12.3.0
  type: timeseries
- title: Active time series ($instance)
  targets:
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    expr: {{ printf "sum(vm_cache_entries{job=~\"$job\",instance=~\"$instance\",type=\"storage/hour_metric_ids\",%s=~\"$cluster\"}) by(%s)" $clusterLabel $clusterLabel }}
    format: time_series
    intervalFactor: 1
    legendFormat: Active time series
    refId: A
  datasource:
    type: {{ $defaultDatasource }}
    uid: $ds
  description: Shows the number of [active time series](https://docs.victoriametrics.com/victoriametrics/faq/#what-is-an-active-time-series) with new data points inserted during the last hour across all storage nodes. High value may result in ingestion slowdown and high memory usage.
  fieldConfig:
    defaults:
      color:
        mode: palette-classic
      custom:
        axisBorderShow: false
        axisCenteredZero: false
        axisColorMode: text
        axisLabel: ""
        axisPlacement: auto
        barAlignment: 0
        barWidthFactor: 0.6
        drawStyle: line
        fillOpacity: 10
        gradientMode: none
        hideFrom:
          legend: false
          tooltip: false
          viz: false
        insertNulls: false
        lineInterpolation: linear
        lineWidth: 1
        pointSize: 5
        scaleDistribution:
          type: linear
        showPoints: never
        showValues: false
        spanNulls: false
        stacking:
          group: A
          mode: none
        thresholdsStyle:
          mode: "off"
      links: []
      mappings: []
      min: 0
      thresholds:
        mode: absolute
        steps:
        - color: green
          value: 0
        - color: red
          value: 80
      unit: short
    overrides: []
  gridPos:
    h: 8
    w: 12
    x: 0
    "y": 21
  id: 12
  options:
    legend:
      calcs:
      - mean
      - lastNotNull
      - max
      displayMode: table
      placement: bottom
      showLegend: true
      sortBy: Last *
      sortDesc: true
    tooltip:
      hideZeros: false
      mode: multi
      sort: none
  pluginVersion: 12.3.0
  type: timeseries
- title: Query duration 0.99 quantile ($instance)
  targets:
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    expr: {{ printf "max(vm_request_duration_seconds{job=~\"$job\",instance=~\"$instance\",quantile=\"0.99\",%s=~\"$cluster\"}) by(path,%s) > 0" $clusterLabel $clusterLabel }}
    editorMode: code
    format: time_series
    intervalFactor: 1
    legendFormat: '{{`{{`}}path{{`}}`}}'
    range: true
    refId: A
  datasource:
    type: {{ $defaultDatasource }}
    uid: $ds
  description: |-
    The less time it takes is better.
    * `*` - unsupported query path
    * `/write` - insert into VM
    * `/metrics` - query VM system metrics
    * `/query` - query instant values
    * `/query_range` - query over a range of time
    * `/series` - match a certain label set
    * `/label/{}/values` - query a list of label values (variables mostly)
  fieldConfig:
    defaults:
      color:
        mode: palette-classic
      custom:
        axisBorderShow: false
        axisCenteredZero: false
        axisColorMode: text
        axisLabel: ""
        axisPlacement: auto
        barAlignment: 0
        barWidthFactor: 0.6
        drawStyle: line
        fillOpacity: 10
        gradientMode: none
        hideFrom:
          legend: false
          tooltip: false
          viz: false
        insertNulls: false
        lineInterpolation: linear
        lineWidth: 1
        pointSize: 5
        scaleDistribution:
          type: linear
        showPoints: never
        showValues: false
        spanNulls: false
        stacking:
          group: A
          mode: none
        thresholdsStyle:
          mode: "off"
      links: []
      mappings: []
      min: 0
      thresholds:
        mode: absolute
        steps:
        - color: green
          value: 0
        - color: red
          value: 80
      unit: s
    overrides: []
  gridPos:
    h: 8
    w: 12
    x: 12
    "y": 21
  id: 8
  options:
    legend:
      calcs:
      - mean
      - lastNotNull
      - max
      displayMode: table
      placement: bottom
      showLegend: true
      sortBy: Last *
      sortDesc: true
    tooltip:
      hideZeros: false
      mode: multi
      sort: desc
  pluginVersion: 12.3.0
  type: timeseries
- title: Requests error rate ($instance)
  targets:
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    expr: {{ printf "sum(rate(vm_http_request_errors_total{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}[$__rate_interval])) by(job,path,%s) > 0" $clusterLabel $clusterLabel }}
    editorMode: code
    format: time_series
    intervalFactor: 1
    legendFormat: '{{`{{`}}path{{`}}`}} ({{`{{`}}job{{`}}`}})'
    range: true
    refId: A
  datasource:
    type: {{ $defaultDatasource }}
    uid: $ds
  description: |-
    * `*` - unsupported query path
    * `/write` - insert into VM
    * `/metrics` - query VM system metrics
    * `/query` - query instant values
    * `/query_range` - query over a range of time
    * `/series` - match a certain label set
    * `/label/{}/values` - query a list of label values (variables mostly)
  fieldConfig:
    defaults:
      color:
        mode: palette-classic
      custom:
        axisBorderShow: false
        axisCenteredZero: false
        axisColorMode: text
        axisLabel: ""
        axisPlacement: auto
        barAlignment: 0
        barWidthFactor: 0.6
        drawStyle: line
        fillOpacity: 10
        gradientMode: none
        hideFrom:
          legend: false
          tooltip: false
          viz: false
        insertNulls: false
        lineInterpolation: linear
        lineWidth: 1
        pointSize: 5
        scaleDistribution:
          type: linear
        showPoints: never
        showValues: false
        spanNulls: false
        stacking:
          group: A
          mode: none
        thresholdsStyle:
          mode: "off"
      links: []
      mappings: []
      min: 0
      thresholds:
        mode: absolute
        steps:
        - color: green
          value: 0
        - color: red
          value: 80
      unit: short
    overrides: []
  gridPos:
    h: 8
    w: 12
    x: 0
    "y": 29
  id: 52
  options:
    legend:
      calcs:
      - mean
      - lastNotNull
      - max
      displayMode: table
      placement: bottom
      showLegend: true
      sortBy: Last *
      sortDesc: true
    tooltip:
      hideZeros: false
      mode: multi
      sort: desc
  pluginVersion: 12.3.0
  type: timeseries
- title: Logging rate
  targets:
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    expr: {{ printf "sum(rate(vm_log_messages_total{job=~\"$job\",instance=~\"$instance\",level!=\"info\",%s=~\"$cluster\"}[$__rate_interval])) by(job,level,%s) > 0" $clusterLabel $clusterLabel }}
    editorMode: code
    exemplar: true
    format: time_series
    hide: false
    interval: 5m
    intervalFactor: 1
    legendFormat: '{{`{{`}}job{{`}}`}} - {{`{{`}}level{{`}}`}}'
    range: true
    refId: A
  datasource:
    type: {{ $defaultDatasource }}
    uid: $ds
  description: Shows the rate of logging the messages by their level. Unexpected spike in rate is a good reason to check logs.
  fieldConfig:
    defaults:
      color:
        mode: palette-classic
      custom:
        axisBorderShow: false
        axisCenteredZero: false
        axisColorMode: text
        axisLabel: ""
        axisPlacement: auto
        barAlignment: 0
        barWidthFactor: 0.6
        drawStyle: bars
        fillOpacity: 100
        gradientMode: none
        hideFrom:
          legend: false
          tooltip: false
          viz: false
        insertNulls: false
        lineInterpolation: linear
        lineWidth: 1
        pointSize: 5
        scaleDistribution:
          type: linear
        showPoints: never
        showValues: false
        spanNulls: false
        stacking:
          group: A
          mode: none
        thresholdsStyle:
          mode: "off"
      links:
      - targetBlank: true
        title: Drilldown
        url: /d/oS7Bi_0Wz?viewPanel=203&var-job=${__field.labels.job}&var-ds=$ds&${__url_time_range}
      mappings: []
      min: 0
      thresholds:
        mode: absolute
        steps:
        - color: green
          value: 0
        - color: red
          value: 80
      unit: short
    overrides: []
  gridPos:
    h: 8
    w: 12
    x: 12
    "y": 29
  id: 104
  options:
    legend:
      calcs:
      - mean
      - lastNotNull
      - max
      displayMode: table
      placement: bottom
      showLegend: true
      sortBy: Last *
      sortDesc: true
    tooltip:
      hideZeros: false
      mode: multi
      sort: desc
  pluginVersion: 12.3.0
  type: timeseries
- title: Resource usage ($job)
  panels:
  - title: RSS memory % usage ($instance)
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "max(max_over_time(process_resident_memory_bytes{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}[$__rate_interval]) / vm_available_memory_bytes{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}) by(job,%s)" $clusterLabel $clusterLabel $clusterLabel }}
      editorMode: code
      exemplar: true
      interval: ""
      legendFormat: __auto
      range: true
      refId: A
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: |-
      How much resident memory (RAM) the process is actually using, compared to its allowed container or system limit.

      - Good: Below 70% most of the time, maybe spiking a bit under load.
      - Bad: Above 90% for more than 5 minutes = risk of out-of-memory (OOM) kill.

      Click on the line and choose Drilldown to show memory usage per instance
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ""
          axisPlacement: auto
          barAlignment: 0
          barWidthFactor: 0.6
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          showValues: false
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: "off"
        links:
        - targetBlank: true
          title: Drilldown - RSS memory usage
          url: /d/oS7Bi_0Wz?viewPanel=189&var-job=${__field.labels.job}&var-ds=$ds&${__url_time_range}
        - targetBlank: true
          title: Drilldown - Memory usage breakdown
          url: /d/oS7Bi_0Wz?viewPanel=225&var-job=${__field.labels.job}&var-ds=$ds&${__url_time_range}
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
            value: 0
          - color: red
            value: 80
        unit: percentunit
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 0
      "y": 38
    id: 66
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        hideZeros: false
        mode: multi
        sort: desc
    pluginVersion: 12.3.0
    type: timeseries
  - title: CPU ($instance)
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "max(rate(process_cpu_seconds_total{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}[$__rate_interval]) / process_cpu_cores_available{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}) by(job,%s)" $clusterLabel $clusterLabel $clusterLabel }}
      editorMode: code
      exemplar: true
      format: time_series
      interval: ""
      intervalFactor: 1
      legendFormat: __auto
      range: true
      refId: A
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ""
          axisPlacement: auto
          barAlignment: 0
          barWidthFactor: 0.6
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          showValues: false
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: "off"
        links:
        - targetBlank: true
          title: Drilldown
          url: /d/oS7Bi_0Wz?viewPanel=192&var-job=${__field.labels.job}&var-ds=$ds&${__url_time_range}
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
            value: 0
          - color: red
            value: 80
        unit: percentunit
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 12
      "y": 38
    id: 64
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        hideZeros: false
        mode: multi
        sort: none
    pluginVersion: 12.3.0
    type: timeseries
  - title: RSS anonymous memory % usage ($instance)
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "max(max_over_time(process_resident_memory_anon_bytes{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}[$__rate_interval]) / vm_available_memory_bytes{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}) by(job,%s)" $clusterLabel $clusterLabel $clusterLabel }}
      editorMode: code
      exemplar: true
      interval: ""
      legendFormat: __auto
      range: true
      refId: A
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: |-
      Share for memory allocated by the process itself. When memory usage reaches 100% it will be likely OOM-killed.
      Safe memory usage % considered to be below 80%

      Click on the line and choose Drilldown to show memory usage per instance
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ""
          axisPlacement: auto
          barAlignment: 0
          barWidthFactor: 0.6
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          showValues: false
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: "off"
        links:
        - targetBlank: true
          title: Drilldown
          url: /d/oS7Bi_0Wz?viewPanel=190&var-job=${__field.labels.job}&var-ds=$ds&${__url_time_range}
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
            value: 0
          - color: red
            value: 80
        unit: percentunit
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 0
      "y": 88
    id: 138
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        hideZeros: false
        mode: multi
        sort: desc
    pluginVersion: 12.3.0
    type: timeseries
  - title: CPU pressure
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "max(rate(process_pressure_cpu_waiting_seconds_total{job=~\"$job\",%s=~\"$cluster\"}[$__rate_interval])) by(job,%s)" $clusterLabel $clusterLabel }}
      editorMode: code
      format: time_series
      interval: ""
      intervalFactor: 2
      legendFormat: '{{`{{`}}job{{`}}`}} - waiting'
      range: true
      refId: A
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "max(rate(process_pressure_cpu_stalled_seconds_total{job=~\"$job\",%s=~\"$cluster\"}[$__rate_interval])) by(job,%s)" $clusterLabel $clusterLabel }}
      editorMode: code
      format: time_series
      hide: false
      interval: ""
      intervalFactor: 2
      legendFormat: '{{`{{`}}job{{`}}`}} - stalled'
      range: true
      refId: B
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: |-
      CPU pressure based on [Pressure Stall Information](https://docs.kernel.org/accounting/psi.html). Helps troubleshoot high CPU usage or throttling.
      Lower is better.

      - waiting: The percentage of time at least one task in the process was ready to run (runnable) but couldn't get scheduled on the CPU.
      - stalled: The percentage of time all tasks in the process (except idle ones) were unable to get CPU time — a full CPU stall.

      If there's a CPU burst, it's normal to see waiting or stalled > 1%. It only becomes a concern if it consistently climbs above 5–10% and aligns with latency spikes or GC slowdowns.
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ""
          axisPlacement: auto
          barAlignment: 0
          barWidthFactor: 0.6
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          showValues: false
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: line
        decimals: 0
        links: []
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
            value: 0
        unit: s
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 12
      "y": 88
    id: 219
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        hideZeros: false
        mode: multi
        sort: desc
    pluginVersion: 12.3.0
    type: timeseries
  - title: Memory pressure
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "max(rate(process_pressure_memory_waiting_seconds_total{job=~\"$job\",%s=~\"$cluster\"}[$__rate_interval])) by(job,%s)" $clusterLabel $clusterLabel }}
      editorMode: code
      format: time_series
      interval: ""
      intervalFactor: 2
      legendFormat: '{{`{{`}}job{{`}}`}} - waiting'
      range: true
      refId: A
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "max(rate(process_pressure_memory_stalled_seconds_total{job=~\"$job\",%s=~\"$cluster\"}[$__rate_interval])) by(job,%s)" $clusterLabel $clusterLabel }}
      editorMode: code
      format: time_series
      hide: false
      interval: ""
      intervalFactor: 2
      legendFormat: '{{`{{`}}job{{`}}`}} - stalled'
      range: true
      refId: B
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: |-
      Shows memory pressure based on [Pressure Stall Information](https://docs.kernel.org/accounting/psi.html).
      Lower is better.

      - waiting: Time fraction where at least one thread was blocked on memory.
      - stalled: Time fraction where every thread was blocked on memory (severe pressure).

      If queries slow down and both series spike, the host is likely limited by RAM or I/O throughput.

      See also major page faults rate panel in Troubleshooting section if this metric continued to be high.
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ""
          axisPlacement: auto
          barAlignment: 0
          barWidthFactor: 0.6
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          showValues: false
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: line
        decimals: 0
        links: []
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
            value: 0
        unit: s
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 0
      "y": 96
    id: 220
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        hideZeros: false
        mode: multi
        sort: desc
    pluginVersion: 12.3.0
    type: timeseries
  - title: Disk writes/reads ($instance)
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "sum(rate(process_io_storage_read_bytes_total{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}[$__rate_interval])) by(job,%s) > 0" $clusterLabel $clusterLabel }}
      editorMode: code
      exemplar: true
      format: time_series
      hide: false
      interval: ""
      intervalFactor: 1
      legendFormat: read {{`{{`}}job{{`}}`}}
      range: true
      refId: A
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "sum(rate(process_io_storage_written_bytes_total{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}[$__rate_interval])) by(job,%s) > 0" $clusterLabel $clusterLabel }}
      editorMode: code
      format: time_series
      hide: false
      interval: ""
      intervalFactor: 1
      legendFormat: write {{`{{`}}job{{`}}`}}
      range: true
      refId: B
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: Shows the number of bytes read/write from the storage layer.
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ""
          axisPlacement: auto
          barAlignment: 0
          barWidthFactor: 0.6
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          showValues: false
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: "off"
        links: []
        mappings: []
        thresholds:
          mode: absolute
          steps:
          - color: green
            value: 0
          - color: red
            value: 80
        unit: Bps
      overrides:
      - matcher:
          id: byRegexp
          options: /read .*/
        properties:
        - id: custom.transform
          value: negative-Y
    gridPos:
      h: 8
      w: 12
      x: 12
      "y": 96
    id: 122
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        hideZeros: false
        mode: multi
        sort: desc
    pluginVersion: 12.3.0
    type: timeseries
  - title: Open FDs usage % ($instance)
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "max(max_over_time(process_open_fds{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}[$__rate_interval]) / process_max_fds{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}) by(job,%s)" $clusterLabel $clusterLabel $clusterLabel }}
      editorMode: code
      exemplar: true
      format: time_series
      interval: ""
      intervalFactor: 2
      legendFormat: '{{`{{`}}job{{`}}`}}'
      range: true
      refId: A
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: |-
      Shows the percentage of open file descriptors compared to the limit set in the OS.
      Reaching the limit of open files can cause various issues and must be prevented.

      See how to change limits here https://medium.com/@muhammadtriwibowo/set-permanently-ulimit-n-open-files-in-ubuntu-4d61064429a
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ""
          axisPlacement: auto
          barAlignment: 0
          barWidthFactor: 0.6
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          showValues: false
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: "off"
        decimals: 2
        links: []
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
            value: 0
          - color: red
            value: 80
        unit: percentunit
      overrides:
      - matcher:
          id: byRegexp
          options: /max.*/
        properties:
        - id: color
          value:
            fixedColor: '#C4162A'
            mode: fixed
    gridPos:
      h: 8
      w: 12
      x: 0
      "y": 104
    id: 117
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        hideZeros: false
        mode: multi
        sort: desc
    pluginVersion: 12.3.0
    type: timeseries
  - title: Disk write/read calls ($instance)
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "sum(rate(process_io_read_syscalls_total{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}[$__rate_interval])) by(job,%s) > 0" $clusterLabel $clusterLabel }}
      editorMode: code
      exemplar: true
      format: time_series
      hide: false
      interval: ""
      intervalFactor: 1
      legendFormat: read {{`{{`}}job{{`}}`}}
      range: true
      refId: A
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "sum(rate(process_io_write_syscalls_total{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}[$__rate_interval])) by(job,%s) > 0" $clusterLabel $clusterLabel }}
      editorMode: code
      format: time_series
      hide: false
      interval: ""
      intervalFactor: 1
      legendFormat: write {{`{{`}}job{{`}}`}}
      range: true
      refId: B
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: Shows the number of read/write syscalls such as read, pread, write, pwrite.
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ""
          axisPlacement: auto
          barAlignment: 0
          barWidthFactor: 0.6
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          showValues: false
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: "off"
        links: []
        mappings: []
        thresholds:
          mode: absolute
          steps:
          - color: green
            value: 0
          - color: red
            value: 80
        unit: short
      overrides:
      - matcher:
          id: byRegexp
          options: /read .*/
        properties:
        - id: custom.transform
          value: negative-Y
    gridPos:
      h: 8
      w: 12
      x: 12
      "y": 104
    id: 204
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        hideZeros: false
        mode: multi
        sort: desc
    pluginVersion: 12.3.0
    type: timeseries
  - title: Goroutines ($instance)
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "sum(max_over_time(go_goroutines{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}[$__rate_interval])) by(job,%s)" $clusterLabel $clusterLabel }}
      editorMode: code
      exemplar: true
      format: time_series
      interval: ""
      intervalFactor: 2
      legendFormat: __auto
      range: true
      refId: A
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ""
          axisPlacement: auto
          barAlignment: 0
          barWidthFactor: 0.6
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          showValues: false
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: "off"
        decimals: 0
        links: []
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
            value: 0
          - color: red
            value: 80
        unit: short
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 0
      "y": 112
    id: 68
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        hideZeros: false
        mode: multi
        sort: desc
    pluginVersion: 12.3.0
    type: timeseries
  - title: IO pressure
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "max(rate(process_pressure_io_waiting_seconds_total{job=~\"$job\",%s=~\"$cluster\"}[$__rate_interval])) by(job,%s)" $clusterLabel $clusterLabel }}
      editorMode: code
      format: time_series
      interval: ""
      intervalFactor: 2
      legendFormat: '{{`{{`}}job{{`}}`}} - waiting'
      range: true
      refId: A
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "max(rate(process_pressure_io_stalled_seconds_total{job=~\"$job\",%s=~\"$cluster\"}[$__rate_interval])) by(job,%s)" $clusterLabel $clusterLabel }}
      editorMode: code
      format: time_series
      hide: false
      interval: ""
      intervalFactor: 2
      legendFormat: '{{`{{`}}job{{`}}`}} - stalled'
      range: true
      refId: B
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: |-
      Shows IO pressure based on [Pressure Stall Information](https://docs.kernel.org/accounting/psi.html).
      The lower the better.

      - waiting: at least one runnable thread blocked on block-`I/O` (disk, NVMe, network-storage) while others could still make progress.
      - stalled: all non-idle threads simultaneously waiting on `I/O`; no useful user code ran during these periods → true `I/O` thrashing.

      If stalled > 0 while querying, it's recommended to increase queue depth on NVMe, raise blk-mq budgets, or relax cgroup I/O limits.
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ""
          axisPlacement: auto
          barAlignment: 0
          barWidthFactor: 0.6
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          showValues: false
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: line
        decimals: 0
        links: []
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
            value: 0
        unit: s
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 12
      "y": 112
    id: 221
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        hideZeros: false
        mode: multi
        sort: desc
    pluginVersion: 12.3.0
    type: timeseries
  - title: Threads ($instance)
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "sum(max_over_time(process_num_threads{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}[$__rate_interval])) by(job,%s)" $clusterLabel $clusterLabel }}
      editorMode: code
      exemplar: true
      format: time_series
      interval: ""
      intervalFactor: 2
      legendFormat: __auto
      range: true
      refId: A
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ""
          axisPlacement: auto
          barAlignment: 0
          barWidthFactor: 0.6
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          showValues: false
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: "off"
        decimals: 0
        links: []
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
            value: 0
          - color: red
            value: 80
        unit: short
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 0
      "y": 120
    id: 70
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        hideZeros: false
        mode: multi
        sort: desc
    pluginVersion: 12.3.0
    type: timeseries
  - title: TCP connections ($instance)
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "sum(max_over_time(vm_tcplistener_conns{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}[$__interval])) by(job,%s)" $clusterLabel $clusterLabel }}
      editorMode: code
      exemplar: true
      interval: 1m
      legendFormat: '{{`{{`}}job{{`}}`}}'
      range: true
      refId: A
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ""
          axisPlacement: auto
          barAlignment: 0
          barWidthFactor: 0.6
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          showValues: false
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: "off"
        mappings: []
        thresholds:
          mode: absolute
          steps:
          - color: green
            value: 0
          - color: red
            value: 80
        unit: short
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 12
      "y": 120
    id: 119
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        hideZeros: false
        mode: multi
        sort: desc
    pluginVersion: 12.3.0
    type: timeseries
  - title: CPU spent on GC ($instance)
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "max(rate(go_gc_cpu_seconds_total{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}[$__rate_interval]) / rate(process_cpu_seconds_total{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}[$__rate_interval])) by(job,%s)" $clusterLabel $clusterLabel $clusterLabel }}
      editorMode: code
      format: time_series
      interval: ""
      intervalFactor: 2
      legendFormat: __auto
      range: true
      refId: A
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: "Shows the percent of CPU spent on garbage collection.\n\nIf % is high, then CPU usage can be decreased by changing GOGC to higher values. Increasing GOGC value will increase memory usage, and decrease CPU usage.\n\nTry searching for keyword `GOGC` at https://docs.victoriametrics.com/victoriametrics/troubleshooting/ "
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ""
          axisPlacement: auto
          barAlignment: 0
          barWidthFactor: 0.6
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          showValues: false
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: "off"
        decimals: 0
        links: []
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
            value: 0
          - color: red
            value: 80
        unit: percentunit
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 0
      "y": 128
    id: 210
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
      tooltip:
        hideZeros: false
        mode: multi
        sort: desc
    pluginVersion: 12.3.0
    type: timeseries
  - title: TCP connections rate ($instance)
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "sum(rate(vm_tcplistener_accepts_total{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}[$__rate_interval])) by(job,%s)" $clusterLabel $clusterLabel }}
      editorMode: code
      exemplar: true
      interval: ""
      legendFormat: __auto
      range: true
      refId: A
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ""
          axisPlacement: auto
          barAlignment: 0
          barWidthFactor: 0.6
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          showValues: false
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: "off"
        mappings: []
        thresholds:
          mode: absolute
          steps:
          - color: green
            value: 0
          - color: red
            value: 80
        unit: short
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 12
      "y": 128
    id: 120
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        hideZeros: false
        mode: multi
        sort: desc
    pluginVersion: 12.3.0
    type: timeseries
  - title: Memory allocations rate
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "max(rate(go_memstats_alloc_bytes_total{job=~\"$job\",%s=~\"$cluster\"}[$__rate_interval])) by(job,%s)" $clusterLabel $clusterLabel }}
      editorMode: code
      format: time_series
      interval: ""
      intervalFactor: 2
      legendFormat: __auto
      range: true
      refId: A
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: Shows the rate of allocations in memory. Sudden increase in allocations would mean increased pressure on Go Garbage Collector and can saturate CPU resources of the application.
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ""
          axisPlacement: auto
          barAlignment: 0
          barWidthFactor: 0.6
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          showValues: false
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: line
        decimals: 0
        links: []
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
            value: 0
        unit: bytes
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 0
      "y": 136
    id: 218
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
      tooltip:
        hideZeros: false
        mode: multi
        sort: desc
    pluginVersion: 12.3.0
    type: timeseries
  - title: Go scheduling latency
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "max(histogram_quantile(0.99, sum(rate(go_sched_latencies_seconds_bucket{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}[$__rate_interval])) by(job,instance,le,%s))) by(job,%s)" $clusterLabel $clusterLabel $clusterLabel }}
      editorMode: code
      format: time_series
      interval: ""
      intervalFactor: 2
      legendFormat: __auto
      range: true
      refId: A
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: "Shows the time goroutines have spent in runnable state before actually running. The lower is better.\n\nHigh values or values exceeding the threshold is usually a sign of            insufficient CPU resources or CPU throttling. \n\nVerify that service has enough CPU resources. Otherwise, the service could work unreliably with delays in processing."
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ""
          axisPlacement: auto
          barAlignment: 0
          barWidthFactor: 0.6
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          showValues: false
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: line
        decimals: 0
        links: []
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
            value: 0
          - color: red
            value: 0.1
        unit: s
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 12
      "y": 136
    id: 213
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
      tooltip:
        hideZeros: false
        mode: multi
        sort: desc
    pluginVersion: 12.3.0
    type: timeseries
  collapsed: true
  gridPos:
    h: 1
    w: 24
    x: 0
    "y": 37
  id: 46
  type: row
- title: Troubleshooting
  panels:
  - description: ""
    fieldConfig:
      defaults: {}
      overrides: []
    gridPos:
      h: 2
      w: 24
      x: 0
      "y": 204
    id: 211
    options:
      code:
        language: plaintext
        showLineNumbers: false
        showMiniMap: false
      content: See [Troubleshooting](https://docs.victoriametrics.com/victoriametrics/troubleshooting/) docs.
      mode: markdown
    pluginVersion: 12.2.0
    transparent: true
    type: text
  - title: Churn rate ($instance)
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "sum(rate(vm_new_timeseries_created_total{job=~\"$job_storage\",instance=~\"$instance\",%s=~\"$cluster\"}[$__rate_interval])) by(%s)" $clusterLabel $clusterLabel }}
      exemplar: true
      interval: ""
      legendFormat: churn rate
      refId: A
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "sum(increase(vm_new_timeseries_created_total{job=~\"$job_storage\",instance=~\"$instance\",%s=~\"$cluster\"}[24h])) by(%s)" $clusterLabel $clusterLabel }}
      exemplar: true
      hide: false
      interval: ""
      legendFormat: new series over 24h
      refId: B
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: |-
      Shows the rate and total number of new series created over last 24h.

      High [churn rate](https://docs.victoriametrics.com/victoriametrics/faq/#what-is-high-churn-rate) tightly connected with database performance and may result in unexpected OOM's or slow queries. It is recommended to always keep an eye on this metric to avoid unexpected [cardinality](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#cardinality) "explosions".

      The higher churn rate is, the more resources required to handle it. Consider to keep the churn rate as low as possible.

      To investigate stats about most expensive series use `api/v1/status/tsdb` handler. More details here https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#url-format

      Good references to read:
      * https://www.robustperception.io/cardinality-is-key
      * https://valyala.medium.com/high-cardinality-tsdb-benchmarks-victoriametrics-vs-timescaledb-vs-influxdb-13e6ee64dd6b
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ""
          axisPlacement: auto
          barAlignment: 0
          barWidthFactor: 0.6
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          showValues: false
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: "off"
        links: []
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
            value: 0
          - color: red
            value: 80
        unit: short
      overrides:
      - matcher:
          id: byName
          options: new series over 24h
        properties:
        - id: custom.axisPlacement
          value: right
        - id: custom.axisSoftMin
          value: 0
    gridPos:
      h: 8
      w: 12
      x: 0
      "y": 206
    id: 102
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        hideZeros: false
        mode: multi
        sort: none
    pluginVersion: 12.2.0
    type: timeseries
  - title: Slow inserts
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "max(rate(vm_slow_row_inserts_total{job=~\"$job_storage\",%s=~\"$cluster\"}[$__rate_interval]) / rate(vm_rows_added_to_storage_total{job=~\"$job_storage\",%s=~\"$cluster\"}[$__rate_interval])) by(%s)" $clusterLabel $clusterLabel $clusterLabel }}
      editorMode: code
      interval: ""
      legendFormat: slow inserts
      range: true
      refId: A
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: "The percentage of [slow inserts](https://docs.victoriametrics.com/victoriametrics/faq/#what-is-a-slow-insert) compared to the total ingestion rate. \n\nThe lower the better. \n\nIn short, slow insert is a cache miss. There are following reasons for slow inserts to go up: \n* Ingestion of completely new, not seen before time series;\n* [Re-routing](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#cluster-availability) of series when one or more vmstorage nodes are unavailable;\n* Not enough memory to maintain big enough caches for the current workload.\n\nIf percentage remains high (>10%) during extended periods of time, then it is likely more RAM is needed for optimal handling of the current number of [active time series](https://docs.victoriametrics.com/victoriametrics/faq/#what-is-an-active-time-series). \n\nSee [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3976#issuecomment-1476883183) for details."
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ""
          axisPlacement: auto
          barAlignment: 0
          barWidthFactor: 0.6
          drawStyle: line
          fillOpacity: 10
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          showValues: false
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: line+area
        links: []
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: transparent
            value: 0
          - color: red
            value: 0.1
        unit: percentunit
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 12
      "y": 206
    id: 108
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
      tooltip:
        hideZeros: false
        mode: multi
        sort: none
    pluginVersion: 12.2.0
    type: timeseries
  - title: Assisted merges ($instance)
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "sum(increase(vm_assisted_merges_total{job=~\"$job_storage\",instance=~\"$instance\",%s=~\"$cluster\"}[$__rate_interval])) by(type,%s) > 0" $clusterLabel $clusterLabel }}
      editorMode: code
      exemplar: true
      format: time_series
      interval: 5m
      intervalFactor: 1
      legendFormat: __auto
      range: true
      refId: A
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: Merge assist happens when vmstorage can't keep up with merging parts. This is usually a sign of overload for vmstorage.
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ""
          axisPlacement: auto
          barAlignment: 0
          barWidthFactor: 0.6
          drawStyle: bars
          fillOpacity: 100
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          showValues: false
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: "off"
        links: []
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
            value: 0
          - color: red
            value: 80
        unit: short
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 0
      "y": 248
    id: 170
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
      tooltip:
        hideZeros: false
        mode: multi
        sort: desc
    pluginVersion: 12.2.0
    type: timeseries
  - title: Slow queries % ($instance)
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "sum(rate(vm_slow_queries_total{job=~\"$job_select\",instance=~\"$instance\",%s=~\"$cluster\"}[$__rate_interval])) by(%s) / sum(rate(vm_http_requests_total{job=~\"$job_select\",instance=~\"$instance\",path=~\"/select/.*\",%s=~\"$cluster\"}[$__rate_interval])) by(%s)" $clusterLabel $clusterLabel $clusterLabel $clusterLabel }}
      editorMode: code
      interval: ""
      legendFormat: slow queries %
      range: true
      refId: A
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: |-
      Shows % of slow queries according to `search.logSlowQueryDuration` flag, which is `5s` by default.

      The less value is better.
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ""
          axisPlacement: auto
          axisSoftMin: 0
          barAlignment: 0
          barWidthFactor: 0.6
          drawStyle: line
          fillOpacity: 10
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          showValues: false
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: "off"
        links: []
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
            value: 0
          - color: red
            value: 80
        unit: percentunit
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 12
      "y": 248
    id: 107
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
      tooltip:
        hideZeros: false
        mode: multi
        sort: none
    pluginVersion: 12.2.0
    type: timeseries
  - title: Cache usage % by type ($instance)
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "max(vm_cache_size_bytes{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"} / vm_cache_size_max_bytes{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}) by(type,%s)" $clusterLabel $clusterLabel $clusterLabel }}
      editorMode: code
      exemplar: true
      interval: ""
      legendFormat: __auto
      range: true
      refId: A
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: "Shows the percentage of used cache size from the allowed size by type. \nValues close to 100% show the maximum potential utilization.\nValues close to 0% show that cache is underutilized."
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ""
          axisPlacement: auto
          barAlignment: 0
          barWidthFactor: 0.6
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          showValues: false
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: "off"
        mappings: []
        thresholds:
          mode: absolute
          steps:
          - color: green
            value: 0
          - color: red
            value: 80
        unit: percentunit
      overrides: []
    gridPos:
      h: 9
      w: 12
      x: 0
      "y": 256
    id: 144
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
      tooltip:
        hideZeros: false
        mode: multi
        sort: desc
    pluginVersion: 12.2.0
    type: timeseries
  - title: Cache miss ratio ($instance)
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "max(rate(vm_cache_misses_total{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}[$__rate_interval]) / rate(vm_cache_requests_total{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}[$__rate_interval])) by(type,%s)" $clusterLabel $clusterLabel $clusterLabel }}
      editorMode: code
      exemplar: true
      format: time_series
      hide: false
      interval: ""
      intervalFactor: 1
      legendFormat: __auto
      range: true
      refId: A
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: Shows cache miss ratio. Lower is better.
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ""
          axisPlacement: auto
          barAlignment: 0
          barWidthFactor: 0.6
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          showValues: false
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: "off"
        links: []
        mappings: []
        max: 1
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
            value: 0
          - color: red
            value: 80
        unit: percentunit
      overrides: []
    gridPos:
      h: 9
      w: 12
      x: 12
      "y": 256
    id: 58
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        hideZeros: false
        mode: multi
        sort: desc
    pluginVersion: 12.2.0
    type: timeseries
  - title: Storage in readonly status for vminsert ($instance)
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "sum(vm_rpc_vmstorage_is_read_only{job=~\"$job_insert\",instance=~\"$instance\",%s=~\"$cluster\"}) by(instance,addr,%s) > 0" $clusterLabel $clusterLabel }}
      editorMode: code
      exemplar: true
      format: time_series
      interval: ""
      intervalFactor: 1
      legendFormat: '{{`{{`}}instance{{`}}`}} => {{`{{`}}addr{{`}}`}}'
      range: true
      refId: A
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: The value is above 0 when the vmstorage at the given `addr` communicates to the given vminsert node that it cannot accept new data because it is in the read-only mode.
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ""
          axisPlacement: auto
          barAlignment: 0
          barWidthFactor: 0.6
          drawStyle: line
          fillOpacity: 10
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          showValues: false
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: "off"
        links: []
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
            value: 0
          - color: red
            value: 80
        unit: short
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 0
      "y": 265
    id: 142
    links:
    - targetBlank: true
      title: Readonly mode
      url: https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#readonly-mode
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
      tooltip:
        hideZeros: false
        mode: multi
        sort: none
    pluginVersion: 12.2.0
    type: timeseries
  - title: Deduplication rate ($instance)
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "sum(rate(vm_deduplicated_samples_total{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}[$__rate_interval])) by(job,type,%s) > 0" $clusterLabel $clusterLabel }}
      editorMode: code
      exemplar: true
      format: time_series
      interval: ""
      intervalFactor: 1
      legendFormat: __auto
      range: true
      refId: A
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: |-
      Shows rate of deduplicated samples during [deduplication](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#deduplication) or [downsampling](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#downsampling), according to the configured `-dedup.minScrapeInterval` or `-downsampling.period`.

      Should be empty if `-dedup.minScrapeInterval` and `-downsampling.period` isn't set.

      Deduplication happens in two places:
      * `type="merge"` - during [background merges](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#storage) by vmstorages. Deduplication during merges is permanent.
      * `type="select"` - during [read queries](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#query-data) by vmselects. Deduplication happens in-flight and has no permanent effect.

      The more samples need to be deduplicated, the higher will be resource usage on vmselects or vmstorages.
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ""
          axisPlacement: auto
          barAlignment: 0
          barWidthFactor: 0.6
          drawStyle: line
          fillOpacity: 10
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          showValues: false
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: "off"
        links: []
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
            value: 0
          - color: red
            value: 80
        unit: short
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 12
      "y": 265
    id: 215
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
      tooltip:
        hideZeros: false
        mode: multi
        sort: none
    pluginVersion: 12.2.0
    type: timeseries
  - title: Samples dropped for last 1h ($instance)
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "sum(increase(vm_rows_ignored_total{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}[1h])) by(reason,%s)" $clusterLabel $clusterLabel }}
      editorMode: code
      exemplar: true
      hide: false
      interval: ""
      legendFormat: __auto
      range: true
      refId: A
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "sum(increase(vm_relabel_metrics_dropped_total{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}[1h])) by(%s)" $clusterLabel $clusterLabel }}
      editorMode: code
      hide: false
      instant: false
      legendFormat: relabeling
      range: true
      refId: B
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: |-
      Shows how many samples were ignored or dropped on insertion due to various reasons:
      * timestamp out of retention period or timestamp in future;
      * invalid metric name;
      * exceeding limit for labels length or number;
      * dropped by [relabeling configuration](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#relabeling).
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ""
          axisPlacement: auto
          barAlignment: 0
          barWidthFactor: 0.6
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          showValues: false
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: "off"
        mappings: []
        thresholds:
          mode: absolute
          steps:
          - color: green
            value: 0
          - color: red
            value: 80
        unit: short
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 0
      "y": 273
    id: 135
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        hideZeros: false
        mode: multi
        sort: none
    pluginVersion: 12.2.0
    type: timeseries
  - title: Partial query results ($instance)
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "sum(increase(vm_partial_results_total{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}[$__rate_interval])) by(%s) > 0" $clusterLabel $clusterLabel }}
      editorMode: code
      format: time_series
      instant: false
      legendFormat: partial results
      refId: A
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: |-
      Shows the number of partial results served by `vmselects`.

      `vmselect` continues serving queries if at least a single vmstorage nodes is available. It marks responses as `partial` for queries served from the remaining healthy vmstorage nodes.

      If you prefer consistency over availability then run vmselect nodes with `-search.denyPartialResponse` cmd-line flag. In this case vmselect returns an error if at least a single vmstorage node is unavailable.

      See more at [cluster availability](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#cluster-availability).
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ""
          axisPlacement: auto
          axisSoftMin: 0
          barAlignment: 0
          barWidthFactor: 0.6
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: stepAfter
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          showValues: false
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: "off"
        decimals: 0
        links: []
        mappings: []
        thresholds:
          mode: absolute
          steps:
          - color: green
            value: 0
          - color: red
            value: 80
        unit: none
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 12
      "y": 273
    id: 217
    options:
      legend:
        calcs:
        - lastNotNull
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        hideZeros: false
        mode: multi
        sort: desc
    pluginVersion: 12.2.0
    type: timeseries
  - title: Non-default flags
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "sum(flag{is_set=\"true\",job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}) by(job,instance,name,value,%s)" $clusterLabel $clusterLabel }}
      editorMode: code
      exemplar: false
      format: table
      instant: true
      legendFormat: __auto
      range: false
      refId: A
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    fieldConfig:
      defaults:
        color:
          mode: thresholds
        custom:
          align: auto
          cellOptions:
            type: auto
          filterable: true
          footer:
            reducers: []
          inspect: false
        mappings: []
        thresholds:
          mode: absolute
          steps:
          - color: green
            value: 0
          - color: red
            value: 80
      overrides:
      - matcher:
          id: byName
          options: Value
        properties:
        - id: custom.hideFrom.viz
          value: true
      - matcher:
          id: byName
          options: Time
        properties:
        - id: custom.hideFrom.viz
          value: true
    gridPos:
      h: 8
      w: 12
      x: 0
      "y": 281
    id: 183
    options:
      cellHeight: sm
      filterable: true
      showHeader: true
      showSearch: true
      sortBy:
      - desc: true
        displayName: job
    pluginVersion: 12.2.0
    transformations:
    - id: groupBy
      options:
        fields:
          instance:
            aggregations:
            - uniqueValues
            operation: aggregate
          job:
            aggregations: []
            operation: groupby
          name:
            aggregations: []
            operation: groupby
          value:
            aggregations: []
            operation: groupby
    type: table
  - title: Restarts ($job)
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "sum(changes(vm_app_start_timestamp{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}[$__rate_interval])) by(job,%s) > 0" $clusterLabel $clusterLabel }}
      editorMode: code
      format: time_series
      instant: false
      legendFormat: '{{`{{`}}job{{`}}`}}'
      refId: A
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: 'Shows the number of restarts per job. The chart can be useful to identify periodic process restarts and correlate them with potential issues or anomalies. Normally, processes shouldn''t restart unless restart was inited by user. The reason of restarts should be figured out by checking the logs of each specific service. '
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ""
          axisPlacement: auto
          axisSoftMin: 0
          barAlignment: 0
          barWidthFactor: 0.6
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: stepAfter
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          showValues: false
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: "off"
        decimals: 0
        links: []
        mappings: []
        thresholds:
          mode: absolute
          steps:
          - color: green
            value: 0
          - color: red
            value: 80
        unit: none
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 12
      "y": 281
    id: 214
    options:
      legend:
        calcs:
        - lastNotNull
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        hideZeros: false
        mode: multi
        sort: desc
    pluginVersion: 12.2.0
    type: timeseries
  - title: Partitions scheduled for re-processing
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: ${ds}
      expr: {{ printf "sum(vm_downsampling_partitions_scheduled_size_bytes{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}) by(%s) > 0" $clusterLabel $clusterLabel }}
      editorMode: code
      legendFormat: downsampling
      range: true
      refId: A
    - datasource:
        type: {{ $defaultDatasource }}
        uid: ${ds}
      expr: {{ printf "sum(vm_retention_partitions_scheduled_size_bytes{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}) by(%s) > 0" $clusterLabel $clusterLabel }}
      editorMode: code
      hide: false
      instant: false
      legendFormat: retention filters
      range: true
      refId: B
    datasource:
      type: {{ $defaultDatasource }}
      uid: ${ds}
    description: |-
      Shows the amount of data scheduled for [downsampling](https://docs.victoriametrics.com/#downsampling) or [retention filters](https://docs.victoriametrics.com/#retention-filters).

      Applying downsampling or retention filters happens in background. `vmstorage` schedules this process periodically when finds data partitions that satisfy configured retention or downsampling rules.

      This process could consume additional CPU and Disk IO resources.
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ""
          axisPlacement: auto
          axisSoftMin: 0
          barAlignment: 0
          barWidthFactor: 0.6
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: auto
          showValues: false
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: "off"
        mappings: []
        thresholds:
          mode: absolute
          steps:
          - color: green
            value: 0
          - color: red
            value: 80
        unit: bytes
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 0
      "y": 289
    id: 222
    options:
      legend:
        calcs: []
        displayMode: list
        placement: bottom
        showLegend: true
      tooltip:
        hideZeros: false
        mode: single
        sort: none
    pluginVersion: 12.2.0
    type: timeseries
  - title: Major page faults rate  ($instance)
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: ${ds}
      expr: {{ printf "sum(rate(process_major_pagefaults_total{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}[$__rate_interval])) by(job,%s) > 0" $clusterLabel $clusterLabel }}
      editorMode: code
      legendFormat: __auto
      range: true
      refId: A
    datasource:
      type: {{ $defaultDatasource }}
      uid: ${ds}
    description: |-
      Major page faults occur when required data is not in physical memory and must be loaded from disk (storage or swap). It usually happens with:
      - insufficient physical memory
      - active use of swap space
      - file mapping access

      It may cause significant performance slowdown if sustained high.

      Possible workaround:
      - Extend RAM to scale up
      - Reduce swap usage by adjusting vm.swappiness
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ""
          axisPlacement: auto
          barAlignment: 0
          barWidthFactor: 0.6
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: auto
          showValues: false
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: "off"
        links:
        - targetBlank: true
          title: Drilldown
          url: /d/oS7Bi_0Wz?viewPanel=224&var-job=${__field.labels.job}&var-ds=$ds&${__url_time_range}
        mappings: []
        thresholds:
          mode: absolute
          steps:
          - color: green
            value: 0
          - color: red
            value: 80
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 12
      "y": 289
    id: 223
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
      tooltip:
        hideZeros: false
        mode: multi
        sort: desc
    pluginVersion: 12.2.0
    type: timeseries
  collapsed: true
  gridPos:
    h: 1
    w: 24
    x: 0
    "y": 38
  id: 106
  type: row
- title: Interconnection ($job)
  panels:
  - title: Rows ($instance)
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "sum(rate(vm_rpc_rows_pushed_total{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}[$__rate_interval])) by(%s)" $clusterLabel $clusterLabel }}
      editorMode: code
      format: time_series
      intervalFactor: 1
      legendFormat: Pushed
      range: true
      refId: B
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "sum(rate(vm_rpc_rows_sent_total{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}[$__rate_interval])) by(%s)" $clusterLabel $clusterLabel }}
      editorMode: code
      format: time_series
      intervalFactor: 1
      legendFormat: Sent
      range: true
      refId: E
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: "Shows number of pushed and sent rows. \n* `Pushed rows` - rows added to internal inserter buffers before send\n* `Sent rows` - successfully transmitted rows to storage nodes\n\nPlease note, it could be that `Sent > Pushed` because of the replication factor."
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ""
          axisPlacement: auto
          barAlignment: 0
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: "off"
        links: []
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
          - color: red
            value: 80
        unit: short
      overrides: []
    gridPos:
      h: 9
      w: 12
      x: 0
      "y": 8371
    id: 76
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
      tooltip:
        mode: multi
        sort: none
    pluginVersion: 9.1.0
    type: timeseries
  - title: RPC errors ($instance)
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "sum(rate(vm_rpc_connection_errors_total{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}[$__rate_interval])) by(%s)" $clusterLabel $clusterLabel }}
      editorMode: code
      format: time_series
      intervalFactor: 1
      legendFormat: Connection
      range: true
      refId: A
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "sum(rate(vm_rpc_dial_errors_total{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}[$__rate_interval])) by(%s)" $clusterLabel $clusterLabel }}
      format: time_series
      intervalFactor: 1
      legendFormat: Dial
      refId: B
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "sum(rate(vm_rpc_handshake_errors_total{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}[$__rate_interval])) by(%s)" $clusterLabel $clusterLabel }}
      format: time_series
      intervalFactor: 1
      legendFormat: Handshake
      refId: E
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: |-
      Rate of RPC errors by type:
      * `Connection` - the number of connection errors to vmstorage node
      * `Dial` -  the number of dial errors to vmstorage node.
      * `Handshake` - the number of handshake errors to vmstorage node
      * `Rerouted` - errors appeared during rerouting of rows from un-healthy storage node to a healthy one.
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ""
          axisPlacement: auto
          barAlignment: 0
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: "off"
        links: []
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
          - color: red
            value: 80
        unit: short
      overrides: []
    gridPos:
      h: 9
      w: 12
      x: 12
      "y": 8371
    id: 86
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
      tooltip:
        mode: multi
        sort: none
    pluginVersion: 9.1.0
    type: timeseries
  - title: 'Rows ($instance) rerouted to '
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "sum(rate(vm_rpc_rows_rerouted_to_here_total{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}[$__rate_interval])) by(addr,%s) > 0" $clusterLabel $clusterLabel }}
      editorMode: code
      exemplar: true
      format: time_series
      interval: ""
      intervalFactor: 1
      legendFormat: __auto
      range: true
      refId: A
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: The number of rows rerouted to the vmstorage node from other nodes when they were unhealthy.
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ""
          axisPlacement: auto
          barAlignment: 0
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: "off"
        links: []
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
          - color: red
            value: 80
        unit: rps
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 0
      "y": 8380
    id: 80
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
      tooltip:
        mode: multi
        sort: none
    pluginVersion: 9.1.0
    type: timeseries
  - title: Rows ($instance) rerouted from
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "sum(rate(vm_rpc_rows_rerouted_from_here_total{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}[$__rate_interval])) by(addr,%s) > 0" $clusterLabel $clusterLabel }}
      exemplar: true
      format: time_series
      interval: ""
      intervalFactor: 1
      legendFormat: '{{`{{`}}addr{{`}}`}}'
      refId: A
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: The number of rows rerouted from the vmstorage node to healthy nodes when the given node was unhealthy.
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ""
          axisPlacement: auto
          barAlignment: 0
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: "off"
        links: []
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
          - color: red
            value: 80
        unit: rps
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 12
      "y": 8380
    id: 78
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
      tooltip:
        mode: multi
        sort: none
    pluginVersion: 9.1.0
    type: timeseries
  - title: Pending
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "sum(vm_rpc_buf_pending_bytes{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}) by(%s)" $clusterLabel $clusterLabel }}
      legendFormat: bytes
      refId: A
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "sum(vm_rpc_rows_pending{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}) by(%s)" $clusterLabel $clusterLabel }}
      legendFormat: rows
      refId: B
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: The number of rows or bytes that vminsert internal buffer contains at the moment.
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ""
          axisPlacement: auto
          barAlignment: 0
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: "off"
        links: []
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
          - color: red
            value: 80
        unit: short
      overrides:
      - matcher:
          id: byName
          options: bytes
        properties:
        - id: unit
          value: bytes
    gridPos:
      h: 8
      w: 12
      x: 0
      "y": 8388
    id: 82
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
      tooltip:
        mode: multi
        sort: none
    pluginVersion: 9.1.0
    type: timeseries
  - title: RPC network usage ($instance)
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "sum(rate(vm_tcpdialer_written_bytes_total{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}[$__rate_interval])) by(%s) * 8" $clusterLabel $clusterLabel }}
      editorMode: code
      legendFormat: network usage
      range: true
      refId: A
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: Network usage by internal VictoriaMetrics RPC protocol
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ""
          axisPlacement: auto
          barAlignment: 0
          drawStyle: line
          fillOpacity: 10
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: "off"
        links: []
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
          - color: red
            value: 80
        unit: bps
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 12
      "y": 8388
    id: 74
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
      tooltip:
        mode: multi
        sort: none
    pluginVersion: 9.1.0
    type: timeseries
  collapsed: true
  gridPos:
    h: 1
    w: 24
    x: 0
    "y": 39
  id: 48
  type: row
- title: vmstorage ($instance)
  panels:
  - title: Ingestion rate ($instance)
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "sum(rate(vm_vminsert_metrics_read_total{job=~\"$job_storage\",instance=~\"$instance\",%s=~\"$cluster\"}[$__rate_interval])) by(%s)" $clusterLabel $clusterLabel }}
      editorMode: code
      format: time_series
      intervalFactor: 1
      legendFormat: ingestion rate
      range: true
      refId: A
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: Shows the number of datapoints ingested into storage nodes per second. This metric doesn't show all stored datapoints since some of them may be dropped because of wrong timestamps or decode errors.
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ""
          axisPlacement: auto
          barAlignment: 0
          barWidthFactor: 0.6
          drawStyle: line
          fillOpacity: 10
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          showValues: false
          spanNulls: false
          stacking:
            group: A
            mode: normal
          thresholdsStyle:
            mode: "off"
        links: []
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
            value: 0
          - color: red
            value: 80
        unit: short
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 0
      "y": 1037
    id: 100
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        hideZeros: false
        mode: multi
        sort: desc
    pluginVersion: 12.2.0
    type: timeseries
  - title: Storage full ETA ($instance)
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "min(((vm_free_disk_space_bytes{job=~\"$job_storage\",instance=~\"$instance\",%s=~\"$cluster\"} - vm_free_disk_space_limit_bytes{job=~\"$job_storage\",instance=~\"$instance\",%s=~\"$cluster\"}) / ignoring(path) (((rate(vm_rows_added_to_storage_total{job=~\"$job_storage\",instance=~\"$instance\",%s=~\"$cluster\"}[1d]) - sum(rate(vm_deduplicated_samples_total{job=~\"$job_storage\",instance=~\"$instance\",%s=~\"$cluster\"}[1d])) without(type)) * (sum(vm_data_size_bytes{job=~\"$job_storage\",instance=~\"$instance\",type!~\"indexdb.*\",%s=~\"$cluster\"}) without(type) / sum(vm_rows{job=~\"$job_storage\",instance=~\"$instance\",type!~\"indexdb.*\",%s=~\"$cluster\"}) without(type))) + (rate(vm_new_timeseries_created_total{job=~\"$job_storage\",instance=~\"$instance\",%s=~\"$cluster\"}[1d]) * (sum(vm_data_size_bytes{job=~\"$job_storage\",instance=~\"$instance\",type=\"indexdb/file\",%s=~\"$cluster\"}) without(type) / sum(vm_rows{job=~\"$job_storage\",instance=~\"$instance\",type=\"indexdb/file\",%s=~\"$cluster\"}) without(type))))) > 0) by(%s)" $clusterLabel $clusterLabel $clusterLabel $clusterLabel $clusterLabel $clusterLabel $clusterLabel $clusterLabel $clusterLabel $clusterLabel }}
      editorMode: code
      format: time_series
      interval: ""
      intervalFactor: 1
      legendFormat: min ETA
      range: true
      refId: A
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: |-
      Shows the approx time needed to reach 100% of allowed disk capacity for at least one vmstorage node based on the following params:
      * free disk space (after -storage.minFreeDiskSpaceBytes);
      * row ingestion rate;
      * compression.
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ""
          axisPlacement: auto
          barAlignment: 0
          barWidthFactor: 0.6
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          showValues: false
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: "off"
        links:
        - targetBlank: true
          title: Drilldown
          url: /d/oS7Bi_0Wz?viewPanel=196&${__url_time_range}&${__all_variables}
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
            value: 0
          - color: red
            value: 80
        unit: s
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 12
      "y": 1037
    id: 113
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - min
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        hideZeros: false
        mode: multi
        sort: none
    pluginVersion: 12.2.0
    type: timeseries
  - title: CPU usage % ($instance)
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "max(rate(process_cpu_seconds_total{job=~\"$job_storage\",instance=~\"$instance\",%s=~\"$cluster\"}[$__rate_interval]) / process_cpu_cores_available{job=~\"$job_storage\",instance=~\"$instance\",%s=~\"$cluster\"}) by(%s)" $clusterLabel $clusterLabel $clusterLabel }}
      editorMode: code
      exemplar: false
      format: time_series
      interval: ""
      intervalFactor: 1
      legendFormat: max
      range: true
      refId: A
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "min(rate(process_cpu_seconds_total{job=~\"$job_storage\",instance=~\"$instance\",%s=~\"$cluster\"}[$__rate_interval]) / process_cpu_cores_available{job=~\"$job_storage\",instance=~\"$instance\",%s=~\"$cluster\"}) by(%s)" $clusterLabel $clusterLabel $clusterLabel }}
      editorMode: code
      exemplar: false
      format: time_series
      hide: false
      interval: ""
      intervalFactor: 1
      legendFormat: min
      range: true
      refId: B
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "quantile(0.5, rate(process_cpu_seconds_total{job=~\"$job_storage\",instance=~\"$instance\",%s=~\"$cluster\"}[$__rate_interval]) / process_cpu_cores_available{job=~\"$job_storage\",instance=~\"$instance\",%s=~\"$cluster\"}) by(%s)" $clusterLabel $clusterLabel $clusterLabel }}
      editorMode: code
      exemplar: false
      format: time_series
      hide: false
      interval: ""
      intervalFactor: 1
      legendFormat: median
      range: true
      refId: C
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: ""
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ""
          axisPlacement: auto
          barAlignment: 0
          barWidthFactor: 0.6
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          showValues: false
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: line
        links:
        - targetBlank: true
          title: Drilldown
          url: /d/oS7Bi_0Wz?viewPanel=192&var-job=$job_storage&var-ds=$ds&${__url_time_range}
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
            value: 0
          - color: red
            value: 0.9
        unit: percentunit
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 0
      "y": 1045
    id: 151
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        hideZeros: false
        mode: multi
        sort: desc
    pluginVersion: 12.2.0
    type: timeseries
  - title: Memory (anon) usage % ($instance)
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "max(max_over_time(process_resident_memory_anon_bytes{job=~\"$job_storage\",instance=~\"$instance\",%s=~\"$cluster\"}[$__rate_interval]) / vm_available_memory_bytes{job=~\"$job_storage\",instance=~\"$instance\",%s=~\"$cluster\"}) by(%s)" $clusterLabel $clusterLabel $clusterLabel }}
      editorMode: code
      exemplar: true
      format: time_series
      interval: ""
      intervalFactor: 1
      legendFormat: max
      range: true
      refId: A
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "min(max_over_time(process_resident_memory_anon_bytes{job=~\"$job_storage\",instance=~\"$instance\",%s=~\"$cluster\"}[$__rate_interval]) / vm_available_memory_bytes{job=~\"$job_storage\",instance=~\"$instance\",%s=~\"$cluster\"}) by(%s)" $clusterLabel $clusterLabel $clusterLabel }}
      editorMode: code
      exemplar: true
      format: time_series
      hide: false
      interval: ""
      intervalFactor: 1
      legendFormat: min
      range: true
      refId: B
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "quantile(0.5, max_over_time(process_resident_memory_anon_bytes{job=~\"$job_storage\",instance=~\"$instance\",%s=~\"$cluster\"}[$__rate_interval]) / vm_available_memory_bytes{job=~\"$job_storage\",instance=~\"$instance\",%s=~\"$cluster\"}) by(%s)" $clusterLabel $clusterLabel $clusterLabel }}
      editorMode: code
      exemplar: true
      format: time_series
      hide: false
      interval: ""
      intervalFactor: 1
      legendFormat: median
      range: true
      refId: C
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: |-
      Share for memory allocated by the process itself. When memory usage reaches 100% it will be likely OOM-killed.
      Safe memory usage % considered to be below 80%

      Click on the line and choose Drilldown to show memory usage per instance
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ""
          axisPlacement: auto
          barAlignment: 0
          barWidthFactor: 0.6
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          showValues: false
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: line
        links:
        - targetBlank: true
          title: Drilldown
          url: /d/oS7Bi_0Wz?viewPanel=190&var-job=$job_storage&var-ds=$ds&${__url_time_range}
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
            value: 0
          - color: red
            value: 0.9
        unit: percentunit
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 12
      "y": 1045
    id: 167
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        hideZeros: false
        mode: multi
        sort: desc
    pluginVersion: 12.2.0
    type: timeseries
  - title: Concurrent inserts ($instance)
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "max(max_over_time(vm_concurrent_insert_current{job=~\"$job_storage\",instance=~\"$instance\",%s=~\"$cluster\"}[$__rate_interval])) by(%s)" $clusterLabel $clusterLabel }}
      editorMode: code
      exemplar: true
      interval: ""
      legendFormat: current
      range: true
      refId: A
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "min(vm_concurrent_insert_capacity{job=~\"$job_storage\",instance=~\"$instance\",%s=~\"$cluster\"}) by(%s)" $clusterLabel $clusterLabel }}
      editorMode: code
      exemplar: true
      hide: false
      interval: ""
      legendFormat: max
      range: true
      refId: B
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: "Shows the number of ongoing Cluster Native insert API operations sent to `vmstorage`, where:\n* `max` - the maximum number of calls that can be processed in parallel;\n* `current` - Shows the number of ongoing insert operations being processed by the storage.\n\nEach successful API call results in a flush to disk. The `max` value is controlled by the `-maxConcurrentInserts` flag set on the storage (2 * cgroup.AvailableCPUs() by default). \n\nIf `current` consistently reaches `max`, it indicates that the storage is overloaded and likely requires more CPU (see CPU usage) or disks with higher IOPS (see disk read/write panels in the Resource Usage row)"
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ""
          axisPlacement: auto
          barAlignment: 0
          barWidthFactor: 0.6
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          showValues: false
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: "off"
        mappings: []
        thresholds:
          mode: absolute
          steps:
          - color: green
            value: 0
          - color: red
            value: 80
        unit: short
      overrides:
      - matcher:
          id: byName
          options: max
        properties:
        - id: color
          value:
            fixedColor: '#C4162A'
            mode: fixed
    gridPos:
      h: 8
      w: 12
      x: 0
      "y": 1053
    id: 212
    links:
    - targetBlank: true
      title: Related discussion
      url: https://github.com/VictoriaMetrics/VictoriaMetrics/issues/632
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        hideZeros: false
        mode: multi
        sort: none
    pluginVersion: 12.2.0
    type: timeseries
  - title: Concurrent selects ($instance)
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "max(max_over_time(vm_vmselect_concurrent_requests_current{job=~\"$job_storage\",instance=~\"$instance\",%s=~\"$cluster\"}[$__rate_interval])) by(%s)" $clusterLabel $clusterLabel }}
      editorMode: code
      exemplar: true
      interval: ""
      legendFormat: current
      range: true
      refId: A
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "min(vm_vmselect_concurrent_requests_capacity{job=~\"$job_storage\",instance=~\"$instance\",%s=~\"$cluster\"}) by(%s)" $clusterLabel $clusterLabel }}
      editorMode: code
      exemplar: true
      hide: false
      interval: ""
      legendFormat: max
      range: true
      refId: B
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: |-
      Shows the number of concurrently executed read requests, where:
      * `max` - equal to `-search.maxConcurrentRequest` cmd-line flag;
      * `current` - current number of concurrent select requests executed by storage.

      Each concurrent select request requires RAM, CPU and disk IO resources during processing. Too many concurrent requests could result into OOM exceptions.

      When `current` hits `max` constantly, it means storage is overloaded and requires more CPU (see CPU usage) or disks with more IOPS (see disk writes and reads panels in Resource Usage row).

      See more at https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#resource-usage-limits
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ""
          axisPlacement: auto
          barAlignment: 0
          barWidthFactor: 0.6
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          showValues: false
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: "off"
        mappings: []
        thresholds:
          mode: absolute
          steps:
          - color: green
            value: 0
          - color: red
            value: 80
        unit: short
      overrides:
      - matcher:
          id: byName
          options: max
        properties:
        - id: color
          value:
            fixedColor: '#C4162A'
            mode: fixed
    gridPos:
      h: 8
      w: 12
      x: 12
      "y": 1053
    id: 133
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        hideZeros: false
        mode: multi
        sort: none
    pluginVersion: 12.2.0
    type: timeseries
  - title: Active merges ($instance)
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "max(max_over_time(vm_active_merges{job=~\"$job_storage\",instance=~\"$instance\",%s=~\"$cluster\"}[$__rate_interval])) by(type,%s)" $clusterLabel $clusterLabel }}
      editorMode: code
      legendFormat: __auto
      range: true
      refId: A
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: |-
      The max number of on-going merges across storage nodes.
       The drastic change in number of merges could be a sign of on-going deduplication/downsampling activity.
       It is expected to have high numbers for `storage/small` metric.
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ""
          axisPlacement: auto
          barAlignment: 0
          barWidthFactor: 0.6
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          showValues: false
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: "off"
        decimals: 0
        links: []
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
            value: 0
          - color: red
            value: 80
        unit: short
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 0
      "y": 1094
    id: 54
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        hideZeros: false
        mode: multi
        sort: desc
    pluginVersion: 12.2.0
    type: timeseries
  - title: Merge speed
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "sum(rate(vm_rows_merged_total{job=~\"$job_storage\",instance=~\"$instance\",%s=~\"$cluster\"}[$__rate_interval])) by(type,%s)" $clusterLabel $clusterLabel }}
      editorMode: code
      legendFormat: __auto
      range: true
      refId: A
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: The number of rows merged per second by storage nodes. Merge speed depends on available CPU and disk IO bandwidth.
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ""
          axisPlacement: auto
          barAlignment: 0
          barWidthFactor: 0.6
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          showValues: false
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: "off"
        decimals: 0
        links: []
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
            value: 0
          - color: red
            value: 80
        unit: short
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 12
      "y": 1094
    id: 55
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        hideZeros: false
        mode: multi
        sort: desc
    pluginVersion: 12.2.0
    type: timeseries
  - title: Disk space usage %  ($instance)
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "max(sum(vm_data_size_bytes{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}) by(job,instance,%s) / (sum(vm_free_disk_space_bytes{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"} - vm_free_disk_space_limit_bytes{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}) by(job,instance,%s) + sum(vm_data_size_bytes{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}) by(job,instance,%s))) by(%s)" $clusterLabel $clusterLabel $clusterLabel $clusterLabel $clusterLabel $clusterLabel $clusterLabel $clusterLabel }}
      editorMode: code
      format: time_series
      intervalFactor: 1
      legendFormat: max
      range: true
      refId: A
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "min(sum(vm_data_size_bytes{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}) by(job,instance,%s) / (sum(vm_free_disk_space_bytes{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"} - vm_free_disk_space_limit_bytes{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}) by(job,instance,%s) + sum(vm_data_size_bytes{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}) by(job,instance,%s))) by(%s)" $clusterLabel $clusterLabel $clusterLabel $clusterLabel $clusterLabel $clusterLabel $clusterLabel $clusterLabel }}
      editorMode: code
      format: time_series
      hide: false
      intervalFactor: 1
      legendFormat: min
      range: true
      refId: B
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "quantile(0.5, sum(vm_data_size_bytes{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}) by(job,instance,%s) / (sum(vm_free_disk_space_bytes{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"} - vm_free_disk_space_limit_bytes{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}) by(job,instance,%s) + sum(vm_data_size_bytes{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}) by(job,instance,%s))) by(%s)" $clusterLabel $clusterLabel $clusterLabel $clusterLabel $clusterLabel $clusterLabel $clusterLabel $clusterLabel }}
      editorMode: code
      format: time_series
      hide: false
      intervalFactor: 1
      legendFormat: median
      range: true
      refId: C
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: Shows the percentage of used disk space. It is recommended to have at least 20% of free disk space for the best performance.
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ""
          axisPlacement: auto
          barAlignment: 0
          barWidthFactor: 0.6
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          showValues: false
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: line
        links:
        - targetBlank: true
          title: Drilldown
          url: /d/oS7Bi_0Wz?viewPanel=200&var-ds=$ds&${__url_time_range}
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
            value: 0
          - color: red
            value: 80
        unit: percentunit
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 0
      "y": 1102
    id: 20
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        hideZeros: false
        mode: multi
        sort: desc
    pluginVersion: 12.2.0
    type: timeseries
  - title: LSM parts max by type ($instance)
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "max(vm_parts{job=~\"$job_storage\",instance=~\"$instance\",type=~\"indexdb.*\",%s=~\"$cluster\"}) by(type,%s)" $clusterLabel $clusterLabel }}
      editorMode: code
      format: time_series
      interval: ""
      intervalFactor: 1
      legendFormat: __auto
      range: true
      refId: A
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "max(vm_last_partition_parts{job=~\"$job_storage\",instance=~\"$instance\",%s=~\"$cluster\"}) by(type,%s)" $clusterLabel $clusterLabel }}
      editorMode: code
      format: time_series
      hide: false
      interval: ""
      intervalFactor: 1
      legendFormat: __auto
      range: true
      refId: B
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: |-
      The max number of data parts of LSM tree across all storage nodes in the last [partition](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#storage).
      Increase in number of parts (the hard limit is 512) is an evidence of slow merge performance - check the resource utilization.
      * `indexdb` - inverted index
      * `storage/small` - recently added parts of data ingested into storage (hot data)
      * `storage/big` -  small parts gradually merged into bigger parts (cold data)
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ""
          axisPlacement: auto
          barAlignment: 0
          barWidthFactor: 0.6
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          showValues: false
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: "off"
        links: []
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
            value: 0
          - color: red
            value: 80
        unit: short
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 12
      "y": 1102
    id: 22
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        hideZeros: false
        mode: multi
        sort: desc
    pluginVersion: 12.2.0
    type: timeseries
  - title: Disk space usage % by type  ($instance)
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "max(sum(vm_data_size_bytes{job=~\"$job\",instance=~\"$instance\",type=~\"indexdb.*\",%s=~\"$cluster\"}) by(job,instance,%s) / sum(vm_data_size_bytes{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}) by(job,instance,%s)) by(%s)" $clusterLabel $clusterLabel $clusterLabel $clusterLabel $clusterLabel }}
      editorMode: code
      format: time_series
      intervalFactor: 1
      legendFormat: indexdb
      range: true
      refId: A
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "max(sum(vm_data_size_bytes{job=~\"$job\",instance=~\"$instance\",type!~\"indexdb.*\",%s=~\"$cluster\"}) by(job,instance,%s) / sum(vm_data_size_bytes{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}) by(job,instance,%s)) by(%s)" $clusterLabel $clusterLabel $clusterLabel $clusterLabel $clusterLabel }}
      editorMode: code
      format: time_series
      hide: false
      intervalFactor: 1
      legendFormat: datapoints
      range: true
      refId: B
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: |-
      Shows the percentage of used disk space by type: datapoints or indexdb. Normally, indexdb takes much less space comparing to datapoints. But with high [churn rate](https://docs.victoriametrics.com/victoriametrics/faq/#what-is-high-churn-rate) the size of the indexdb could grow significantly.

      The sum of the % can be > 100% since panel shows max % per-job and per-instance. It means different instance can have different ratio between datapoints and indexdb size.
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ""
          axisPlacement: auto
          barAlignment: 0
          barWidthFactor: 0.6
          drawStyle: line
          fillOpacity: 10
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          showValues: false
          spanNulls: false
          stacking:
            group: A
            mode: percent
          thresholdsStyle:
            mode: line
        links:
        - targetBlank: true
          title: Drilldown
          url: /d/oS7Bi_0Wz?viewPanel=201&var-ds=$ds&${__url_time_range}
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
            value: 0
        unit: percentunit
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 0
      "y": 1110
    id: 202
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        hideZeros: false
        mode: multi
        sort: desc
    pluginVersion: 12.2.0
    type: timeseries
  - title: Pending datapoints ($instance)
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "sum(max_over_time(vm_pending_rows{job=~\"$job_storage\",instance=~\"$instance\",type=\"storage\",%s=~\"$cluster\"}[$__rate_interval])) by(%s)" $clusterLabel $clusterLabel }}
      editorMode: code
      format: time_series
      hide: false
      intervalFactor: 1
      legendFormat: pending datapoints
      range: true
      refId: A
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "sum(max_over_time(vm_pending_rows{job=~\"$job_storage\",instance=~\"$instance\",type=\"indexdb\",%s=~\"$cluster\"}[$__rate_interval])) by(%s)" $clusterLabel $clusterLabel }}
      editorMode: code
      format: time_series
      hide: false
      intervalFactor: 1
      legendFormat: pending index entries
      range: true
      refId: B
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: How many data points are in RAM queue waiting to be written into storage. The number of pending data points should be in the range from 0 to `5*<ingestion_rate>`, since VictoriaMetrics pushes pending data to persistent storage every 5 seconds. The index datapoints value in general is much lower.
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ""
          axisPlacement: auto
          barAlignment: 0
          barWidthFactor: 0.6
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          showValues: false
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: "off"
        links: []
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
            value: 0
          - color: red
            value: 80
        unit: short
      overrides:
      - matcher:
          id: byName
          options: pending index entries
        properties:
        - id: unit
          value: none
        - id: decimals
          value: 3
    gridPos:
      h: 8
      w: 12
      x: 12
      "y": 1110
    id: 14
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        hideZeros: false
        mode: multi
        sort: none
    pluginVersion: 12.2.0
    type: timeseries
  - title: Network usage ($instance)
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "(sum(rate(vm_tcplistener_read_bytes_total{job=~\"$job_storage\",instance=~\"$instance\",%s=~\"$cluster\"}[$__rate_interval])) by(name,%s) * 8) > 0" $clusterLabel $clusterLabel }}
      editorMode: code
      format: time_series
      intervalFactor: 1
      legendFormat: read from {{`{{`}}name{{`}}`}}
      range: true
      refId: A
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "(sum(rate(vm_tcplistener_written_bytes_total{job=~\"$job_storage\",instance=~\"$instance\",%s=~\"$cluster\"}[$__rate_interval])) by(name,%s) * 8) > 0" $clusterLabel $clusterLabel }}
      editorMode: code
      format: time_series
      hide: false
      intervalFactor: 1
      legendFormat: write to {{`{{`}}name{{`}}`}}
      range: true
      refId: B
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: |-
      Shows network usage by vmstorage services.
      * Writes show traffic sent to clients.
      * Reads show traffic received from clients.
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ""
          axisPlacement: auto
          barAlignment: 0
          barWidthFactor: 0.6
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          showValues: false
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: "off"
        links: []
        mappings: []
        thresholds:
          mode: absolute
          steps:
          - color: green
            value: 0
          - color: red
            value: 80
        unit: bps
      overrides:
      - matcher:
          id: byRegexp
          options: /read.*/
        properties:
        - id: custom.transform
          value: negative-Y
    gridPos:
      h: 8
      w: 12
      x: 0
      "y": 1118
    id: 206
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        hideZeros: false
        mode: multi
        sort: desc
    pluginVersion: 12.2.0
    type: timeseries
  - title: Readonly mode
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "vm_storage_is_read_only{job=~\"$job_storage\",instance=~\"$instance\",%s=~\"$cluster\"} > 0" $clusterLabel }}
      editorMode: code
      exemplar: true
      interval: ""
      legendFormat: '{{`{{`}} instance {{`}}`}}'
      range: true
      refId: A
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: Value > 0 means vmstorage is in readonly mode.
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ""
          axisPlacement: auto
          barAlignment: 0
          barWidthFactor: 0.6
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          showValues: false
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: "off"
        mappings: []
        thresholds:
          mode: absolute
          steps:
          - color: green
            value: 0
          - color: red
            value: 80
        unit: short
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 12
      "y": 1118
    id: 141
    links:
    - targetBlank: true
      title: Readonly mode
      url: https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#readonly-mode
    options:
      legend:
        calcs: []
        displayMode: list
        placement: bottom
        showLegend: false
      tooltip:
        hideZeros: false
        mode: multi
        sort: desc
    pluginVersion: 12.2.0
    type: timeseries
  - title: Number of snapshots
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "max(vm_snapshots{job=~\"$job_storage\",instance=~\"$instance\",%s=~\"$cluster\"}) by(%s)" $clusterLabel $clusterLabel }}
      editorMode: code
      exemplar: true
      interval: ""
      legendFormat: snapshots
      range: true
      refId: A
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: |-
      Shows the max number of existing [snapshots](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-work-with-snapshots) across vmstorages.

      Recently made snapshots do not occupy disk space. But with time, as snapshots become older, they start to occupy more and more disk space.

      It is recommended deleting old snapshots when they are no longer needed to free up disk space.
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ""
          axisPlacement: auto
          barAlignment: 0
          barWidthFactor: 0.6
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          showValues: false
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: "off"
        mappings: []
        thresholds:
          mode: absolute
          steps:
          - color: green
            value: 0
          - color: red
            value: 80
        unit: short
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 12
      "y": 1126
    id: 216
    options:
      legend:
        calcs: []
        displayMode: list
        placement: bottom
        showLegend: false
      tooltip:
        hideZeros: false
        mode: multi
        sort: desc
    pluginVersion: 12.2.0
    type: timeseries
  collapsed: true
  gridPos:
    h: 1
    w: 24
    x: 0
    "y": 40
  id: 24
  type: row
- title: vmselect ($instance)
  panels:
  - title: Requests rate ($instance)
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "sum(rate(vm_http_requests_total{job=~\"$job_select\",instance=~\"$instance\",%s=~\"$cluster\"}[$__rate_interval])) by(path,%s) > 0" $clusterLabel $clusterLabel }}
      editorMode: code
      format: time_series
      intervalFactor: 1
      legendFormat: __auto
      range: true
      refId: A
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: Request rate accepted by vmselect nodes
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ""
          axisPlacement: auto
          barAlignment: 0
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: "off"
        links: []
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
          - color: red
            value: 80
        unit: short
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 0
      "y": 8384
    id: 92
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        mode: multi
        sort: desc
    pluginVersion: 9.1.0
    type: timeseries
  - title: Concurrent selects ($instance)
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "max(max_over_time(vm_concurrent_select_current{job=~\"$job_select\",instance=~\"$instance\",%s=~\"$cluster\"}[$__rate_interval])) by(%s)" $clusterLabel $clusterLabel }}
      editorMode: code
      format: time_series
      interval: ""
      intervalFactor: 1
      legendFormat: current
      range: true
      refId: A
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "min(vm_concurrent_select_capacity{job=~\"$job_select\",instance=~\"$instance\",%s=~\"$cluster\"}) by(%s)" $clusterLabel $clusterLabel }}
      format: time_series
      intervalFactor: 1
      legendFormat: max
      refId: B
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: |-
      Shows the max number of concurrent selects across instances.
      * `max` limit can be configured via `search.maxConcurrentRequests` flag
      * `current` shows the current number of goroutines busy with processing requests

      When `current` hits `max` constantly, it means one or more vmselect nodes are overloaded with number of requests. If you observe that CPU for vmselects is saturated, consider adding more vmselect replicas or increase CPU resources. If CPU and Memory panels show a plenty of free resources - try increasing `-search.maxConcurrentRequests`. Please note, the higher is `-search.maxConcurrentRequests`, the higher could be [peak memory usage](https://docs.victoriametrics.com/victoriametrics/troubleshooting/#out-of-memory-errors).
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ""
          axisPlacement: auto
          barAlignment: 0
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: "off"
        decimals: 0
        links: []
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
          - color: red
            value: 80
        unit: short
      overrides:
      - matcher:
          id: byName
          options: max
        properties:
        - id: color
          value:
            fixedColor: '#C4162A'
            mode: fixed
        - id: custom.fillOpacity
          value: 0
    gridPos:
      h: 8
      w: 12
      x: 12
      "y": 8384
    id: 95
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        mode: multi
        sort: desc
    pluginVersion: 9.1.0
    type: timeseries
  - title: CPU usage % ($instance)
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "max(rate(process_cpu_seconds_total{job=~\"$job_select\",instance=~\"$instance\",%s=~\"$cluster\"}[$__rate_interval]) / process_cpu_cores_available{job=~\"$job_select\",instance=~\"$instance\",%s=~\"$cluster\"}) by(%s)" $clusterLabel $clusterLabel $clusterLabel }}
      editorMode: code
      exemplar: true
      format: time_series
      interval: ""
      intervalFactor: 1
      legendFormat: max
      range: true
      refId: A
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "min(rate(process_cpu_seconds_total{job=~\"$job_select\",instance=~\"$instance\",%s=~\"$cluster\"}[$__rate_interval]) / process_cpu_cores_available{job=~\"$job_select\",instance=~\"$instance\",%s=~\"$cluster\"}) by(%s)" $clusterLabel $clusterLabel $clusterLabel }}
      editorMode: code
      exemplar: true
      format: time_series
      hide: false
      interval: ""
      intervalFactor: 1
      legendFormat: min
      range: true
      refId: B
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "quantile(0.5, rate(process_cpu_seconds_total{job=~\"$job_select\",instance=~\"$instance\",%s=~\"$cluster\"}[$__rate_interval]) / process_cpu_cores_available{job=~\"$job_select\",instance=~\"$instance\",%s=~\"$cluster\"}) by(%s)" $clusterLabel $clusterLabel $clusterLabel }}
      editorMode: code
      exemplar: true
      format: time_series
      hide: false
      interval: ""
      intervalFactor: 1
      legendFormat: median
      range: true
      refId: C
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ""
          axisPlacement: auto
          barAlignment: 0
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: line
        links:
        - targetBlank: true
          title: Drilldown
          url: /d/oS7Bi_0Wz?viewPanel=192&var-job=$job_select&var-ds=$ds&${__url_time_range}
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
          - color: red
            value: 0.9
        unit: percentunit
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 0
      "y": 8392
    id: 163
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        mode: multi
        sort: none
    pluginVersion: 9.1.0
    type: timeseries
  - title: Memory (anon) usage % ($instance)
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "max(max_over_time(process_resident_memory_anon_bytes{job=~\"$job_select\",instance=~\"$instance\",%s=~\"$cluster\"}[$__rate_interval]) / vm_available_memory_bytes{job=~\"$job_select\",instance=~\"$instance\",%s=~\"$cluster\"}) by(%s)" $clusterLabel $clusterLabel $clusterLabel }}
      editorMode: code
      exemplar: true
      format: time_series
      interval: ""
      intervalFactor: 1
      legendFormat: max
      range: true
      refId: A
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "min(max_over_time(process_resident_memory_anon_bytes{job=~\"$job_select\",instance=~\"$instance\",%s=~\"$cluster\"}[$__rate_interval]) / vm_available_memory_bytes{job=~\"$job_select\",instance=~\"$instance\",%s=~\"$cluster\"}) by(%s)" $clusterLabel $clusterLabel $clusterLabel }}
      editorMode: code
      exemplar: true
      format: time_series
      hide: false
      interval: ""
      intervalFactor: 1
      legendFormat: min
      range: true
      refId: B
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "quantile(0.5, max_over_time(process_resident_memory_anon_bytes{job=~\"$job_select\",instance=~\"$instance\",%s=~\"$cluster\"}[$__rate_interval]) / vm_available_memory_bytes{job=~\"$job_select\",instance=~\"$instance\",%s=~\"$cluster\"}) by(%s)" $clusterLabel $clusterLabel $clusterLabel }}
      editorMode: code
      exemplar: true
      format: time_series
      hide: false
      interval: ""
      intervalFactor: 1
      legendFormat: median
      range: true
      refId: C
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: |-
      Share for memory allocated by the process itself. When memory usage reaches 100% it will be likely OOM-killed.
      Safe memory usage % considered to be below 80%

      Click on the line and choose Drilldown to show memory usage per instance
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ""
          axisPlacement: auto
          barAlignment: 0
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: line
        links:
        - targetBlank: true
          title: Drilldown
          url: /d/oS7Bi_0Wz?viewPanel=190&var-job=$job_select&var-ds=$ds&${__url_time_range}
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
          - color: red
            value: 0.9
        unit: percentunit
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 12
      "y": 8392
    id: 165
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        mode: multi
        sort: none
    pluginVersion: 9.1.0
    type: timeseries
  - title: Series read per query ($instance)
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "max(histogram_quantile(0.99, sum(rate(vm_series_read_per_query_bucket{job=~\"$job_select\",instance=~\"$instance\",%s=~\"$cluster\"}[$__rate_interval])) by(instance,vmrange,%s))) by(%s)" $clusterLabel $clusterLabel $clusterLabel }}
      editorMode: code
      format: time_series
      interval: ""
      intervalFactor: 1
      legendFormat: series
      range: true
      refId: A
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: Shows the max 99th percentile of time series read per query across all vmselects.
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ""
          axisPlacement: auto
          barAlignment: 0
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: "off"
        decimals: 2
        links: []
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
          - color: red
            value: 80
        unit: short
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 0
      "y": 8400
    id: 178
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        mode: multi
        sort: desc
    pluginVersion: 9.1.0
    type: timeseries
  - title: Datapoints read per series ($instance)
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "max(histogram_quantile(0.99, sum(rate(vm_rows_read_per_series_bucket{job=~\"$job_select\",instance=~\"$instance\",%s=~\"$cluster\"}[$__rate_interval])) by(instance,vmrange,%s))) by(%s)" $clusterLabel $clusterLabel $clusterLabel }}
      editorMode: code
      format: time_series
      interval: ""
      intervalFactor: 1
      legendFormat: datapoints
      range: true
      refId: A
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: Shows the max 99th percentile of number of [data samples](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#raw-samples) read per time series.
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ""
          axisPlacement: auto
          barAlignment: 0
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: "off"
        decimals: 2
        links: []
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
          - color: red
            value: 80
        unit: short
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 12
      "y": 8400
    id: 180
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        mode: multi
        sort: desc
    pluginVersion: 9.1.0
    type: timeseries
  - title: Datapoints read per query ($instance)
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "max(histogram_quantile(0.99, sum(rate(vm_rows_read_per_query_bucket{job=~\"$job_select\",instance=~\"$instance\",%s=~\"$cluster\"}[$__rate_interval])) by(instance,vmrange,%s))) by(%s)" $clusterLabel $clusterLabel $clusterLabel }}
      editorMode: code
      format: time_series
      interval: ""
      intervalFactor: 1
      legendFormat: datapoints
      range: true
      refId: A
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: Shows the max 99th percentile of number of [data samples](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#raw-samples) read per query.
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ""
          axisPlacement: auto
          barAlignment: 0
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: "off"
        decimals: 2
        links: []
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
          - color: red
            value: 80
        unit: short
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 0
      "y": 8408
    id: 179
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        mode: multi
        sort: desc
    pluginVersion: 9.1.0
    type: timeseries
  - title: Datapoints scanned per query ($instance)
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "max(histogram_quantile(0.99, sum(rate(vm_rows_scanned_per_query_bucket{job=~\"$job_select\",instance=~\"$instance\",%s=~\"$cluster\"}[$__rate_interval])) by(instance,vmrange,%s))) by(%s)" $clusterLabel $clusterLabel $clusterLabel }}
      editorMode: code
      format: time_series
      interval: ""
      intervalFactor: 1
      legendFormat: datapoints
      range: true
      refId: A
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: |-
      Shows the max 99th percentile of number of [data samples](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#raw-samples) scanner per query.

      This number can exceed number of DatapointsReadPerQuery if `step` query arg passed to [/api/v1/query_range](https://prometheus.io/docs/prometheus/latest/querying/api/#range-queries) is smaller than the lookbehind window set in square brackets of [rollup function](https://docs.victoriametrics.com/victoriametrics/metricsql/#rollup-functions). For example, if `increase(some_metric[1h])` is executed with the `step=5m`, then the same [data samples](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#raw-samples) on a hour time range are scanned `1h/5m=12` times. See [this article](https://valyala.medium.com/how-to-optimize-promql-and-metricsql-queries-85a1b75bf986) for details.
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ""
          axisPlacement: auto
          barAlignment: 0
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: "off"
        decimals: 2
        links: []
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
          - color: red
            value: 80
        unit: short
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 12
      "y": 8408
    id: 181
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        mode: multi
        sort: desc
    pluginVersion: 9.1.0
    type: timeseries
  - title: 'Network usage: clients ($instance)'
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "(sum(rate(vm_tcplistener_read_bytes_total{job=~\"$job_select\",instance=~\"$instance\",%s=~\"$cluster\"}[$__rate_interval])) by(%s) * 8) > 0" $clusterLabel $clusterLabel }}
      editorMode: code
      format: time_series
      intervalFactor: 1
      legendFormat: read from client
      range: true
      refId: A
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "(sum(rate(vm_tcplistener_written_bytes_total{job=~\"$job_select\",instance=~\"$instance\",%s=~\"$cluster\"}[$__rate_interval])) by(%s) * 8) > 0" $clusterLabel $clusterLabel }}
      editorMode: code
      format: time_series
      hide: false
      intervalFactor: 1
      legendFormat: write to client
      range: true
      refId: B
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: Shows network usage between vmselects and clients, such as vmalert, Grafana, vmui, etc.
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ""
          axisPlacement: auto
          barAlignment: 0
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: "off"
        links: []
        mappings: []
        thresholds:
          mode: absolute
          steps:
          - color: green
          - color: red
            value: 80
        unit: bps
      overrides:
      - matcher:
          id: byRegexp
          options: /read.*/
        properties:
        - id: custom.transform
          value: negative-Y
    gridPos:
      h: 8
      w: 12
      x: 0
      "y": 8416
    id: 93
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        mode: multi
        sort: desc
    pluginVersion: 9.1.0
    type: timeseries
  - title: 'Network usage: vmstorage ($instance)'
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "(sum(rate(vm_tcpdialer_read_bytes_total{job=~\"$job_select\",instance=~\"$instance\",%s=~\"$cluster\"}[$__rate_interval])) by(%s) * 8) > 0" $clusterLabel $clusterLabel }}
      editorMode: code
      format: time_series
      intervalFactor: 1
      legendFormat: read from vmstorage
      range: true
      refId: A
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "(sum(rate(vm_tcpdialer_written_bytes_total{job=~\"$job_select\",instance=~\"$instance\",%s=~\"$cluster\"}[$__rate_interval])) by(%s) * 8) > 0" $clusterLabel $clusterLabel }}
      editorMode: code
      format: time_series
      hide: false
      intervalFactor: 1
      legendFormat: write to vmstorage
      range: true
      refId: B
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: Shows network usage between vmselects and vmstorages.
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ""
          axisPlacement: auto
          barAlignment: 0
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: "off"
        links: []
        mappings: []
        thresholds:
          mode: absolute
          steps:
          - color: green
          - color: red
            value: 80
        unit: bps
      overrides:
      - matcher:
          id: byRegexp
          options: /read.*/
        properties:
        - id: custom.transform
          value: negative-Y
    gridPos:
      h: 8
      w: 12
      x: 12
      "y": 8416
    id: 207
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        mode: multi
        sort: desc
    pluginVersion: 9.1.0
    type: timeseries
  - title: Rollup result cache miss ratio ($instance)
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "sum(rate(vm_rollup_result_cache_miss_total{job=~\"$job_select\",instance=~\"$instance\",%s=~\"$cluster\"}[$__rate_interval])) by(%s) / sum(rate(vm_rollup_result_cache_requests_total{job=~\"$job_select\",instance=~\"$instance\",%s=~\"$cluster\"}[$__rate_interval])) by(%s)" $clusterLabel $clusterLabel $clusterLabel $clusterLabel }}
      editorMode: code
      format: time_series
      intervalFactor: 1
      legendFormat: miss
      range: true
      refId: A
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: "Shows the [rollup result cache](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#rollup-result-cache) miss ratio for query when cache is enabled. \nRollup cache is typically hit in two scenarios:\n1. Repeated [range queries](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#range-query) with increasing time, start and end arguments;\n2. Repeated [instant queries](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#instant-query) containing rollup functions with lookbehind window exceeding `-search.minWindowForInstantRollupOptimization`.\n\nA lower value indicates high cache utilization, suggesting that most queries are repeated from stable clients such as vmalert rules or Grafana dashboards."
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ""
          axisPlacement: auto
          barAlignment: 0
          barWidthFactor: 0.6
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          showValues: false
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: "off"
        links: []
        mappings: []
        max: 1
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
            value: 0
        unit: percentunit
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 0
      "y": 8424
    id: 226
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        hideZeros: false
        mode: multi
        sort: desc
    pluginVersion: 12.3.0
    type: timeseries
  collapsed: true
  gridPos:
    h: 1
    w: 24
    x: 0
    "y": 41
  id: 42
  type: row
- title: vminsert ($instance)
  panels:
  - title: Requests rate ($instance)
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "sum(rate(vm_http_requests_total{job=~\"$job_insert\",instance=~\"$instance\",%s=~\"$cluster\"}[$__rate_interval])) by(path,%s) > 0" $clusterLabel $clusterLabel }}
      editorMode: code
      format: time_series
      intervalFactor: 1
      legendFormat: __auto
      range: true
      refId: A
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: |-
      * `*` - unsupported query path
      * `/write` - insert into VM
      * `/metrics` - query VM system metrics
      * `/query` - query instant values
      * `/query_range` - query over a range of time
      * `/series` - match a certain label set
      * `/label/{}/values` - query a list of label values (variables mostly)
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ""
          axisPlacement: auto
          barAlignment: 0
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: "off"
        links: []
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
          - color: red
            value: 80
        unit: short
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 0
      "y": 8385
    id: 97
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        mode: multi
        sort: desc
    pluginVersion: 9.1.0
    type: timeseries
  - title: Concurrent inserts ($instance)
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "max(max_over_time(vm_concurrent_insert_current{job=~\"$job_insert\",instance=~\"$instance\",%s=~\"$cluster\"}[$__rate_interval])) by(%s)" $clusterLabel $clusterLabel }}
      editorMode: code
      format: time_series
      interval: ""
      intervalFactor: 1
      legendFormat: current
      range: true
      refId: A
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "min(vm_concurrent_insert_capacity{job=~\"$job_insert\",instance=~\"$instance\",%s=~\"$cluster\"}) by(%s)" $clusterLabel $clusterLabel }}
      exemplar: true
      format: time_series
      interval: ""
      intervalFactor: 1
      legendFormat: max
      refId: B
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: |-
      Shows the max number of ongoing insertions.
      * `max` - equal to number of CPU * 2 by default. May be configured with `maxConcurrentInserts` flag;
      * `current` - current number of goroutines busy with processing requests.

      `-maxConcurrentInserts` limits the number of insert requests which may be actively processed at any given point in time. All the other insert requests are queued for up to `-insert.maxQueueDuration` in the hope they will get a chance to be processed. This queue is used mostly for absorbing spikes for incoming insert request rate.

      When `current` hits `max` constantly, it means vminsert node is overloaded and requires more CPU or higher limits.
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ""
          axisPlacement: auto
          barAlignment: 0
          drawStyle: line
          fillOpacity: 10
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: "off"
        decimals: 0
        links: []
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
          - color: red
            value: 80
        unit: short
      overrides:
      - matcher:
          id: byName
          options: max
        properties:
        - id: color
          value:
            fixedColor: '#C4162A'
            mode: fixed
        - id: custom.fillOpacity
          value: 0
    gridPos:
      h: 8
      w: 12
      x: 12
      "y": 8385
    id: 99
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        mode: multi
        sort: desc
    pluginVersion: 9.1.0
    type: timeseries
  - title: CPU usage % ($instance)
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "max(rate(process_cpu_seconds_total{job=~\"$job_insert\",instance=~\"$instance\",%s=~\"$cluster\"}[$__rate_interval]) / process_cpu_cores_available{job=~\"$job_insert\",instance=~\"$instance\",%s=~\"$cluster\"}) by(%s)" $clusterLabel $clusterLabel $clusterLabel }}
      editorMode: code
      exemplar: true
      format: time_series
      interval: ""
      intervalFactor: 1
      legendFormat: max
      range: true
      refId: A
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "min(rate(process_cpu_seconds_total{job=~\"$job_insert\",instance=~\"$instance\",%s=~\"$cluster\"}[$__rate_interval]) / process_cpu_cores_available{job=~\"$job_insert\",instance=~\"$instance\",%s=~\"$cluster\"}) by(%s)" $clusterLabel $clusterLabel $clusterLabel }}
      editorMode: code
      exemplar: true
      format: time_series
      hide: false
      interval: ""
      intervalFactor: 1
      legendFormat: min
      range: true
      refId: B
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "quantile(0.5, rate(process_cpu_seconds_total{job=~\"$job_insert\",instance=~\"$instance\",%s=~\"$cluster\"}[$__rate_interval]) / process_cpu_cores_available{job=~\"$job_insert\",instance=~\"$instance\",%s=~\"$cluster\"}) by(%s)" $clusterLabel $clusterLabel $clusterLabel }}
      editorMode: code
      exemplar: true
      format: time_series
      hide: false
      interval: ""
      intervalFactor: 1
      legendFormat: median
      range: true
      refId: C
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ""
          axisPlacement: auto
          barAlignment: 0
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: line
        links:
        - targetBlank: true
          title: Drilldown
          url: /d/oS7Bi_0Wz?viewPanel=192&var-job=$job_insert&var-ds=$ds&${__url_time_range}
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
          - color: red
            value: 0.9
        unit: percentunit
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 0
      "y": 8393
    id: 185
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        mode: multi
        sort: none
    pluginVersion: 9.1.0
    type: timeseries
  - title: Memory (anon) usage % ($instance)
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "max(max_over_time(process_resident_memory_anon_bytes{job=~\"$job_insert\",instance=~\"$instance\",%s=~\"$cluster\"}[$__rate_interval]) / vm_available_memory_bytes{job=~\"$job_insert\",instance=~\"$instance\",%s=~\"$cluster\"}) by(%s)" $clusterLabel $clusterLabel $clusterLabel }}
      editorMode: code
      exemplar: true
      format: time_series
      interval: ""
      intervalFactor: 1
      legendFormat: max
      range: true
      refId: A
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "min(max_over_time(process_resident_memory_anon_bytes{job=~\"$job_insert\",instance=~\"$instance\",%s=~\"$cluster\"}[$__rate_interval]) / vm_available_memory_bytes{job=~\"$job_insert\",instance=~\"$instance\",%s=~\"$cluster\"}) by(%s)" $clusterLabel $clusterLabel $clusterLabel }}
      editorMode: code
      exemplar: true
      format: time_series
      hide: false
      interval: ""
      intervalFactor: 1
      legendFormat: min
      range: true
      refId: B
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "quantile(0.5, max_over_time(process_resident_memory_anon_bytes{job=~\"$job_insert\",instance=~\"$instance\",%s=~\"$cluster\"}[$__rate_interval]) / vm_available_memory_bytes{job=~\"$job_insert\",instance=~\"$instance\",%s=~\"$cluster\"}) by(%s)" $clusterLabel $clusterLabel $clusterLabel }}
      editorMode: code
      exemplar: true
      format: time_series
      hide: false
      interval: ""
      intervalFactor: 1
      legendFormat: median
      range: true
      refId: C
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: |-
      Share for memory allocated by the process itself. When memory usage reaches 100% it will be likely OOM-killed.
      Safe memory usage % considered to be below 80%

      Click on the line and choose Drilldown to show memory usage per instance
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ""
          axisPlacement: auto
          barAlignment: 0
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: line
        links:
        - targetBlank: true
          title: Drilldown
          url: /d/oS7Bi_0Wz?viewPanel=190&var-job=$job_insert&var-ds=$ds&${__url_time_range}
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
          - color: red
            value: 0.9
        unit: percentunit
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 12
      "y": 8393
    id: 187
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        mode: multi
        sort: none
    pluginVersion: 9.1.0
    type: timeseries
  - title: Storage connection saturation ($instance)
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "max(rate(vm_rpc_send_duration_seconds_total{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}[$__rate_interval])) by(addr,%s)" $clusterLabel $clusterLabel }}
      editorMode: code
      exemplar: true
      format: time_series
      interval: ""
      intervalFactor: 1
      legendFormat: __auto
      range: true
      refId: A
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: "Shows the saturation level of connection between vminsert and vmstorage components. \n\nIf the threshold of 0.9sec is reached, then the connection is saturated by more than 90% and vminsert won't be able to keep up. This usually means that either vminsert or vmstorage nodes are struggling with the load. Verify CPU/mem saturation of both components and network saturation between them.\nIf vminsert resources are saturated - consider adding more resources or scale vminserts horizontally.\n\nIf vminsert resources and network are fine, check vmstorage metrics for anomalies."
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ""
          axisPlacement: auto
          barAlignment: 0
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: line+area
        decimals: 0
        links: []
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: transparent
          - color: red
            value: 0.9
        unit: s
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 0
      "y": 8401
    id: 139
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        mode: multi
        sort: desc
    pluginVersion: 9.1.0
    type: timeseries
  - title: Storage reachability ($instance)
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "vm_rpc_vmstorage_is_reachable{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"} != 1" $clusterLabel }}
      editorMode: code
      format: time_series
      interval: ""
      intervalFactor: 1
      legendFormat: '{{`{{`}}instance{{`}}`}} => {{`{{`}}addr{{`}}`}}'
      range: true
      refId: A
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: Shows when vmstorage node is unreachable for vminsert.
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ""
          axisPlacement: auto
          barAlignment: 0
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: "off"
        decimals: 0
        links: []
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
          - color: red
            value: 80
        unit: short
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 12
      "y": 8401
    id: 114
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        displayMode: table
        placement: bottom
        showLegend: true
      tooltip:
        mode: multi
        sort: desc
    pluginVersion: 9.1.0
    type: timeseries
  - title: 'Network usage: clients ($instance)'
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "(sum(rate(vm_tcplistener_read_bytes_total{job=~\"$job_insert\",instance=~\"$instance\",%s=~\"$cluster\"}[$__rate_interval])) by(%s) * 8) > 0" $clusterLabel $clusterLabel }}
      editorMode: code
      format: time_series
      intervalFactor: 1
      legendFormat: read from client
      range: true
      refId: A
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "(sum(rate(vm_tcplistener_written_bytes_total{job=~\"$job_insert\",instance=~\"$instance\",%s=~\"$cluster\"}[$__rate_interval])) by(%s) * 8) > 0" $clusterLabel $clusterLabel }}
      editorMode: code
      format: time_series
      hide: false
      intervalFactor: 1
      legendFormat: write to client
      range: true
      refId: B
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: Shows network usage between vminserts and clients, such as vmagent, Prometheus, or any other client pushing metrics to vminsert.
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ""
          axisPlacement: auto
          barAlignment: 0
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: "off"
        links: []
        mappings: []
        thresholds:
          mode: absolute
          steps:
          - color: green
          - color: red
            value: 80
        unit: bps
      overrides:
      - matcher:
          id: byRegexp
          options: /read.*/
        properties:
        - id: custom.transform
          value: negative-Y
    gridPos:
      h: 8
      w: 12
      x: 0
      "y": 8409
    id: 208
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        mode: multi
        sort: desc
    pluginVersion: 9.1.0
    type: timeseries
  - title: 'Network usage: vmstorage ($instance)'
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "(sum(rate(vm_tcpdialer_read_bytes_total{job=~\"$job_insert\",instance=~\"$instance\",%s=~\"$cluster\"}[$__rate_interval])) by(%s) * 8) > 0" $clusterLabel $clusterLabel }}
      editorMode: code
      format: time_series
      intervalFactor: 1
      legendFormat: read from vmstorage
      range: true
      refId: A
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "(sum(rate(vm_tcpdialer_written_bytes_total{job=~\"$job_insert\",instance=~\"$instance\",%s=~\"$cluster\"}[$__rate_interval])) by(%s) * 8) > 0" $clusterLabel $clusterLabel }}
      editorMode: code
      format: time_series
      hide: false
      intervalFactor: 1
      legendFormat: write to vmstorage
      range: true
      refId: B
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: Shows network usage between vminserts and vmstorages.
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ""
          axisPlacement: auto
          barAlignment: 0
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: "off"
        links: []
        mappings: []
        thresholds:
          mode: absolute
          steps:
          - color: green
          - color: red
            value: 80
        unit: bps
      overrides:
      - matcher:
          id: byRegexp
          options: /read.*/
        properties:
        - id: custom.transform
          value: negative-Y
    gridPos:
      h: 8
      w: 12
      x: 12
      "y": 8409
    id: 209
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        mode: multi
        sort: desc
    pluginVersion: 9.1.0
    type: timeseries
  - title: Rows per insert ($instance)
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "max(histogram_quantile(0.99, sum(increase(vm_rows_per_insert_bucket{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}[$__rate_interval])) by(instance,vmrange,%s))) by(%s)" $clusterLabel $clusterLabel $clusterLabel }}
      editorMode: code
      format: time_series
      interval: ""
      intervalFactor: 1
      legendFormat: max
      range: true
      refId: A
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: ""
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ""
          axisPlacement: auto
          barAlignment: 0
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: "off"
        decimals: 2
        links: []
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
          - color: red
            value: 80
        unit: short
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 12
      "y": 8417
    id: 88
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        mode: multi
        sort: desc
    pluginVersion: 9.1.0
    type: timeseries
  collapsed: true
  gridPos:
    h: 1
    w: 24
    x: 0
    "y": 42
  id: 40
  type: row
- title: Drilldown
  panels:
  - fieldConfig:
      defaults: {}
      overrides: []
    gridPos:
      h: 2
      w: 24
      x: 0
      "y": 44
    id: 198
    options:
      code:
        language: plaintext
        showLineNumbers: false
        showMiniMap: false
      content: Drilldown row is used by other panels on the dashboard to show more detailed metrics per-instance, or a single instance in cases like memory breakdown.
      mode: markdown
    pluginVersion: 12.3.0
    transparent: true
    type: text
  - title: RSS memory usage ($instance)
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "max_over_time(process_resident_memory_bytes{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}[$__rate_interval])" $clusterLabel }}
      editorMode: code
      exemplar: true
      interval: ""
      legendFormat: '{{`{{`}}instance{{`}}`}} ({{`{{`}}job{{`}}`}})'
      range: true
      refId: A
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: |-
      Shows the used memory (resident).
      The application's performance will significantly degrade when memory usage is close to 100%.
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ""
          axisPlacement: auto
          barAlignment: 0
          barWidthFactor: 0.6
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          showValues: false
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: "off"
        links: []
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
            value: 0
        unit: decbytes
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 0
      "y": 46
    id: 189
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        hideZeros: false
        mode: multi
        sort: desc
    pluginVersion: 12.3.0
    type: timeseries
  - title: RSS anonymous memory usage ($instance)
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "max_over_time(process_resident_memory_anon_bytes{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}[$__rate_interval])" $clusterLabel }}
      editorMode: code
      exemplar: true
      interval: ""
      legendFormat: '{{`{{`}}instance{{`}}`}} ({{`{{`}}job{{`}}`}})'
      range: true
      refId: A
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: RSS share for memory allocated by the process itself. This share cannot be freed by the OS, so it must be taken into account by OOM killer.
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ""
          axisPlacement: auto
          barAlignment: 0
          barWidthFactor: 0.6
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          showValues: false
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: "off"
        links: []
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
            value: 0
        unit: decbytes
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 12
      "y": 46
    id: 190
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        hideZeros: false
        mode: multi
        sort: desc
    pluginVersion: 12.3.0
    type: timeseries
  - title: Memory usage breakdown
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "max(((go_memstats_gc_sys_bytes{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"} + go_memstats_mspan_sys_bytes{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}) + go_memstats_other_sys_bytes{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}) + go_memstats_mcache_sys_bytes{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}) by(%s)" $clusterLabel $clusterLabel $clusterLabel $clusterLabel $clusterLabel }}
      editorMode: code
      exemplar: true
      interval: ""
      legendFormat: 'Go: Other'
      range: true
      refId: A
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "max(go_memstats_stack_inuse_bytes{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}) by(%s)" $clusterLabel $clusterLabel }}
      editorMode: code
      hide: false
      instant: false
      legendFormat: 'Go: Stack'
      range: true
      refId: B
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "max(go_memstats_heap_inuse_bytes{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"} + (go_memstats_heap_idle_bytes{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"} - go_memstats_heap_released_bytes{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"})) by(%s)" $clusterLabel $clusterLabel $clusterLabel $clusterLabel }}
      editorMode: code
      hide: false
      instant: false
      legendFormat: 'Go: Heap'
      range: true
      refId: C
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "max(go_memstats_heap_released_bytes{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}) by(%s)" $clusterLabel $clusterLabel }}
      editorMode: code
      hide: false
      instant: false
      legendFormat: 'Go: Heap Reserved'
      range: true
      refId: D
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "max(sum(vm_cache_size_bytes{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}) by(job,instance,%s)) by(%s)" $clusterLabel $clusterLabel $clusterLabel }}
      editorMode: code
      hide: false
      instant: false
      legendFormat: 'Mmap: VM Cache'
      range: true
      refId: E
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "max(process_resident_memory_file_bytes{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}) by(%s)" $clusterLabel $clusterLabel }}
      editorMode: code
      hide: false
      instant: false
      legendFormat: File cache
      range: true
      refId: F
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: |-
      This panel breaks down memory usage by type. It’s intended for inspecting a single instance.

      How to use:
      - Start from the high-level RSS panel.
      - Identify an instance with unexpected or abnormal memory growth.
      - Filter to that instance to inspect the detailed breakdown.

      Interpretation
      - A steadily rising Go Heap usually indicates a memory leak. Collect pprof memory profile.
      - A growing Go Stack commonly points to a goroutine leak.
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ""
          axisPlacement: auto
          barAlignment: 0
          barWidthFactor: 0.6
          drawStyle: line
          fillOpacity: 10
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          showValues: false
          spanNulls: false
          stacking:
            group: A
            mode: normal
          thresholdsStyle:
            mode: "off"
        links: []
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
            value: 0
        unit: decbytes
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 0
      "y": 54
    id: 225
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        hideZeros: false
        mode: multi
        sort: desc
    pluginVersion: 12.3.0
    type: timeseries
  - title: Storage full ETA ($instance)
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "((vm_free_disk_space_bytes{job=~\"$job_storage\",instance=~\"$instance\",%s=~\"$cluster\"} - vm_free_disk_space_limit_bytes{job=~\"$job_storage\",instance=~\"$instance\",%s=~\"$cluster\"}) / ignoring(path) (((rate(vm_rows_added_to_storage_total{job=~\"$job_storage\",instance=~\"$instance\",%s=~\"$cluster\"}[1d]) - sum(rate(vm_deduplicated_samples_total{job=~\"$job_storage\",instance=~\"$instance\",%s=~\"$cluster\"}[1d])) without(type)) * (sum(vm_data_size_bytes{job=~\"$job_storage\",instance=~\"$instance\",type!~\"indexdb.*\",%s=~\"$cluster\"}) without(type) / sum(vm_rows{job=~\"$job_storage\",instance=~\"$instance\",type!~\"indexdb.*\",%s=~\"$cluster\"}) without(type))) + (rate(vm_new_timeseries_created_total{job=~\"$job_storage\",instance=~\"$instance\",%s=~\"$cluster\"}[1d]) * (sum(vm_data_size_bytes{job=~\"$job_storage\",instance=~\"$instance\",type=\"indexdb/file\",%s=~\"$cluster\"}) without(type) / sum(vm_rows{job=~\"$job_storage\",instance=~\"$instance\",type=\"indexdb/file\",%s=~\"$cluster\"}) without(type))))) > 0" $clusterLabel $clusterLabel $clusterLabel $clusterLabel $clusterLabel $clusterLabel $clusterLabel $clusterLabel $clusterLabel }}
      editorMode: code
      format: time_series
      interval: ""
      intervalFactor: 1
      legendFormat: '{{`{{`}}instance{{`}}`}}'
      range: true
      refId: A
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: |-
      Shows the approx time needed to reach 100% of allowed disk capacity based on the following params:
      * free disk space (after -storage.minFreeDiskSpaceBytes);
      * row ingestion rate;
      * compression.
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ""
          axisPlacement: auto
          barAlignment: 0
          barWidthFactor: 0.6
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          showValues: false
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: "off"
        links: []
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
            value: 0
          - color: red
            value: 80
        unit: s
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 12
      "y": 54
    id: 196
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - min
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        hideZeros: false
        mode: multi
        sort: none
    pluginVersion: 12.3.0
    type: timeseries
  - title: CPU usage ($instance)
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "sum(rate(process_cpu_seconds_total{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}[$__rate_interval])) by(job,instance,%s)" $clusterLabel $clusterLabel }}
      editorMode: code
      exemplar: false
      format: time_series
      interval: ""
      intervalFactor: 1
      legendFormat: '{{`{{`}}instance{{`}}`}} ({{`{{`}}job{{`}}`}})'
      range: true
      refId: A
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: ""
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ""
          axisPlacement: auto
          barAlignment: 0
          barWidthFactor: 0.6
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          showValues: false
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: "off"
        links: []
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
            value: 0
        unit: short
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 0
      "y": 62
    id: 192
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        hideZeros: false
        mode: multi
        sort: desc
    pluginVersion: 12.3.0
    type: timeseries
  - title: Disk space usage by type ($instance)
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "sum(vm_data_size_bytes{job=~\"$job\",instance=~\"$instance\",type=~\"indexdb.*\",%s=~\"$cluster\"}) by(job,instance,%s)" $clusterLabel $clusterLabel }}
      editorMode: code
      format: time_series
      intervalFactor: 1
      legendFormat: '{{`{{`}}job{{`}}`}}:{{`{{`}}instance{{`}}`}} (indexdb)'
      range: true
      refId: A
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "sum(vm_data_size_bytes{job=~\"$job\",instance=~\"$instance\",type!~\"indexdb.*\",%s=~\"$cluster\"}) by(job,instance,%s)" $clusterLabel $clusterLabel }}
      editorMode: code
      format: time_series
      hide: false
      intervalFactor: 1
      legendFormat: '{{`{{`}}job{{`}}`}}:{{`{{`}}instance{{`}}`}} (datapoints)'
      range: true
      refId: B
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: ""
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ""
          axisPlacement: auto
          barAlignment: 0
          barWidthFactor: 0.6
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          showValues: false
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: line
        links: []
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
            value: 0
        unit: decbytes
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 12
      "y": 62
    id: 201
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        hideZeros: false
        mode: multi
        sort: desc
    pluginVersion: 12.3.0
    type: timeseries
  - title: Disk space usage ($instance)
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "sum(vm_data_size_bytes{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}) by(job,instance,%s)" $clusterLabel $clusterLabel }}
      editorMode: code
      format: time_series
      intervalFactor: 1
      legendFormat: ""
      range: true
      refId: A
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: Shows the percentage of used disk space. It is recommended to have at least 20% of free disk space for the best performance.
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ""
          axisPlacement: auto
          barAlignment: 0
          barWidthFactor: 0.6
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          showValues: false
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: line
        links: []
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
            value: 0
        unit: decbytes
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 0
      "y": 70
    id: 200
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        hideZeros: false
        mode: multi
        sort: desc
    pluginVersion: 12.3.0
    type: timeseries
  - title: Major page faults rate  ($instance)
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: ${ds}
      expr: {{ printf "sum(rate(process_major_pagefaults_total{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}[$__rate_interval])) by(job,instance,%s) > 0" $clusterLabel $clusterLabel }}
      editorMode: code
      legendFormat: '{{`{{`}}instance{{`}}`}} ({{`{{`}}job{{`}}`}})'
      range: true
      refId: A
    datasource:
      type: {{ $defaultDatasource }}
      uid: ${ds}
    description: |-
      Major page faults occur when required data is not in physical memory and must be loaded from disk (storage or swap). It usually happens with:
      - insufficient physical memory
      - active use of swap space
      - file mapping access

      It may cause significant performance slowdown if sustained high.

      Possible workaround:
      - Extend RAM to scale up
      - Reduce swap usage by adjusting vm.swappiness
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ""
          axisPlacement: auto
          barAlignment: 0
          barWidthFactor: 0.6
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: auto
          showValues: false
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: "off"
        mappings: []
        thresholds:
          mode: absolute
          steps:
          - color: green
            value: 0
          - color: red
            value: 80
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 12
      "y": 70
    id: 224
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
      tooltip:
        hideZeros: false
        mode: multi
        sort: desc
    pluginVersion: 12.3.0
    type: timeseries
  - title: Logging rate
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "sum(rate(vm_log_messages_total{job=~\"$job\",instance=~\"$instance\",level!=\"info\",%s=~\"$cluster\"}[$__rate_interval])) by(job,instance,level,app_version,location,%s) > 0" $clusterLabel $clusterLabel }}
      editorMode: code
      exemplar: true
      format: time_series
      hide: false
      interval: 5m
      intervalFactor: 1
      legendFormat: '{{`{{`}}instance{{`}}`}} ({{`{{`}}job{{`}}`}}) - {{`{{`}}level{{`}}`}}: {{`{{`}}location{{`}}`}}'
      range: true
      refId: A
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: Shows the rate of logging the messages by their level. Unexpected spike in rate is a good reason to check logs.
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ""
          axisPlacement: auto
          barAlignment: 0
          barWidthFactor: 0.6
          drawStyle: bars
          fillOpacity: 100
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          showValues: false
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: "off"
        links:
        - targetBlank: true
          title: Source Code
          url: https://l2s.victoriametrics.com/?app_version=${__field.labels.app_version}&location=${__field.labels.location}
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
            value: 0
          - color: red
            value: 80
        unit: short
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 0
      "y": 78
    id: 203
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        hideZeros: false
        mode: multi
        sort: desc
    pluginVersion: 12.3.0
    type: timeseries
  collapsed: true
  gridPos:
    h: 1
    w: 24
    x: 0
    "y": 43
  id: 194
  type: row
condition: {{ ($Values.vmcluster).enabled }}
description: Overview for cluster VictoriaMetrics v1.117.0 or higher
fiscalYearStartMonth: 0
graphTooltip: 1
id: 0
links:
- icon: doc
  tags: []
  targetBlank: true
  title: Docs
  type: link
  url: https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/
- asDropdown: false
  icon: question
  includeVars: false
  keepTime: false
  tags: []
  targetBlank: true
  title: Troubleshooting
  tooltip: ""
  type: link
  url: https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#troubleshooting
- icon: info
  tags: []
  targetBlank: true
  title: Community support
  type: link
  url: https://victoriametrics.com/support/community-support/
- asDropdown: false
  icon: bolt
  includeVars: false
  keepTime: false
  tags: []
  targetBlank: true
  title: Enterprise support
  tooltip: ""
  type: link
  url: https://victoriametrics.com/support/enterprise-support/
- icon: info
  tags: []
  targetBlank: true
  title: Releases
  type: link
  url: https://github.com/VictoriaMetrics/VictoriaMetrics/releases
preload: false
refresh: ""
schemaVersion: 42
time:
  from: now-3h
  to: now
timepicker:
  refresh_intervals:
  - 10s
  - 30s
  - 1m
  - 5m
  - 15m
  - 30m
  - 1h
  - 2h
  - 1d
uid: oS7Bi_0Wz
version: 1
