
{{- $Values := (.helm).Values | default .Values }}
{{- $multicluster := ((($Values.grafana).sidecar).dashboards).multicluster | default false }}
{{- $defaultDatasource := "prometheus" -}}
{{- $clusterLabel := ($Values.global).clusterLabel | default "cluster" }}
{{- range (($Values.defaultDatasources).victoriametrics).datasources | default list }}
  {{- if and .isDefault .type }}{{ $defaultDatasource = .type }}{{- end }}
{{- end }}
title: VictoriaMetrics - single-node
templating:
  list:
  - name: ds
    type: datasource
    query: {{ $defaultDatasource }}
    current:
      text: prometheus
      value: P1809F7CD0C75ACF3
    options: []
    refresh: 1
    regex: ""
  - name: job
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    type: query
    query:
      query: {{ printf "label_values(vm_app_version{version=~\"victoria-metrics-.*\",%s=~\"$cluster\"}, job)" $clusterLabel }}
      refId: VictoriaMetrics-job-Variable-Query
    definition: {{ printf "label_values(vm_app_version{version=~\"victoria-metrics-.*\",%s=~\"$cluster\"}, job)" $clusterLabel }}
    current:
      text: vmsingle-homelab
      value: vmsingle-homelab
    options: []
    refresh: 1
    regex: ""
    regexApplyTo: value
  - name: version
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    type: query
    query:
      query: {{ printf "label_values(vm_app_version{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}, version)" $clusterLabel }}
      refId: VictoriaMetrics-version-Variable-Query
    definition: {{ printf "label_values(vm_app_version{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}, version)" $clusterLabel }}
    hide: 2
    current:
      text: v1.132.0
      value: v1.132.0
    options: []
    refresh: 1
    regex: /.*-(?:tags|heads)-(.*)-(?:0|dirty)-.*/
    regexApplyTo: value
    sort: 2
  - name: instance
    multi: true
    includeAll: true
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    type: query
    query:
      query: {{ printf "label_values(vm_app_version{job=~\"$job\",%s=~\"$cluster\"}, instance)" $clusterLabel }}
      refId: VictoriaMetrics-instance-Variable-Query
    definition: {{ printf "label_values(vm_app_version{job=~\"$job\",%s=~\"$cluster\"}, instance)" $clusterLabel }}
    allValue: .*
    current:
      text: All
      value: $__all
    options: []
    refresh: 1
    regex: ""
    regexApplyTo: value
  - name: adhoc
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    type: adhoc
    baseFilters: []
    filters: []
  - name: cluster
    label: cluster
    multi: {{ ternary true false $multicluster }}
    includeAll: {{ ternary true false $multicluster }}
    datasource:
      type: {{ $defaultDatasource }}
    type: {{ ternary "query" "constant" $multicluster }}
    query: {{ ternary (printf "label_values(vm_app_version, %s)" $clusterLabel) ".*" $multicluster }}
    hide: {{ ternary 0 2 $multicluster }}
    allValue: .*
editable: false
timezone: {{ default "browser" ($Values.defaultDashboards).defaultTimezone }}
tags:
- victoriametrics
- vm-k8s-stack
annotations:
  list:
  - name: Annotations & Alerts
    datasource:
      type: datasource
      uid: grafana
    builtIn: 1
    enable: true
    hide: true
    iconColor: rgba(0, 211, 255, 1)
    target:
      limit: 100
      matchAny: false
      tags: []
      type: dashboard
    type: dashboard
  - name: alerts
    expr: {{ printf "sum(ALERTS{job=~\"$job\",instance=~\"$instance\",alertgroup=\"vmsingle\",alertstate=\"firing\",show_at=\"dashboard\",%s=~\"$cluster\"}) by(alertname,%s)" $clusterLabel $clusterLabel }}
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    enable: false
    hide: false
    iconColor: red
    titleFormat: '{{`{{`}}alertname{{`}}`}}'
  - name: version
    expr: {{ printf "sum(vm_app_version{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}) by(version,%s) unless sum(vm_app_version{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"} offset $__interval) by(version,%s)" $clusterLabel $clusterLabel $clusterLabel $clusterLabel }}
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    enable: true
    hide: true
    iconColor: dark-blue
    textFormat: '{{`{{`}}version{{`}}`}}'
    titleFormat: Version change
  - name: restarts
    expr: {{ printf "sum(changes(vm_app_start_timestamp{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}[$__interval])) by(job,%s)" $clusterLabel $clusterLabel }}
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    enable: true
    hide: false
    iconColor: dark-yellow
    textFormat: '{{`{{`}}job{{`}}`}} restarted'
panels:
- title: Stats
  collapsed: false
  gridPos:
    h: 1
    w: 24
    x: 0
    "y": 0
  id: 6
  type: row
- title: Total datapoints
  targets:
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    expr: {{ printf "sum(vm_rows{job=~\"$job\",instance=~\"$instance\",type!~\"indexdb.*\",%s=~\"$cluster\"}) by(%s)" $clusterLabel $clusterLabel }}
    editorMode: code
    exemplar: false
    format: time_series
    instant: true
    interval: ""
    intervalFactor: 1
    legendFormat: ""
    refId: A
  datasource:
    type: {{ $defaultDatasource }}
    uid: $ds
  description: How many datapoints are in storage
  fieldConfig:
    defaults:
      color:
        mode: thresholds
      mappings: []
      thresholds:
        mode: absolute
        steps:
        - color: green
          value: 0
      unit: short
    overrides: []
  gridPos:
    h: 3
    w: 6
    x: 0
    "y": 1
  id: 26
  maxDataPoints: 100
  options:
    colorMode: value
    graphMode: area
    justifyMode: auto
    orientation: horizontal
    percentChangeColorMode: standard
    reduceOptions:
      calcs:
      - lastNotNull
      fields: ""
      values: false
    showPercentChange: false
    text: {}
    textMode: auto
    wideLayout: true
  pluginVersion: 12.4.0-20418128491
  type: stat
- title: Ingestion rate
  targets:
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    expr: {{ printf "sum(rate(vm_rows_inserted_total{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}[$__rate_interval])) by(%s)" $clusterLabel $clusterLabel }}
    editorMode: code
    exemplar: false
    format: time_series
    instant: true
    interval: ""
    intervalFactor: 1
    legendFormat: ""
    refId: A
  datasource:
    type: {{ $defaultDatasource }}
    uid: $ds
  description: Shows the datapoints ingestion rate.
  fieldConfig:
    defaults:
      color:
        mode: thresholds
      mappings: []
      thresholds:
        mode: absolute
        steps:
        - color: green
          value: 0
      unit: short
    overrides: []
  gridPos:
    h: 3
    w: 6
    x: 6
    "y": 1
  id: 107
  maxDataPoints: 100
  options:
    colorMode: value
    graphMode: area
    justifyMode: auto
    orientation: horizontal
    percentChangeColorMode: standard
    reduceOptions:
      calcs:
      - lastNotNull
      fields: ""
      values: false
    showPercentChange: false
    text: {}
    textMode: auto
    wideLayout: true
  pluginVersion: 12.4.0-20418128491
  type: stat
- title: Read requests
  targets:
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    expr: {{ printf "sum(rate(vm_http_requests_total{job=~\"$job\",instance=~\"$instance\",path!~\".*(/write|/metrics)\",%s=~\"$cluster\"}[$__rate_interval])) by(%s)" $clusterLabel $clusterLabel }}
    editorMode: code
    exemplar: false
    format: time_series
    instant: true
    interval: ""
    intervalFactor: 1
    legendFormat: ""
    refId: A
  datasource:
    type: {{ $defaultDatasource }}
    uid: $ds
  description: Shows the rate of HTTP read requests.
  fieldConfig:
    defaults:
      color:
        mode: thresholds
      mappings: []
      thresholds:
        mode: absolute
        steps:
        - color: green
          value: 0
      unit: req/s
    overrides: []
  gridPos:
    h: 3
    w: 6
    x: 12
    "y": 1
  id: 108
  maxDataPoints: 100
  options:
    colorMode: value
    graphMode: area
    justifyMode: auto
    orientation: horizontal
    percentChangeColorMode: standard
    reduceOptions:
      calcs:
      - lastNotNull
      fields: ""
      values: false
    showPercentChange: false
    text: {}
    textMode: auto
    wideLayout: true
  pluginVersion: 12.4.0-20418128491
  type: stat
- title: Available memory
  targets:
  - datasource:
      uid: $ds
    expr: {{ printf "sum(vm_available_memory_bytes{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}) by(%s)" $clusterLabel $clusterLabel }}
    exemplar: false
    format: time_series
    instant: true
    interval: ""
    intervalFactor: 1
    legendFormat: ""
    refId: A
  datasource:
    type: {{ $defaultDatasource }}
    uid: $ds
  description: Total size of available memory for VM process
  fieldConfig:
    defaults:
      color:
        mode: thresholds
      mappings: []
      thresholds:
        mode: absolute
        steps:
        - color: green
          value: 0
      unit: bytes
    overrides: []
  gridPos:
    h: 3
    w: 6
    x: 18
    "y": 1
  id: 78
  maxDataPoints: 100
  options:
    colorMode: value
    graphMode: area
    justifyMode: auto
    orientation: horizontal
    percentChangeColorMode: standard
    reduceOptions:
      calcs:
      - lastNotNull
      fields: ""
      values: false
    showPercentChange: false
    text: {}
    textMode: auto
    wideLayout: true
  pluginVersion: 12.4.0-20418128491
  type: stat
- title: Active series
  targets:
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    expr: {{ printf "sum(vm_cache_entries{job=~\"$job\",instance=~\"$instance\",type=\"storage/hour_metric_ids\",%s=~\"$cluster\"}) by(%s)" $clusterLabel $clusterLabel }}
    editorMode: code
    exemplar: false
    format: time_series
    instant: true
    interval: ""
    intervalFactor: 1
    legendFormat: ""
    refId: A
  datasource:
    type: {{ $defaultDatasource }}
    uid: $ds
  description: Shows the number of [active time series](https://docs.victoriametrics.com/victoriametrics/faq/#what-is-an-active-time-series) with new data points inserted during the last hour. High value may result in ingestion slowdown.
  fieldConfig:
    defaults:
      color:
        mode: thresholds
      mappings: []
      thresholds:
        mode: absolute
        steps:
        - color: green
          value: 0
      unit: short
    overrides: []
  gridPos:
    h: 3
    w: 6
    x: 0
    "y": 4
  id: 38
  maxDataPoints: 100
  options:
    colorMode: value
    graphMode: area
    justifyMode: auto
    orientation: horizontal
    percentChangeColorMode: standard
    reduceOptions:
      calcs:
      - lastNotNull
      fields: ""
      values: false
    showPercentChange: false
    text: {}
    textMode: auto
    wideLayout: true
  pluginVersion: 12.4.0-20418128491
  type: stat
- title: Disk space usage
  targets:
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    expr: {{ printf "sum(vm_data_size_bytes{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}) by(%s)" $clusterLabel $clusterLabel }}
    editorMode: code
    exemplar: false
    format: time_series
    instant: true
    interval: ""
    intervalFactor: 1
    legendFormat: ""
    refId: A
  datasource:
    type: {{ $defaultDatasource }}
    uid: $ds
  description: Total amount of used disk space
  fieldConfig:
    defaults:
      color:
        mode: thresholds
      mappings: []
      thresholds:
        mode: absolute
        steps:
        - color: green
          value: 0
      unit: bytes
    overrides: []
  gridPos:
    h: 3
    w: 6
    x: 6
    "y": 4
  id: 81
  maxDataPoints: 100
  options:
    colorMode: value
    graphMode: area
    justifyMode: auto
    orientation: horizontal
    percentChangeColorMode: standard
    reduceOptions:
      calcs:
      - lastNotNull
      fields: ""
      values: false
    showPercentChange: false
    text: {}
    textMode: auto
    wideLayout: true
  pluginVersion: 12.4.0-20418128491
  type: stat
- title: Bytes per point
  targets:
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    expr: {{ printf "max(sum(vm_data_size_bytes{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}) by(%s) / sum(vm_rows{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}) by(%s)) by(%s)" $clusterLabel $clusterLabel $clusterLabel $clusterLabel $clusterLabel }}
    editorMode: code
    exemplar: false
    format: time_series
    instant: true
    interval: ""
    intervalFactor: 1
    legendFormat: ""
    refId: A
  datasource:
    type: {{ $defaultDatasource }}
    uid: $ds
  description: Average disk usage per datapoint.
  fieldConfig:
    defaults:
      color:
        mode: thresholds
      mappings: []
      thresholds:
        mode: absolute
        steps:
        - color: green
          value: 0
      unit: bytes
    overrides: []
  gridPos:
    h: 3
    w: 6
    x: 12
    "y": 4
  id: 82
  maxDataPoints: 100
  options:
    colorMode: value
    graphMode: area
    justifyMode: auto
    orientation: horizontal
    percentChangeColorMode: standard
    reduceOptions:
      calcs:
      - lastNotNull
      fields: ""
      values: false
    showPercentChange: false
    text: {}
    textMode: auto
    wideLayout: true
  pluginVersion: 12.4.0-20418128491
  type: stat
- title: Available CPU
  targets:
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    expr: {{ printf "sum(vm_available_cpu_cores{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}) by(%s)" $clusterLabel $clusterLabel }}
    exemplar: false
    format: time_series
    instant: true
    interval: ""
    intervalFactor: 1
    legendFormat: ""
    refId: A
  datasource:
    type: {{ $defaultDatasource }}
    uid: $ds
  description: Total number of available CPUs for VM process
  fieldConfig:
    defaults:
      color:
        mode: thresholds
      mappings: []
      thresholds:
        mode: absolute
        steps:
        - color: green
          value: 0
      unit: short
    overrides: []
  gridPos:
    h: 3
    w: 6
    x: 18
    "y": 4
  id: 77
  maxDataPoints: 100
  options:
    colorMode: value
    graphMode: area
    justifyMode: auto
    orientation: horizontal
    percentChangeColorMode: standard
    reduceOptions:
      calcs:
      - lastNotNull
      fields: ""
      values: false
    showPercentChange: false
    text: {}
    textMode: auto
    wideLayout: true
  pluginVersion: 12.4.0-20418128491
  type: stat
- targets:
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    expr: {{ printf "sum(vm_app_version{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}) by(job,short_version,%s)" $clusterLabel $clusterLabel }}
    editorMode: code
    exemplar: false
    format: table
    instant: true
    range: false
    refId: A
  datasource:
    type: {{ $defaultDatasource }}
    uid: $ds
  fieldConfig:
    defaults:
      color:
        mode: thresholds
      custom:
        align: auto
        cellOptions:
          type: auto
        footer:
          reducers: []
        inspect: false
        minWidth: 50
      mappings: []
      thresholds:
        mode: absolute
        steps:
        - color: green
          value: 0
        - color: red
          value: 80
    overrides:
    - matcher:
        id: byName
        options: Time
      properties:
      - id: custom.hideFrom.viz
        value: true
    - matcher:
        id: byName
        options: Value
      properties:
      - id: displayName
        value: Count
  gridPos:
    h: 5
    w: 9
    x: 0
    "y": 7
  id: 155
  options:
    cellHeight: sm
    showHeader: true
    sortBy:
    - desc: true
      displayName: Count
  pluginVersion: 12.4.0-20418128491
  type: table
- title: Uptime ($job)
  targets:
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    expr: {{ printf "sum(min_over_time(vm_app_version{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}[$__rate_interval])) by(job,%s)" $clusterLabel $clusterLabel }}
    editorMode: code
    format: time_series
    instant: false
    legendFormat: '{{`{{`}}job{{`}}`}}'
    refId: A
  datasource:
    type: {{ $defaultDatasource }}
    uid: $ds
  fieldConfig:
    defaults:
      color:
        mode: palette-classic
      custom:
        axisBorderShow: false
        axisCenteredZero: false
        axisColorMode: text
        axisLabel: ""
        axisPlacement: auto
        axisSoftMin: 0
        barAlignment: 0
        barWidthFactor: 0.6
        drawStyle: line
        fillOpacity: 0
        gradientMode: none
        hideFrom:
          legend: false
          tooltip: false
          viz: false
        insertNulls: false
        lineInterpolation: stepAfter
        lineWidth: 1
        pointSize: 5
        scaleDistribution:
          type: linear
        showPoints: never
        showValues: false
        spanNulls: false
        stacking:
          group: A
          mode: none
        thresholdsStyle:
          mode: "off"
      decimals: 0
      links: []
      mappings: []
      thresholds:
        mode: absolute
        steps:
        - color: green
          value: 0
        - color: red
          value: 80
      unit: none
    overrides: []
  gridPos:
    h: 5
    w: 15
    x: 9
    "y": 7
  id: 156
  options:
    legend:
      calcs:
      - lastNotNull
      displayMode: table
      placement: right
      showLegend: true
      sortBy: Last *
      sortDesc: true
    tooltip:
      hideZeros: false
      mode: multi
      sort: desc
  pluginVersion: 12.4.0-20418128491
  type: timeseries
- title: Overview
  collapsed: false
  gridPos:
    h: 1
    w: 24
    x: 0
    "y": 12
  id: 24
  type: row
- title: Datapoints ingestion rate
  targets:
  - datasource:
      uid: $ds
    expr: {{ printf "sum(rate(vm_rows_inserted_total{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}[$__rate_interval])) by(type,instance,%s) > 0" $clusterLabel $clusterLabel }}
    editorMode: code
    format: time_series
    intervalFactor: 1
    legendFormat: '{{`{{`}}instance{{`}}`}} - {{`{{`}}type{{`}}`}}'
    range: true
    refId: A
  datasource:
    type: {{ $defaultDatasource }}
    uid: $ds
  description: How many datapoints are inserted into storage per second
  fieldConfig:
    defaults:
      color:
        mode: palette-classic
      custom:
        axisBorderShow: false
        axisCenteredZero: false
        axisColorMode: text
        axisLabel: ""
        axisPlacement: auto
        barAlignment: 0
        barWidthFactor: 0.6
        drawStyle: line
        fillOpacity: 10
        gradientMode: none
        hideFrom:
          legend: false
          tooltip: false
          viz: false
        insertNulls: false
        lineInterpolation: linear
        lineWidth: 1
        pointSize: 5
        scaleDistribution:
          type: linear
        showPoints: never
        showValues: false
        spanNulls: false
        stacking:
          group: A
          mode: none
        thresholdsStyle:
          mode: "off"
      links: []
      mappings: []
      min: 0
      thresholds:
        mode: absolute
        steps:
        - color: green
          value: 0
        - color: red
          value: 80
      unit: short
    overrides: []
  gridPos:
    h: 8
    w: 12
    x: 0
    "y": 13
  id: 106
  options:
    legend:
      calcs:
      - mean
      - lastNotNull
      displayMode: table
      placement: bottom
      showLegend: true
      sortBy: Last *
      sortDesc: true
    tooltip:
      hideZeros: false
      mode: multi
      sort: desc
  pluginVersion: 12.4.0-20418128491
  type: timeseries
- title: Requests rate ($instance)
  targets:
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    expr: {{ printf "sum(rate(vm_http_requests_total{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}[$__rate_interval])) by(path,%s) > 0" $clusterLabel $clusterLabel }}
    editorMode: code
    format: time_series
    interval: ""
    intervalFactor: 1
    legendFormat: '{{`{{`}}path{{`}}`}}'
    range: true
    refId: A
  datasource:
    type: {{ $defaultDatasource }}
    uid: $ds
  description: |-
    * `*` - unsupported query path
    * `/write` - insert into VM
    * `/metrics` - query VM system metrics
    * `/query` - query instant values
    * `/query_range` - query over a range of time
    * `/series` - match a certain label set
    * `/label/{}/values` - query a list of label values (variables mostly)
  fieldConfig:
    defaults:
      color:
        mode: palette-classic
      custom:
        axisBorderShow: false
        axisCenteredZero: false
        axisColorMode: text
        axisLabel: ""
        axisPlacement: auto
        barAlignment: 0
        barWidthFactor: 0.6
        drawStyle: line
        fillOpacity: 0
        gradientMode: none
        hideFrom:
          legend: false
          tooltip: false
          viz: false
        insertNulls: false
        lineInterpolation: linear
        lineWidth: 1
        pointSize: 5
        scaleDistribution:
          type: linear
        showPoints: never
        showValues: false
        spanNulls: false
        stacking:
          group: A
          mode: none
        thresholdsStyle:
          mode: "off"
      links: []
      mappings: []
      min: 0
      thresholds:
        mode: absolute
        steps:
        - color: green
          value: 0
        - color: red
          value: 80
      unit: short
    overrides: []
  gridPos:
    h: 8
    w: 12
    x: 12
    "y": 13
  id: 12
  options:
    legend:
      calcs:
      - mean
      - lastNotNull
      - max
      displayMode: table
      placement: bottom
      showLegend: true
      sortBy: Last *
      sortDesc: true
    tooltip:
      hideZeros: false
      mode: multi
      sort: desc
  pluginVersion: 12.4.0-20418128491
  type: timeseries
- title: Active time series
  targets:
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    expr: {{ printf "sum(vm_cache_entries{job=~\"$job\",instance=~\"$instance\",type=\"storage/hour_metric_ids\",%s=~\"$cluster\"}) by(%s)" $clusterLabel $clusterLabel }}
    editorMode: code
    format: time_series
    intervalFactor: 1
    legendFormat: '{{`{{`}}instance{{`}}`}}'
    range: true
    refId: A
  datasource:
    type: {{ $defaultDatasource }}
    uid: $ds
  description: "Shows the number of [active time series](https://docs.victoriametrics.com/victoriametrics/faq/#what-is-an-active-time-series) with new data points inserted during the last hour. High value may result in ingestion slowdown. \n\nSee following link for details:"
  fieldConfig:
    defaults:
      color:
        mode: palette-classic
      custom:
        axisBorderShow: false
        axisCenteredZero: false
        axisColorMode: text
        axisLabel: ""
        axisPlacement: auto
        barAlignment: 0
        barWidthFactor: 0.6
        drawStyle: line
        fillOpacity: 10
        gradientMode: none
        hideFrom:
          legend: false
          tooltip: false
          viz: false
        insertNulls: false
        lineInterpolation: linear
        lineWidth: 1
        pointSize: 5
        scaleDistribution:
          type: linear
        showPoints: never
        showValues: false
        spanNulls: false
        stacking:
          group: A
          mode: none
        thresholdsStyle:
          mode: "off"
      links: []
      mappings: []
      min: 0
      thresholds:
        mode: absolute
        steps:
        - color: green
          value: 0
        - color: red
          value: 80
      unit: short
    overrides: []
  gridPos:
    h: 8
    w: 12
    x: 0
    "y": 21
  id: 51
  links:
  - targetBlank: true
    title: troubleshooting
    url: https://docs.victoriametrics.com/victoriametrics/troubleshooting/
  options:
    legend:
      calcs:
      - mean
      - lastNotNull
      - max
      displayMode: table
      placement: bottom
      showLegend: true
    tooltip:
      hideZeros: false
      mode: multi
      sort: none
  pluginVersion: 12.4.0-20418128491
  type: timeseries
- title: Query duration 0.99 quantile ($instance)
  targets:
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    expr: {{ printf "max(vm_request_duration_seconds{job=~\"$job\",instance=~\"$instance\",quantile=\"0.99\",%s=~\"$cluster\"}) by(path,%s) > 0" $clusterLabel $clusterLabel }}
    editorMode: code
    format: time_series
    intervalFactor: 1
    legendFormat: '{{`{{`}}path{{`}}`}}'
    range: true
    refId: A
  datasource:
    type: {{ $defaultDatasource }}
    uid: $ds
  description: |-
    The less time it takes is better.
    * `*` - unsupported query path
    * `/write` - insert into VM
    * `/metrics` - query VM system metrics
    * `/query` - query instant values
    * `/query_range` - query over a range of time
    * `/series` - match a certain label set
    * `/label/{}/values` - query a list of label values (variables mostly)
  fieldConfig:
    defaults:
      color:
        mode: palette-classic
      custom:
        axisBorderShow: false
        axisCenteredZero: false
        axisColorMode: text
        axisLabel: ""
        axisPlacement: auto
        barAlignment: 0
        barWidthFactor: 0.6
        drawStyle: line
        fillOpacity: 10
        gradientMode: none
        hideFrom:
          legend: false
          tooltip: false
          viz: false
        insertNulls: false
        lineInterpolation: linear
        lineWidth: 1
        pointSize: 5
        scaleDistribution:
          type: linear
        showPoints: never
        showValues: false
        spanNulls: false
        stacking:
          group: A
          mode: none
        thresholdsStyle:
          mode: "off"
      links: []
      mappings: []
      min: 0
      thresholds:
        mode: absolute
        steps:
        - color: green
          value: 0
        - color: red
          value: 80
      unit: s
    overrides: []
  gridPos:
    h: 8
    w: 12
    x: 12
    "y": 21
  id: 22
  options:
    legend:
      calcs:
      - mean
      - lastNotNull
      - max
      displayMode: table
      placement: bottom
      showLegend: true
      sortBy: Last *
      sortDesc: true
    tooltip:
      hideZeros: false
      mode: multi
      sort: desc
  pluginVersion: 12.4.0-20418128491
  type: timeseries
- title: Requests error rate ($instance)
  targets:
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    expr: {{ printf "sum(rate(vm_http_request_errors_total{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}[$__rate_interval])) by(job,path,%s) > 0" $clusterLabel $clusterLabel }}
    editorMode: code
    exemplar: false
    format: time_series
    interval: ""
    intervalFactor: 1
    legendFormat: '{{`{{`}}path{{`}}`}}'
    range: true
    refId: A
  datasource:
    type: {{ $defaultDatasource }}
    uid: $ds
  description: |-
    * `*` - unsupported query path
    * `/write` - insert into VM
    * `/metrics` - query VM system metrics
    * `/query` - query instant values
    * `/query_range` - query over a range of time
    * `/series` - match a certain label set
    * `/label/{}/values` - query a list of label values (variables mostly)
  fieldConfig:
    defaults:
      color:
        mode: palette-classic
      custom:
        axisBorderShow: false
        axisCenteredZero: false
        axisColorMode: text
        axisLabel: ""
        axisPlacement: auto
        barAlignment: 0
        barWidthFactor: 0.6
        drawStyle: line
        fillOpacity: 10
        gradientMode: none
        hideFrom:
          legend: false
          tooltip: false
          viz: false
        insertNulls: false
        lineInterpolation: linear
        lineWidth: 1
        pointSize: 5
        scaleDistribution:
          type: linear
        showPoints: never
        showValues: false
        spanNulls: false
        stacking:
          group: A
          mode: none
        thresholdsStyle:
          mode: "off"
      links: []
      mappings: []
      min: 0
      thresholds:
        mode: absolute
        steps:
        - color: green
          value: 0
        - color: red
          value: 80
      unit: short
    overrides: []
  gridPos:
    h: 8
    w: 12
    x: 0
    "y": 29
  id: 35
  options:
    legend:
      calcs:
      - mean
      - lastNotNull
      displayMode: table
      placement: bottom
      showLegend: true
    tooltip:
      hideZeros: false
      mode: multi
      sort: desc
  pluginVersion: 12.3.0
  type: timeseries
- title: Logging rate
  targets:
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    expr: {{ printf "sum(rate(vm_log_messages_total{job=~\"$job\",instance=~\"$instance\",level!=\"info\",%s=~\"$cluster\"}[$__rate_interval])) by(job,level,%s) > 0" $clusterLabel $clusterLabel }}
    editorMode: code
    interval: 5m
    legendFormat: '{{`{{`}}job{{`}}`}} - {{`{{`}}level{{`}}`}}'
    range: true
    refId: A
  datasource:
    type: {{ $defaultDatasource }}
    uid: $ds
  description: Shows the rate of logging the messages by their level. Unexpected spike in rate is a good reason to check logs.
  fieldConfig:
    defaults:
      color:
        mode: palette-classic
      custom:
        axisBorderShow: false
        axisCenteredZero: false
        axisColorMode: text
        axisLabel: ""
        axisPlacement: auto
        barAlignment: 0
        barWidthFactor: 0.6
        drawStyle: bars
        fillOpacity: 100
        gradientMode: none
        hideFrom:
          legend: false
          tooltip: false
          viz: false
        insertNulls: false
        lineInterpolation: linear
        lineWidth: 1
        pointSize: 5
        scaleDistribution:
          type: linear
        showPoints: auto
        showValues: false
        spanNulls: false
        stacking:
          group: A
          mode: none
        thresholdsStyle:
          mode: "off"
      links:
      - targetBlank: true
        title: Drilldown
        url: /d/wNf0q_kZk?viewPanel=154&var-job=${__field.labels.job}&var-ds=$ds&${__url_time_range}
      mappings: []
      thresholds:
        mode: absolute
        steps:
        - color: green
          value: 0
        - color: red
          value: 80
    overrides: []
  gridPos:
    h: 8
    w: 12
    x: 12
    "y": 29
  id: 110
  options:
    legend:
      calcs:
      - lastNotNull
      - mean
      - max
      displayMode: table
      placement: bottom
      showLegend: true
      sortBy: Last *
      sortDesc: true
    tooltip:
      hideZeros: false
      mode: single
      sort: none
  pluginVersion: 12.3.0
  type: timeseries
- title: Resource usage
  panels:
  - title: RSS memory % usage ($instance)
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "max(max_over_time(process_resident_memory_bytes{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}[$__rate_interval]) / vm_available_memory_bytes{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}) by(job,%s)" $clusterLabel $clusterLabel $clusterLabel }}
      editorMode: code
      exemplar: false
      interval: ""
      legendFormat: __auto
      range: true
      refId: A
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: |-
      How much resident memory (RAM) the process is actually using, compared to its allowed container or system limit.

      - Good: Below 70% most of the time, maybe spiking a bit under load.
      - Bad: Above 90% for more than 5 minutes = risk of out-of-memory (OOM) kill.
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ""
          axisPlacement: auto
          barAlignment: 0
          barWidthFactor: 0.6
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          showValues: false
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: "off"
        links:
        - targetBlank: true
          title: Drilldown - RSS memory usage
          url: /d/wNf0q_kZk?viewPanel=148&var-job=${__field.labels.job}&var-ds=$ds&${__url_time_range}
        - targetBlank: true
          title: Drilldown - Memory usage breakdown
          url: /d/wNf0q_kZk?viewPanel=141&var-job=${__field.labels.job}&var-ds=$ds&${__url_time_range}
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
            value: 0
          - color: red
            value: 80
        unit: percentunit
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 0
      "y": 31
    id: 112
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        hideZeros: false
        mode: multi
        sort: desc
    pluginVersion: 12.2.0
    type: timeseries
  - title: CPU % usage ($instance)
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "max(rate(process_cpu_seconds_total{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}[$__rate_interval]) / process_cpu_cores_available{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}) by(job,%s)" $clusterLabel $clusterLabel $clusterLabel }}
      editorMode: code
      exemplar: false
      format: time_series
      interval: ""
      intervalFactor: 1
      legendFormat: __auto
      range: true
      refId: A
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ""
          axisPlacement: auto
          barAlignment: 0
          barWidthFactor: 0.6
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          showValues: false
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: "off"
        links:
        - targetBlank: true
          title: Drilldown
          url: /d/wNf0q_kZk?viewPanel=151&var-job=${__field.labels.job}&var-ds=$ds&${__url_time_range}
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
            value: 0
          - color: red
            value: 80
        unit: percentunit
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 12
      "y": 31
    id: 114
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        hideZeros: false
        mode: multi
        sort: none
    pluginVersion: 12.2.0
    type: timeseries
  - title: RSS anonymous memory % usage ($instance)
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "max(max_over_time(process_resident_memory_anon_bytes{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}[$__rate_interval]) / vm_available_memory_bytes{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}) by(job,%s)" $clusterLabel $clusterLabel $clusterLabel }}
      editorMode: code
      exemplar: false
      interval: ""
      legendFormat: __auto
      range: true
      refId: A
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: |-
      Share for memory allocated by the process itself. When memory usage reaches 100% it will be likely OOM-killed.
      Safe memory usage % considered to be below 80%
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ""
          axisPlacement: auto
          barAlignment: 0
          barWidthFactor: 0.6
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          showValues: false
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: "off"
        links:
        - targetBlank: true
          title: Drilldown
          url: /d/wNf0q_kZk?viewPanel=149&var-job=${__field.labels.job}&var-ds=$ds&${__url_time_range}
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
            value: 0
          - color: red
            value: 80
        unit: percentunit
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 0
      "y": 39
    id: 123
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        hideZeros: false
        mode: multi
        sort: desc
    pluginVersion: 12.2.0
    type: timeseries
  - title: CPU pressure
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "sum(rate(process_pressure_cpu_waiting_seconds_total{job=~\"$job\",%s=~\"$cluster\"}[$__rate_interval])) by(job,instance,%s)" $clusterLabel $clusterLabel }}
      editorMode: code
      format: time_series
      interval: ""
      intervalFactor: 2
      legendFormat: '{{`{{`}}job{{`}}`}}-{{`{{`}}instance{{`}}`}} - waiting'
      range: true
      refId: A
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "sum(rate(process_pressure_cpu_stalled_seconds_total{job=~\"$job\",%s=~\"$cluster\"}[$__rate_interval])) by(job,instance,%s)" $clusterLabel $clusterLabel }}
      editorMode: code
      format: time_series
      interval: ""
      intervalFactor: 2
      legendFormat: '{{`{{`}}job{{`}}`}}-{{`{{`}}instance{{`}}`}} - waiting'
      range: true
      refId: B
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: |-
      CPU pressure based on [Pressure Stall Information](https://docs.kernel.org/accounting/psi.html). Helps troubleshoot high CPU usage or throttling.
      Lower is better.

      - waiting: The percentage of time at least one task in the process was ready to run (runnable) but couldn't get scheduled on the CPU.
      - stalled: The percentage of time all tasks in the process (except idle ones) were unable to get CPU time — a full CPU stall.

      If there's a CPU burst, it's normal to see waiting or stalled > 1%. It only becomes a concern if it consistently climbs above 5–10% and aligns with latency spikes or GC slowdowns.
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ""
          axisPlacement: auto
          barAlignment: 0
          barWidthFactor: 0.6
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          showValues: false
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: line
        decimals: 0
        links: []
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
            value: 0
        unit: s
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 12
      "y": 39
    id: 134
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        hideZeros: false
        mode: multi
        sort: desc
    pluginVersion: 12.2.0
    type: timeseries
  - title: Memory usage
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "sum(go_memstats_sys_bytes{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}) by(job,%s) + sum(vm_cache_size_bytes{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}) by(job,%s)" $clusterLabel $clusterLabel $clusterLabel $clusterLabel }}
      editorMode: code
      format: time_series
      intervalFactor: 1
      legendFormat: '{{`{{`}}job{{`}}`}} - requested from system'
      range: true
      refId: A
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "sum(go_memstats_heap_inuse_bytes{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}) by(job,%s) + sum(vm_cache_size_bytes{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}) by(job,%s)" $clusterLabel $clusterLabel $clusterLabel $clusterLabel }}
      editorMode: code
      format: time_series
      intervalFactor: 1
      legendFormat: '{{`{{`}}job{{`}}`}} - heap inuse'
      range: true
      refId: B
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "sum(go_memstats_stack_inuse_bytes{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}) by(job,%s)" $clusterLabel $clusterLabel }}
      editorMode: code
      format: time_series
      intervalFactor: 1
      legendFormat: '{{`{{`}}job{{`}}`}} - stack inuse'
      range: true
      refId: C
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "sum(process_resident_memory_bytes{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}) by(job,%s)" $clusterLabel $clusterLabel }}
      editorMode: code
      format: time_series
      interval: ""
      intervalFactor: 1
      legendFormat: '{{`{{`}}job{{`}}`}} - resident'
      range: true
      refId: D
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "sum(process_resident_memory_anon_bytes{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}) by(job,%s)" $clusterLabel $clusterLabel }}
      editorMode: code
      exemplar: false
      format: time_series
      interval: ""
      intervalFactor: 1
      legendFormat: '{{`{{`}}job{{`}}`}} - resident anonymous'
      range: true
      refId: E
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ""
          axisPlacement: auto
          barAlignment: 0
          barWidthFactor: 0.6
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          showValues: false
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: "off"
        links:
        - targetBlank: true
          title: "111"
          url: "111"
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
            value: 0
          - color: red
            value: 80
        unit: bytes
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 0
      "y": 47
    id: 44
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        hideZeros: false
        mode: multi
        sort: desc
    pluginVersion: 12.2.0
    type: timeseries
  - title: CPU
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "rate(process_cpu_seconds_total{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}[$__rate_interval])" $clusterLabel }}
      editorMode: code
      format: time_series
      interval: ""
      intervalFactor: 1
      legendFormat: '{{`{{`}}instance{{`}}`}}'
      range: true
      refId: A
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "min(process_cpu_cores_available{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}) by(%s)" $clusterLabel $clusterLabel }}
      editorMode: code
      exemplar: false
      format: time_series
      interval: ""
      intervalFactor: 1
      legendFormat: Limit
      range: true
      refId: B
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: CPU cores used by instance
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ""
          axisPlacement: auto
          barAlignment: 0
          barWidthFactor: 0.6
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          showValues: false
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: "off"
        links: []
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
            value: 0
          - color: red
            value: 80
        unit: short
      overrides:
      - matcher:
          id: byName
          options: Limit
        properties:
        - id: color
          value:
            fixedColor: '#F2495C'
            mode: fixed
    gridPos:
      h: 8
      w: 12
      x: 12
      "y": 47
    id: 57
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        hideZeros: false
        mode: multi
        sort: desc
    pluginVersion: 12.2.0
    type: timeseries
  - title: Memory pressure
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "sum(rate(process_pressure_memory_waiting_seconds_total{job=~\"$job\",%s=~\"$cluster\"}[$__rate_interval])) by(job,%s)" $clusterLabel $clusterLabel }}
      editorMode: code
      format: time_series
      interval: ""
      intervalFactor: 2
      legendFormat: '{{`{{`}}job{{`}}`}} - waiting'
      range: true
      refId: A
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "sum(rate(process_pressure_memory_stalled_seconds_total{job=~\"$job\",%s=~\"$cluster\"}[$__rate_interval])) by(job,%s)" $clusterLabel $clusterLabel }}
      editorMode: code
      format: time_series
      interval: ""
      intervalFactor: 2
      legendFormat: '{{`{{`}}job{{`}}`}} - stalled'
      range: true
      refId: B
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: |-
      Shows memory pressure based on [Pressure Stall Information](https://docs.kernel.org/accounting/psi.html).
      Lower is better.

      - waiting: Time fraction where at least one thread was blocked on memory.
      - stalled: Time fraction where every thread was blocked on memory (severe pressure).

      If queries slow down and both series spike, the host is likely limited by RAM or I/O throughput.

      See also major page faults rate panel in Troubleshooting section if this metric continued to be high.
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ""
          axisPlacement: auto
          barAlignment: 0
          barWidthFactor: 0.6
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          showValues: false
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: line
        decimals: 0
        links: []
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
            value: 0
        unit: s
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 0
      "y": 55
    id: 135
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        hideZeros: false
        mode: multi
        sort: desc
    pluginVersion: 12.2.0
    type: timeseries
  - title: Disk writes/reads ($instance)
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "sum(rate(process_io_storage_read_bytes_total{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}[$__rate_interval])) by(job,%s)" $clusterLabel $clusterLabel }}
      editorMode: code
      format: time_series
      interval: ""
      intervalFactor: 1
      legendFormat: '{{`{{`}}job{{`}}`}} - read'
      range: true
      refId: A
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "sum(rate(process_io_storage_written_bytes_total{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}[$__rate_interval])) by(job,%s)" $clusterLabel $clusterLabel }}
      editorMode: code
      format: time_series
      interval: ""
      intervalFactor: 1
      legendFormat: '{{`{{`}}job{{`}}`}} - write'
      range: true
      refId: B
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: Shows the number of bytes read/write from the storage layer.
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ""
          axisPlacement: auto
          barAlignment: 0
          barWidthFactor: 0.6
          drawStyle: line
          fillOpacity: 10
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          showValues: false
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: "off"
        links: []
        mappings: []
        thresholds:
          mode: absolute
          steps:
          - color: green
            value: 0
          - color: red
            value: 80
        unit: bytes
      overrides:
      - matcher:
          id: byRegexp
          options: /.*read/
        properties:
        - id: custom.transform
          value: negative-Y
    gridPos:
      h: 8
      w: 12
      x: 12
      "y": 55
    id: 76
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        hideZeros: false
        mode: multi
        sort: none
    pluginVersion: 12.2.0
    type: timeseries
  - title: Open FDs usage % ($instance)
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "max(max_over_time(process_open_fds{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}[$__rate_interval]) / process_max_fds{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}) by(job,%s)" $clusterLabel $clusterLabel $clusterLabel }}
      editorMode: code
      exemplar: true
      format: time_series
      interval: ""
      intervalFactor: 2
      legendFormat: '{{`{{`}}job{{`}}`}}'
      range: true
      refId: A
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: |-
      Shows the percentage of open file descriptors compared to the limit set in the OS.
      Reaching the limit of open files can cause various issues and must be prevented.

      See how to change limits here https://medium.com/@muhammadtriwibowo/set-permanently-ulimit-n-open-files-in-ubuntu-4d61064429a
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ""
          axisPlacement: auto
          barAlignment: 0
          barWidthFactor: 0.6
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          showValues: false
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: "off"
        decimals: 2
        links: []
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
            value: 0
          - color: red
            value: 80
        unit: percentunit
      overrides:
      - matcher:
          id: byRegexp
          options: /max.*/
        properties:
        - id: color
          value:
            fixedColor: '#C4162A'
            mode: fixed
    gridPos:
      h: 8
      w: 12
      x: 0
      "y": 63
    id: 142
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        hideZeros: false
        mode: multi
        sort: desc
    pluginVersion: 12.2.0
    type: timeseries
  - title: Disk write/read calls ($instance)
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "sum(rate(process_io_read_syscalls_total{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}[$__rate_interval])) by(job,%s)" $clusterLabel $clusterLabel }}
      editorMode: code
      format: time_series
      interval: ""
      intervalFactor: 1
      legendFormat: '{{`{{`}}job{{`}}`}} - read calls'
      range: true
      refId: A
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "sum(rate(process_io_write_syscalls_total{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}[$__rate_interval])) by(job,%s)" $clusterLabel $clusterLabel }}
      editorMode: code
      format: time_series
      interval: ""
      intervalFactor: 1
      legendFormat: '{{`{{`}}job{{`}}`}} - write calls'
      range: true
      refId: B
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: Shows the number of read/write syscalls such as read, pread, write, pwrite.
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ""
          axisPlacement: auto
          barAlignment: 0
          barWidthFactor: 0.6
          drawStyle: line
          fillOpacity: 10
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          showValues: false
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: "off"
        links: []
        mappings: []
        thresholds:
          mode: absolute
          steps:
          - color: green
            value: 0
          - color: red
            value: 80
        unit: short
      overrides:
      - matcher:
          id: byRegexp
          options: /.*read.*/
        properties:
        - id: custom.transform
          value: negative-Y
    gridPos:
      h: 8
      w: 12
      x: 12
      "y": 63
    id: 124
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        hideZeros: false
        mode: multi
        sort: none
    pluginVersion: 12.2.0
    type: timeseries
  - title: Goroutines ($instance)
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "sum(go_goroutines{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}) by(job,%s)" $clusterLabel $clusterLabel }}
      editorMode: code
      format: time_series
      intervalFactor: 2
      legendFormat: '{{`{{`}}job{{`}}`}}'
      range: true
      refId: A
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ""
          axisPlacement: auto
          barAlignment: 0
          barWidthFactor: 0.6
          drawStyle: line
          fillOpacity: 10
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          showValues: false
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: "off"
        decimals: 0
        links: []
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
            value: 0
          - color: red
            value: 80
        unit: short
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 0
      "y": 71
    id: 47
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        hideZeros: false
        mode: multi
        sort: none
    pluginVersion: 12.2.0
    type: timeseries
  - title: IO pressure
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "sum(rate(process_pressure_io_waiting_seconds_total{job=~\"$job\",%s=~\"$cluster\"}[$__rate_interval])) by(job,instance,%s)" $clusterLabel $clusterLabel }}
      editorMode: code
      format: time_series
      interval: ""
      intervalFactor: 2
      legendFormat: '{{`{{`}}job{{`}}`}} - waiting'
      range: true
      refId: A
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "sum(rate(process_pressure_io_stalled_seconds_total{job=~\"$job\",%s=~\"$cluster\"}[$__rate_interval])) by(job,instance,%s)" $clusterLabel $clusterLabel }}
      editorMode: code
      format: time_series
      interval: ""
      intervalFactor: 2
      legendFormat: '{{`{{`}}job{{`}}`}} - stalled'
      range: true
      refId: B
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: |-
      Shows IO pressure based on [Pressure Stall Information](https://docs.kernel.org/accounting/psi.html).

      The lower the better.
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ""
          axisPlacement: auto
          barAlignment: 0
          barWidthFactor: 0.6
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          showValues: false
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: line
        decimals: 0
        links: []
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
            value: 0
        unit: s
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 12
      "y": 71
    id: 136
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        hideZeros: false
        mode: multi
        sort: desc
    pluginVersion: 12.2.0
    type: timeseries
  - title: Threads ($instance)
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "sum(max_over_time(process_num_threads{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}[$__rate_interval])) by(job,%s)" $clusterLabel $clusterLabel }}
      editorMode: code
      exemplar: true
      format: time_series
      interval: ""
      intervalFactor: 2
      legendFormat: __auto
      range: true
      refId: A
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ""
          axisPlacement: auto
          barAlignment: 0
          barWidthFactor: 0.6
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          showValues: false
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: "off"
        decimals: 0
        links: []
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
            value: 0
          - color: red
            value: 80
        unit: short
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 0
      "y": 79
    id: 143
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        hideZeros: false
        mode: multi
        sort: desc
    pluginVersion: 12.2.0
    type: timeseries
  - title: TCP connections ($instance)
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "sum(vm_tcplistener_conns{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}) by(job,%s)" $clusterLabel $clusterLabel }}
      editorMode: code
      format: time_series
      intervalFactor: 1
      legendFormat: '{{`{{`}}job{{`}}`}}'
      range: true
      refId: A
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ""
          axisPlacement: auto
          barAlignment: 0
          barWidthFactor: 0.6
          drawStyle: line
          fillOpacity: 10
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          showValues: false
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: "off"
        links: []
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
            value: 0
          - color: red
            value: 80
        unit: short
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 12
      "y": 79
    id: 37
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        hideZeros: false
        mode: multi
        sort: none
    pluginVersion: 12.2.0
    type: timeseries
  - title: CPU spent on GC ($instance)
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "max(rate(go_gc_cpu_seconds_total{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}[$__rate_interval]) / rate(process_cpu_seconds_total{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}[$__rate_interval])) by(job,%s)" $clusterLabel $clusterLabel $clusterLabel }}
      editorMode: code
      format: time_series
      interval: ""
      intervalFactor: 2
      legendFormat: __auto
      range: true
      refId: A
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: "Shows the percent of CPU spent on garbage collection.\n\nIf % is high, then CPU usage can be decreased by changing GOGC to higher values. Increasing GOGC value will increase memory usage, and decrease CPU usage.\n\nTry searching for keyword `GOGC` at https://docs.victoriametrics.com/victoriametrics/troubleshooting/ "
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ""
          axisPlacement: auto
          barAlignment: 0
          barWidthFactor: 0.6
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          showValues: false
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: "off"
        decimals: 0
        links: []
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
            value: 0
          - color: red
            value: 80
        unit: percentunit
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 0
      "y": 87
    id: 125
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
      tooltip:
        hideZeros: false
        mode: multi
        sort: desc
    pluginVersion: 12.2.0
    type: timeseries
  - title: TCP connections rate ($instance)
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "sum(rate(vm_tcplistener_accepts_total{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}[$__rate_interval])) by(job,%s)" $clusterLabel $clusterLabel }}
      editorMode: code
      format: time_series
      intervalFactor: 1
      legendFormat: '{{`{{`}}job{{`}}`}}'
      range: true
      refId: A
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ""
          axisPlacement: auto
          barAlignment: 0
          barWidthFactor: 0.6
          drawStyle: line
          fillOpacity: 10
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          showValues: false
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: "off"
        links: []
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
            value: 0
          - color: red
            value: 80
        unit: short
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 12
      "y": 87
    id: 49
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        hideZeros: false
        mode: multi
        sort: none
    pluginVersion: 12.2.0
    type: timeseries
  - title: Memory allocations rate
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "max(rate(go_memstats_alloc_bytes_total{job=~\"$job\",%s=~\"$cluster\"}[$__rate_interval])) by(job,%s)" $clusterLabel $clusterLabel }}
      editorMode: code
      format: time_series
      interval: ""
      intervalFactor: 2
      legendFormat: __auto
      range: true
      refId: A
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: Shows the rate of allocations in memory. Sudden increase in allocations would mean increased pressure on Go Garbage Collector and can saturate CPU resources of the application.
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ""
          axisPlacement: auto
          barAlignment: 0
          barWidthFactor: 0.6
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          showValues: false
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: line
        decimals: 0
        links: []
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
            value: 0
        unit: bytes
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 0
      "y": 95
    id: 133
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
      tooltip:
        hideZeros: false
        mode: multi
        sort: desc
    pluginVersion: 12.2.0
    type: timeseries
  - title: Go scheduling latency
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "max(histogram_quantile(0.99, sum(rate(go_sched_latencies_seconds_bucket{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}[$__rate_interval])) by(job,instance,le,%s))) by(job,%s)" $clusterLabel $clusterLabel $clusterLabel }}
      editorMode: code
      format: time_series
      interval: ""
      intervalFactor: 2
      legendFormat: __auto
      range: true
      refId: A
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: "Shows the time goroutines have spent in runnable state before actually running. The lower is better.\n\nHigh values or values exceeding the threshold is usually a sign of            insufficient CPU resources or CPU throttling. \n\nVerify that service has enough CPU resources. Otherwise, the service could work unreliably with delays in processing."
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ""
          axisPlacement: auto
          barAlignment: 0
          barWidthFactor: 0.6
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          showValues: false
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: line
        decimals: 0
        links: []
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
            value: 0
          - color: red
            value: 0.1
        unit: s
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 12
      "y": 95
    id: 128
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
      tooltip:
        hideZeros: false
        mode: multi
        sort: desc
    pluginVersion: 12.2.0
    type: timeseries
  - title: Network usage
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "(sum(rate(vm_tcplistener_read_bytes_total{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}[$__rate_interval])) by(name,%s) * 8) > 0" $clusterLabel $clusterLabel }}
      editorMode: code
      format: time_series
      intervalFactor: 1
      legendFormat: read via {{`{{`}}name{{`}}`}}
      range: true
      refId: A
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "(sum(rate(vm_tcplistener_written_bytes_total{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}[$__rate_interval])) by(name,%s) * 8) > 0" $clusterLabel $clusterLabel }}
      editorMode: code
      format: time_series
      intervalFactor: 1
      legendFormat: write via {{`{{`}}name{{`}}`}}
      range: true
      refId: B
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: |-
      Shows network usage by VM:
      * Writes show traffic sent to clients
      * Reads show traffic received from clients
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ""
          axisPlacement: auto
          barAlignment: 0
          barWidthFactor: 0.6
          drawStyle: line
          fillOpacity: 10
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          showValues: false
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: "off"
        links: []
        mappings: []
        thresholds:
          mode: absolute
          steps:
          - color: green
            value: 0
          - color: red
            value: 80
        unit: decbits
      overrides:
      - matcher:
          id: byRegexp
          options: /read.*/
        properties:
        - id: custom.transform
          value: negative-Y
    gridPos:
      h: 8
      w: 12
      x: 0
      "y": 103
    id: 127
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        hideZeros: false
        mode: multi
        sort: none
    pluginVersion: 12.2.0
    type: timeseries
  collapsed: true
  gridPos:
    h: 1
    w: 24
    x: 0
    "y": 37
  id: 46
  type: row
- title: Troubleshooting
  panels:
  - title: Churn rate ($instance)
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "sum(rate(vm_new_timeseries_created_total{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}[$__rate_interval])) by(%s)" $clusterLabel $clusterLabel }}
      editorMode: code
      interval: ""
      legendFormat: churn rate
      range: true
      refId: A
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "sum(increase(vm_new_timeseries_created_total{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}[24h])) by(%s)" $clusterLabel $clusterLabel }}
      editorMode: code
      interval: ""
      legendFormat: new series over 24h
      range: true
      refId: B
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: |-
      Shows the rate and total number of new series created over last 24h.

      High [churn rate](https://docs.victoriametrics.com/victoriametrics/faq/#what-is-high-churn-rate) tightly connected with database performance and may result in unexpected OOM's or slow queries. It is recommended to always keep an eye on this metric to avoid unexpected [cardinality](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#cardinality) "explosions".

      The higher churn rate is, the more resources required to handle it. Consider to keep the churn rate as low as possible.

      Good references to read:
      * https://www.robustperception.io/cardinality-is-key
      * https://www.robustperception.io/using-tsdb-analyze-to-investigate-churn-and-cardinality
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ""
          axisPlacement: auto
          barAlignment: 0
          barWidthFactor: 0.6
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          showValues: false
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: "off"
        links: []
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
            value: 0
          - color: red
            value: 80
        unit: short
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 0
      "y": 32
    id: 66
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        hideZeros: false
        mode: multi
        sort: none
    pluginVersion: 12.2.0
    type: timeseries
  - title: Slow inserts %
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "max(rate(vm_slow_row_inserts_total{job=~\"$job\",%s=~\"$cluster\"}[$__rate_interval]) / rate(vm_rows_added_to_storage_total{job=~\"$job\",%s=~\"$cluster\"}[$__rate_interval])) by(%s)" $clusterLabel $clusterLabel $clusterLabel }}
      editorMode: code
      format: time_series
      interval: ""
      intervalFactor: 1
      legendFormat: slow inserts
      range: true
      refId: A
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: "The percentage of [slow inserts](https://docs.victoriametrics.com/victoriametrics/faq/#what-is-a-slow-insert) compared to the total ingestion rate. \n\nThe lower the better. \n\nIn short, slow insert is a cache miss. There are following reasons for slow inserts to go up: \n* Ingestion of completely new, not seen before time series;\n* Not enough memory to maintain big enough caches for the current workload.\n\nIf percentage remains high (>10%) during extended periods of time, then it is likely more RAM is needed for optimal handling of the current number of [active time series](https://docs.victoriametrics.com/victoriametrics/faq/#what-is-an-active-time-series). \n\nSee [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3976#issuecomment-1476883183) for details."
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ""
          axisPlacement: auto
          barAlignment: 0
          barWidthFactor: 0.6
          drawStyle: line
          fillOpacity: 10
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          showValues: false
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: line+area
        decimals: 2
        links: []
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: transparent
            value: 0
          - color: red
            value: 0.1
        unit: percentunit
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 12
      "y": 32
    id: 68
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        hideZeros: false
        mode: multi
        sort: none
    pluginVersion: 12.2.0
    type: timeseries
  - title: Assisted merges
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "sum(increase(vm_assisted_merges_total{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}[$__rate_interval])) by(instance,type,%s) > 0" $clusterLabel $clusterLabel }}
      editorMode: code
      exemplar: false
      format: time_series
      interval: 5m
      intervalFactor: 1
      legendFormat: '{{`{{`}}instance{{`}}`}} - {{`{{`}}type{{`}}`}}'
      range: true
      refId: A
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: Merge assist happens when storage can't keep up with merging parts. This is usually a sign of overload for storage.
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ""
          axisPlacement: auto
          barAlignment: 0
          barWidthFactor: 0.6
          drawStyle: bars
          fillOpacity: 100
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          showValues: false
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: "off"
        links: []
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
            value: 0
          - color: red
            value: 80
        unit: short
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 0
      "y": 40
    id: 116
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
      tooltip:
        hideZeros: false
        mode: multi
        sort: desc
    pluginVersion: 12.2.0
    type: timeseries
  - title: Slow queries rate
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "sum(rate(vm_slow_queries_total{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}[$__rate_interval])) by(job,%s)" $clusterLabel $clusterLabel }}
      editorMode: code
      format: time_series
      intervalFactor: 1
      legendFormat: '{{`{{`}}job{{`}}`}}'
      range: true
      refId: A
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: Slow queries rate according to `search.logSlowQueryDuration` flag, which is `5s` by default.
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ""
          axisPlacement: auto
          barAlignment: 0
          barWidthFactor: 0.6
          drawStyle: line
          fillOpacity: 10
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          showValues: false
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: "off"
        links: []
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
            value: 0
          - color: red
            value: 80
        unit: short
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 12
      "y": 40
    id: 60
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        hideZeros: false
        mode: multi
        sort: none
    pluginVersion: 12.2.0
    type: timeseries
  - title: Cache usage % by type
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "max(vm_cache_size_bytes{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"} / vm_cache_size_max_bytes{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}) by(type,%s)" $clusterLabel $clusterLabel $clusterLabel }}
      editorMode: code
      exemplar: false
      interval: ""
      legendFormat: __auto
      range: true
      refId: A
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: "Shows the percentage of used cache size from the allowed size by type. \nValues close to 100% show the maximum potential utilization.\nValues close to 0% show that cache is underutilized."
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ""
          axisPlacement: auto
          barAlignment: 0
          barWidthFactor: 0.6
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          showValues: false
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: "off"
        mappings: []
        thresholds:
          mode: absolute
          steps:
          - color: green
            value: 0
          - color: red
            value: 80
        unit: percentunit
      overrides: []
    gridPos:
      h: 9
      w: 12
      x: 0
      "y": 48
    id: 90
    options:
      legend:
        calcs:
        - lastNotNull
        - mean
        - max
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        hideZeros: false
        mode: multi
        sort: desc
    pluginVersion: 12.2.0
    type: timeseries
  - title: Cache miss ratio ($instance)
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "max(rate(vm_cache_misses_total{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}[$__rate_interval]) / rate(vm_cache_requests_total{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}[$__rate_interval])) by(type,%s)" $clusterLabel $clusterLabel $clusterLabel }}
      editorMode: code
      exemplar: false
      format: time_series
      interval: ""
      intervalFactor: 1
      legendFormat: __auto
      range: true
      refId: A
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: Shows cache miss ratio. Lower is better.
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ""
          axisPlacement: auto
          barAlignment: 0
          barWidthFactor: 0.6
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          showValues: false
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: "off"
        links: []
        mappings: []
        max: 1
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
            value: 0
          - color: red
            value: 80
        unit: percentunit
      overrides: []
    gridPos:
      h: 9
      w: 12
      x: 12
      "y": 48
    id: 118
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        hideZeros: false
        mode: multi
        sort: desc
    pluginVersion: 12.2.0
    type: timeseries
  - title: Samples dropped for last 1h
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "sum(increase(vm_rows_ignored_total{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}[1h])) by(job,reason,%s)" $clusterLabel $clusterLabel }}
      editorMode: code
      exemplar: true
      interval: ""
      legendFormat: '{{`{{`}}instance{{`}}`}} - {{`{{`}}reason{{`}}`}}'
      range: true
      refId: A
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "sum(increase(vm_relabel_metrics_dropped_total{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}[1h])) by(job,%s)" $clusterLabel $clusterLabel }}
      editorMode: code
      instant: false
      legendFormat: '{{`{{`}}job{{`}}`}} - relabeling'
      range: true
      refId: B
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: |-
      Shows how many samples were ignored or dropped on insertion due to various reasons:
      * timestamp out of retention period or timestamp in future;
      * invalid metric name;
      * exceeding limit for labels length or number;
      * dropped by [relabeling configuration](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#relabeling).
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ""
          axisPlacement: auto
          barAlignment: 0
          barWidthFactor: 0.6
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          showValues: false
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: "off"
        mappings: []
        thresholds:
          mode: absolute
          steps:
          - color: green
            value: 0
          - color: red
            value: 80
        unit: short
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 0
      "y": 57
    id: 131
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        hideZeros: false
        mode: multi
        sort: none
    pluginVersion: 12.2.0
    type: timeseries
  - title: Deduplication rate ($instance)
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "sum(rate(vm_deduplicated_samples_total{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}[$__rate_interval])) by(job,type,%s) > 0" $clusterLabel $clusterLabel }}
      editorMode: code
      exemplar: true
      format: time_series
      interval: ""
      intervalFactor: 1
      legendFormat: __auto
      range: true
      refId: A
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: |-
      Shows rate of deduplicated samples during [deduplication](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#deduplication) or [downsampling](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#downsampling), according to the configured `-dedup.minScrapeInterval` or `-downsampling.period`.

      Should be empty if `-dedup.minScrapeInterval` and `-downsampling.period` isn't set.

      Deduplication happens in two places:
      * `type="merge"` - during [background merges](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#storage). Deduplication during merges is permanent.
      * `type="select"` - during [read queries](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#query-data). Deduplication happens in-flight and has no permanent effect.

      The more samples need to be deduplicated, the higher will be resource usage.
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ""
          axisPlacement: auto
          barAlignment: 0
          barWidthFactor: 0.6
          drawStyle: line
          fillOpacity: 10
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          showValues: false
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: "off"
        links: []
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
            value: 0
          - color: red
            value: 80
        unit: short
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 12
      "y": 57
    id: 132
    links:
    - targetBlank: true
      title: Readonly mode
      url: https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#readonly-mode
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
      tooltip:
        hideZeros: false
        mode: multi
        sort: none
    pluginVersion: 12.2.0
    type: timeseries
  - title: Non-default flags
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "sum(flag{is_set=\"true\",job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}) by(job,instance,name,value,%s)" $clusterLabel $clusterLabel }}
      editorMode: code
      exemplar: false
      format: table
      instant: true
      legendFormat: __auto
      range: false
      refId: A
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: Flags explicitly set to non-default values
    fieldConfig:
      defaults:
        color:
          mode: thresholds
        custom:
          align: auto
          cellOptions:
            type: auto
          filterable: true
          footer:
            reducers: []
          inspect: false
        mappings: []
        thresholds:
          mode: absolute
          steps:
          - color: green
            value: 0
          - color: red
            value: 80
      overrides:
      - matcher:
          id: byName
          options: Time
        properties:
        - id: custom.hideFrom.viz
          value: true
      - matcher:
          id: byName
          options: Value
        properties:
        - id: custom.hideFrom.viz
          value: true
      - matcher:
          id: byName
          options: job
        properties:
        - id: custom.hideFrom.viz
          value: true
    gridPos:
      h: 8
      w: 12
      x: 0
      "y": 65
    id: 126
    options:
      cellHeight: sm
      filterable: true
      showHeader: true
      showSearch: true
    pluginVersion: 12.2.0
    type: table
  - title: Restarts ($job)
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "sum(changes(vm_app_start_timestamp{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}[$__rate_interval]) > 0) by(job,%s)" $clusterLabel $clusterLabel }}
      editorMode: code
      format: time_series
      instant: false
      legendFormat: '{{`{{`}}job{{`}}`}}'
      refId: A
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: 'Shows the number of restarts per job. The chart can be useful to identify periodic process restarts and correlate them with potential issues or anomalies. Normally, processes shouldn''t restart unless restart was inited by user. The reason of restarts should be figured out by checking the logs of each specific service. '
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ""
          axisPlacement: auto
          axisSoftMin: 0
          barAlignment: 0
          barWidthFactor: 0.6
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: stepAfter
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          showValues: false
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: "off"
        decimals: 0
        links: []
        mappings: []
        thresholds:
          mode: absolute
          steps:
          - color: green
            value: 0
          - color: red
            value: 80
        unit: none
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 12
      "y": 65
    id: 129
    options:
      legend:
        calcs:
        - lastNotNull
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        hideZeros: false
        mode: multi
        sort: desc
    pluginVersion: 12.2.0
    type: timeseries
  - title: Major page faults rate
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: ${ds}
      expr: {{ printf "sum(rate(process_major_pagefaults_total{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}[$__rate_interval])) by(job,%s) > 0" $clusterLabel $clusterLabel }}
      editorMode: code
      legendFormat: __auto
      range: true
      refId: A
    datasource:
      type: {{ $defaultDatasource }}
      uid: ${ds}
    description: |-
      Major page faults occur when required data is not in physical memory and must be loaded from disk (storage or swap). It usually happens with:
      - insufficient physical memory
      - active use of swap space
      - file mapping access

      It may cause significant performance slowdown if sustained high.

      Possible workaround:
      - Extend RAM to scale up
      - Reduce swap usage by adjusting vm.swappiness
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ""
          axisPlacement: auto
          barAlignment: 0
          barWidthFactor: 0.6
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: auto
          showValues: false
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: "off"
        links:
        - targetBlank: true
          title: Drilldown
          url: /d/wNf0q_kZk?viewPanel=140&var-job=${__field.labels.job}&var-ds=$ds&${__url_time_range}
        mappings: []
        thresholds:
          mode: absolute
          steps:
          - color: green
            value: 0
          - color: red
            value: 80
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 0
      "y": 73
    id: 137
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
      tooltip:
        hideZeros: false
        mode: multi
        sort: desc
    pluginVersion: 12.2.0
    type: timeseries
  collapsed: true
  gridPos:
    h: 1
    w: 24
    x: 0
    "y": 38
  id: 71
  type: row
- title: Storage
  panels:
  - title: Datapoints ingestion rate ($instance)
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "sum(rate(vm_rows_inserted_total{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}[$__rate_interval])) by(job,type,%s) > 0" $clusterLabel $clusterLabel }}
      editorMode: code
      format: time_series
      intervalFactor: 1
      legendFormat: '{{`{{`}}job{{`}}`}} - {{`{{`}}type{{`}}`}}'
      range: true
      refId: A
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: How many datapoints are inserted into storage per second
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ""
          axisPlacement: auto
          barAlignment: 0
          barWidthFactor: 0.6
          drawStyle: line
          fillOpacity: 10
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          showValues: false
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: "off"
        links: []
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
            value: 0
          - color: red
            value: 80
        unit: short
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 0
      "y": 33
    id: 10
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        hideZeros: false
        mode: multi
        sort: desc
    pluginVersion: 12.2.0
    type: timeseries
  - title: Storage full ETA ($instance)
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "((vm_free_disk_space_bytes{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"} - vm_free_disk_space_limit_bytes{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}) / ignoring(path) (((rate(vm_rows_added_to_storage_total{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}[1d]) - sum(rate(vm_deduplicated_samples_total{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}[1d])) without(type)) * (sum(vm_data_size_bytes{job=~\"$job\",instance=~\"$instance\",type!~\"indexdb.*\",%s=~\"$cluster\"}) without(type) / sum(vm_rows{job=~\"$job\",instance=~\"$instance\",type!~\"indexdb.*\",%s=~\"$cluster\"}) without(type))) + (rate(vm_new_timeseries_created_total{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}[1d]) * (sum(vm_data_size_bytes{job=~\"$job\",instance=~\"$instance\",type=\"indexdb/file\",%s=~\"$cluster\"}) without(type) / sum(vm_rows{job=~\"$job\",instance=~\"$instance\",type=\"indexdb/file\",%s=~\"$cluster\"}) without(type))))) > 0" $clusterLabel $clusterLabel $clusterLabel $clusterLabel $clusterLabel $clusterLabel $clusterLabel $clusterLabel $clusterLabel }}
      editorMode: code
      format: time_series
      interval: ""
      intervalFactor: 1
      legendFormat: min ETA
      range: true
      refId: A
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: |-
      Shows the approx time needed to reach 100% of allowed disk capacity based on the following params:
      * free disk space (after -storage.minFreeDiskSpaceBytes);
      * row ingestion rate;
      * compression.
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ""
          axisPlacement: auto
          barAlignment: 0
          barWidthFactor: 0.6
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          showValues: false
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: "off"
        links:
        - targetBlank: true
          title: Drilldown
          url: /d/wNf0q_kZk?viewPanel=150&var-job=${__field.labels.job}&var-ds=$ds&${__url_time_range}
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
            value: 0
          - color: red
            value: 80
        unit: s
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 12
      "y": 33
    id: 73
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - min
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        hideZeros: false
        mode: multi
        sort: desc
    pluginVersion: 12.2.0
    type: timeseries
  - title: Concurrent flushes on disk ($instance)
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "max(max_over_time(vm_concurrent_insert_current{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}[$__rate_interval])) by(%s)" $clusterLabel $clusterLabel }}
      editorMode: code
      exemplar: true
      interval: ""
      legendFormat: current
      range: true
      refId: A
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "min(vm_concurrent_insert_capacity{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}) by(%s)" $clusterLabel $clusterLabel }}
      editorMode: code
      exemplar: true
      interval: ""
      legendFormat: max
      range: true
      refId: B
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: "Shows how many ongoing insertions (not API /write calls) on disk are taking place, where:\n* `max` - equal to number of CPUs;\n* `current` - current number of goroutines busy with inserting rows into underlying storage.\n\nEvery successful API /write call results into flush on disk. The `max` is an internal limit and can't be changed. It is always equal to the number of CPUs. \n\nWhen `current` hits `max` constantly, it means storage is overloaded and requires more CPU (see CPU usage) or disks with more IOPS (see disk writes and reads panels in Resource Usage row)."
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ""
          axisPlacement: auto
          barAlignment: 0
          barWidthFactor: 0.6
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          showValues: false
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: "off"
        mappings: []
        thresholds:
          mode: absolute
          steps:
          - color: green
            value: 0
          - color: red
            value: 80
        unit: short
      overrides:
      - matcher:
          id: byName
          options: max
        properties:
        - id: color
          value:
            fixedColor: '#C4162A'
            mode: fixed
    gridPos:
      h: 8
      w: 12
      x: 0
      "y": 41
    id: 144
    links:
    - targetBlank: true
      title: Related discussion
      url: https://github.com/VictoriaMetrics/VictoriaMetrics/issues/632
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        hideZeros: false
        mode: multi
        sort: none
    pluginVersion: 12.2.0
    type: timeseries
  - title: 'Pending datapoints '
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "vm_pending_rows{job=~\"$job\",instance=~\"$instance\",type=\"storage\",%s=~\"$cluster\"}" $clusterLabel }}
      editorMode: code
      format: time_series
      intervalFactor: 1
      legendFormat: '{{`{{`}}job{{`}}`}} - pending datapoints'
      range: true
      refId: A
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "vm_pending_rows{job=~\"$job\",instance=~\"$instance\",type=\"indexdb\",%s=~\"$cluster\"}" $clusterLabel }}
      editorMode: code
      format: time_series
      intervalFactor: 1
      legendFormat: '{{`{{`}}job{{`}}`}} - pending index entries'
      range: true
      refId: B
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: How many data points are in RAM queue waiting to be written into storage. The number of pending data points should be in the range from 0 to `5*<ingestion_rate>`, since VictoriaMetrics pushes pending data to persistent storage every 5 seconds. The index datapoints value in general is much lower
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ""
          axisPlacement: auto
          barAlignment: 0
          barWidthFactor: 0.6
          drawStyle: line
          fillOpacity: 10
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          showValues: false
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: "off"
        links: []
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
            value: 0
          - color: red
            value: 80
        unit: short
      overrides:
      - matcher:
          id: byName
          options: pending index entries
        properties:
        - id: unit
          value: none
        - id: decimals
          value: 3
    gridPos:
      h: 8
      w: 12
      x: 12
      "y": 41
    id: 34
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        hideZeros: false
        mode: multi
        sort: none
    pluginVersion: 12.2.0
    type: timeseries
  - title: Active merges ($instance)
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "sum(vm_active_merges{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}) by(job,type,%s)" $clusterLabel $clusterLabel }}
      editorMode: code
      legendFormat: '{{`{{`}}job{{`}}`}} - {{`{{`}}type{{`}}`}}'
      range: true
      refId: A
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: The number of on-going merges in storage nodes.  It is expected to have high numbers for `storage/small` metric.
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ""
          axisPlacement: auto
          barAlignment: 0
          barWidthFactor: 0.6
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          showValues: false
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: "off"
        decimals: 0
        links: []
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
            value: 0
          - color: red
            value: 80
        unit: short
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 0
      "y": 49
    id: 62
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
      tooltip:
        hideZeros: false
        mode: multi
        sort: none
    pluginVersion: 12.2.0
    type: timeseries
  - title: Merge speed
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "sum(rate(vm_rows_merged_total{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}[$__rate_interval])) by(job,type,%s)" $clusterLabel $clusterLabel }}
      editorMode: code
      legendFormat: '{{`{{`}}job{{`}}`}} - {{`{{`}}type{{`}}`}}'
      range: true
      refId: A
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: The number of rows merged per second by storage nodes.
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ""
          axisPlacement: auto
          barAlignment: 0
          barWidthFactor: 0.6
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          showValues: false
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: "off"
        decimals: 0
        links: []
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
            value: 0
          - color: red
            value: 80
        unit: short
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 12
      "y": 49
    id: 64
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        hideZeros: false
        mode: multi
        sort: none
    pluginVersion: 12.2.0
    type: timeseries
  - title: Disk space usage %  ($instance)
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "max(sum(vm_data_size_bytes{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}) by(job,instance,%s) / (sum(vm_free_disk_space_bytes{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"} - vm_free_disk_space_limit_bytes{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}) by(job,instance,%s) + sum(vm_data_size_bytes{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}) by(job,instance,%s))) by(%s)" $clusterLabel $clusterLabel $clusterLabel $clusterLabel $clusterLabel $clusterLabel $clusterLabel $clusterLabel }}
      editorMode: code
      format: time_series
      intervalFactor: 1
      legendFormat: max
      range: true
      refId: A
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "min(sum(vm_data_size_bytes{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}) by(job,instance,%s) / (sum(vm_free_disk_space_bytes{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"} - vm_free_disk_space_limit_bytes{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}) by(job,instance,%s) + sum(vm_data_size_bytes{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}) by(job,instance,%s))) by(%s)" $clusterLabel $clusterLabel $clusterLabel $clusterLabel $clusterLabel $clusterLabel $clusterLabel $clusterLabel }}
      editorMode: code
      format: time_series
      intervalFactor: 1
      legendFormat: min
      range: true
      refId: B
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "quantile(0.5, sum(vm_data_size_bytes{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}) by(job,instance,%s) / (sum(vm_free_disk_space_bytes{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"} - vm_free_disk_space_limit_bytes{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}) by(job,instance,%s) + sum(vm_data_size_bytes{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}) by(job,instance,%s))) by(%s)" $clusterLabel $clusterLabel $clusterLabel $clusterLabel $clusterLabel $clusterLabel $clusterLabel $clusterLabel }}
      editorMode: code
      format: time_series
      intervalFactor: 1
      legendFormat: median
      range: true
      refId: C
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: Shows the percentage of used disk space. It is recommended to have at least 20% of free disk space for the best performance.
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ""
          axisPlacement: auto
          barAlignment: 0
          barWidthFactor: 0.6
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          showValues: false
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: line
        links:
        - targetBlank: true
          title: Drilldown
          url: /d/wNf0q_kZk?viewPanel=153&var-job=${__field.labels.job}&var-ds=$ds&${__url_time_range}
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
            value: 0
          - color: red
            value: 80
        unit: percentunit
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 0
      "y": 57
    id: 145
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        hideZeros: false
        mode: multi
        sort: desc
    pluginVersion: 12.2.0
    type: timeseries
  - title: LSM parts max by type ($instance)
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "sum(vm_parts{job=~\"$job\",instance=~\"$instance\",type=~\"indexdb.*\",%s=~\"$cluster\"}) by(job,type,%s)" $clusterLabel $clusterLabel }}
      editorMode: code
      format: time_series
      intervalFactor: 1
      legendFormat: '{{`{{`}}job{{`}}`}} - {{`{{`}}type{{`}}`}}'
      range: true
      refId: A
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "max(vm_last_partition_parts{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}) by(job,type,%s)" $clusterLabel $clusterLabel }}
      editorMode: code
      format: time_series
      intervalFactor: 1
      legendFormat: '{{`{{`}}job{{`}}`}} - {{`{{`}}type{{`}}`}}'
      range: true
      refId: B
    datasource:
      uid: $ds
    description: |-
      The max number of data parts of LSM tree across all storage nodes in the last [partition](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#storage).
      Increase in number of parts (the hard limit is 512) is an evidence of slow merge performance - check the resource utilization.
      * `indexdb` - inverted index
      * `storage/small` - recently added parts of data ingested into storage (hot data)
      * `storage/big` -  small parts gradually merged into bigger parts (cold data)
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ""
          axisPlacement: auto
          barAlignment: 0
          barWidthFactor: 0.6
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          showValues: false
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: "off"
        links: []
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
            value: 0
          - color: red
            value: 80
        unit: short
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 12
      "y": 57
    id: 36
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        hideZeros: false
        mode: multi
        sort: desc
    pluginVersion: 12.2.0
    type: timeseries
  - title: Disk space usage % by type  ($instance)
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "max(sum(vm_data_size_bytes{job=~\"$job\",instance=~\"$instance\",type=~\"indexdb.*\",%s=~\"$cluster\"}) by(job,instance,%s) / sum(vm_data_size_bytes{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}) by(job,instance,%s)) by(%s)" $clusterLabel $clusterLabel $clusterLabel $clusterLabel $clusterLabel }}
      editorMode: code
      format: time_series
      intervalFactor: 1
      legendFormat: indexdb
      range: true
      refId: A
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "max(sum(vm_data_size_bytes{job=~\"$job\",instance=~\"$instance\",type!~\"indexdb.*\",%s=~\"$cluster\"}) by(job,instance,%s) / sum(vm_data_size_bytes{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}) by(job,instance,%s)) by(%s)" $clusterLabel $clusterLabel $clusterLabel $clusterLabel $clusterLabel }}
      editorMode: code
      format: time_series
      intervalFactor: 1
      legendFormat: datapoints
      range: true
      refId: B
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: |-
      Shows the percentage of used disk space by type: datapoints or indexdb. Normally, indexdb takes much less space comparing to datapoints. But with high [churn rate](https://docs.victoriametrics.com/victoriametrics/faq/#what-is-high-churn-rate) the size of the indexdb could grow significantly.

      The sum of the % can be > 100% since panel shows max % per-job and per-instance. It means different instance can have different ratio between datapoints and indexdb size.
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ""
          axisPlacement: auto
          barAlignment: 0
          barWidthFactor: 0.6
          drawStyle: line
          fillOpacity: 10
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          showValues: false
          spanNulls: false
          stacking:
            group: A
            mode: percent
          thresholdsStyle:
            mode: line
        links:
        - targetBlank: true
          title: Drilldown
          url: /d/wNf0q_kZk?viewPanel=152&var-job=${__field.labels.job}&var-ds=$ds&${__url_time_range}
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
            value: 0
        unit: percentunit
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 0
      "y": 65
    id: 146
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        hideZeros: false
        mode: multi
        sort: desc
    pluginVersion: 12.2.0
    type: timeseries
  - title: Disk space usage - datapoints
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "sum(vm_data_size_bytes{job=~\"$job\",instance=~\"$instance\",type!~\"indexdb.*\",%s=~\"$cluster\"}) by(job,%s)" $clusterLabel $clusterLabel }}
      editorMode: code
      format: time_series
      interval: ""
      intervalFactor: 1
      legendFormat: '{{`{{`}}job{{`}}`}} - Used (datapoints)'
      range: true
      refId: A
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "vm_free_disk_space_bytes{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"} - vm_free_disk_space_limit_bytes{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}" $clusterLabel $clusterLabel }}
      editorMode: code
      format: time_series
      interval: ""
      intervalFactor: 1
      legendFormat: '{{`{{`}}job{{`}}`}} - Free'
      range: true
      refId: B
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "sum(vm_data_size_bytes{job=~\"$job\",instance=~\"$instance\",type=~\"indexdb.*\",%s=~\"$cluster\"}) by(job,%s)" $clusterLabel $clusterLabel }}
      editorMode: code
      format: time_series
      interval: ""
      intervalFactor: 1
      legendFormat: '{{`{{`}}job{{`}}`}} - Used (index)'
      range: true
      refId: C
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: Shows amount of on-disk space occupied by data points and the remaining disk space at `-storageDataPath`. Calculation accounts for -storage.minFreeDiskSpaceBytes
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ""
          axisPlacement: auto
          barAlignment: 0
          barWidthFactor: 0.6
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          showValues: false
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: "off"
        links: []
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
            value: 0
          - color: red
            value: 80
        unit: bytes
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 12
      "y": 65
    id: 53
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        hideZeros: false
        mode: multi
        sort: desc
    pluginVersion: 12.2.0
    type: timeseries
  - title: Datapoints
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "sum(vm_rows{job=~\"$job\",instance=~\"$instance\",type!~\"indexdb.*\",%s=~\"$cluster\"}) by(job,%s)" $clusterLabel $clusterLabel }}
      editorMode: code
      format: time_series
      interval: ""
      intervalFactor: 1
      legendFormat: '{{`{{`}}job{{`}}`}} - total datapoints'
      range: true
      refId: A
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "sum(vm_data_size_bytes{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}) by(job,%s) / sum(vm_rows{job=~\"$job\",instance=~\"$instance\",type!~\"indexdb.*\",%s=~\"$cluster\"}) by(job,%s)" $clusterLabel $clusterLabel $clusterLabel $clusterLabel }}
      editorMode: code
      format: time_series
      interval: ""
      intervalFactor: 1
      legendFormat: '{{`{{`}}job{{`}}`}} - bytes-per-datapoint'
      range: true
      refId: B
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: Shows how many datapoints are in the storage and what is average disk usage per datapoint.
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ""
          axisPlacement: auto
          barAlignment: 0
          barWidthFactor: 0.6
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          showValues: false
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: "off"
        links: []
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
            value: 0
          - color: red
            value: 80
        unit: short
      overrides:
      - matcher:
          id: byName
          options: bytes-per-datapoint
        properties:
        - id: unit
          value: bytes
        - id: decimals
          value: 2
    gridPos:
      h: 8
      w: 12
      x: 0
      "y": 73
    id: 30
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        hideZeros: false
        mode: multi
        sort: desc
    pluginVersion: 12.2.0
    type: timeseries
  - title: Pending datapoints ($instance)
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "sum(max_over_time(vm_pending_rows{job=~\"$job\",instance=~\"$instance\",type=\"storage\",%s=~\"$cluster\"}[$__rate_interval])) by(%s)" $clusterLabel $clusterLabel }}
      editorMode: code
      format: time_series
      intervalFactor: 1
      legendFormat: pending datapoints
      range: true
      refId: A
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "sum(max_over_time(vm_pending_rows{job=~\"$job\",instance=~\"$instance\",type=\"indexdb\",%s=~\"$cluster\"}[$__rate_interval])) by(%s)" $clusterLabel $clusterLabel }}
      editorMode: code
      format: time_series
      intervalFactor: 1
      legendFormat: pending index entries
      range: true
      refId: B
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: How many data points are in RAM queue waiting to be written into storage. The number of pending data points should be in the range from 0 to `5*<ingestion_rate>`, since VictoriaMetrics pushes pending data to persistent storage every 5 seconds. The index datapoints value in general is much lower.
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ""
          axisPlacement: auto
          barAlignment: 0
          barWidthFactor: 0.6
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          showValues: false
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: "off"
        links: []
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
            value: 0
          - color: red
            value: 80
        unit: short
      overrides:
      - matcher:
          id: byName
          options: pending index entries
        properties:
        - id: unit
          value: none
        - id: decimals
          value: 3
    gridPos:
      h: 8
      w: 12
      x: 12
      "y": 73
    id: 147
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        hideZeros: false
        mode: multi
        sort: none
    pluginVersion: 12.2.0
    type: timeseries
  - title: Rows ignored for last 1h
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "sum(increase(vm_rows_ignored_total{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}[1h])) by(instance,reason,%s)" $clusterLabel $clusterLabel }}
      editorMode: code
      exemplar: false
      format: time_series
      interval: ""
      intervalFactor: 1
      legendFormat: '{{`{{`}}instance{{`}}`}} - {{`{{`}}reason{{`}}`}}'
      range: true
      refId: A
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: Shows how many rows were ignored on insertion due to corrupted or out of retention timestamps and how many series were ignored on insertion or during scrape due to too long label names, values and excessive labels count.
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ""
          axisPlacement: auto
          barAlignment: 0
          barWidthFactor: 0.6
          drawStyle: line
          fillOpacity: 10
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          showValues: false
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: "off"
        links: []
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
            value: 0
          - color: red
            value: 80
        unit: short
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 0
      "y": 81
    id: 58
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
      tooltip:
        hideZeros: false
        mode: multi
        sort: none
    pluginVersion: 12.2.0
    type: timeseries
  - title: Datapoints read per series
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "histogram_quantile(0.99, sum(rate(vm_rows_read_per_series_bucket{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}[$__rate_interval])) by(job,vmrange,%s))" $clusterLabel $clusterLabel }}
      editorMode: code
      format: time_series
      interval: ""
      intervalFactor: 1
      legendFormat: '{{`{{`}}job{{`}}`}}'
      range: true
      refId: A
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: 99th percentile of number of [data samples](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#raw-samples) read per queried series.
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ""
          axisPlacement: auto
          barAlignment: 0
          barWidthFactor: 0.6
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          showValues: false
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: "off"
        decimals: 2
        links: []
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
            value: 0
          - color: red
            value: 80
        unit: short
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 12
      "y": 81
    id: 103
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        hideZeros: false
        mode: multi
        sort: desc
    pluginVersion: 12.2.0
    type: timeseries
  - title: Concurrent flushes on disk {$instance}
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "max_over_time(vm_concurrent_insert_capacity{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}[$__rate_interval])" $clusterLabel }}
      editorMode: code
      format: time_series
      interval: ""
      intervalFactor: 1
      legendFormat: '{{`{{`}}instance{{`}}`}} - max'
      range: true
      refId: A
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "sum(vm_concurrent_insert_current{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}) by(instance,%s)" $clusterLabel $clusterLabel }}
      editorMode: code
      format: time_series
      intervalFactor: 1
      legendFormat: '{{`{{`}}instance{{`}}`}} - current'
      range: true
      refId: B
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: "Shows how many ongoing insertions (not API /write calls) on disk are taking place, where:\n* `max` - equal to number of CPUs;\n* `current` - current number of goroutines busy with inserting rows into underlying storage.\n\nEvery successful API /write call results into flush on disk. However, these two actions are separated and controlled via different concurrency limiters. The `max` on this panel can't be changed and always equal to number of CPUs. \n\nWhen `current` hits `max` constantly, it means storage is overloaded and requires more CPU.\n\n"
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ""
          axisPlacement: auto
          barAlignment: 0
          barWidthFactor: 0.6
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          showValues: false
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: "off"
        decimals: 0
        links: []
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
            value: 0
          - color: red
            value: 80
        unit: short
      overrides:
      - matcher:
          id: byName
          options: max
        properties:
        - id: color
          value:
            fixedColor: '#C4162A'
            mode: fixed
    gridPos:
      h: 8
      w: 12
      x: 0
      "y": 89
    id: 59
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        displayMode: table
        placement: bottom
        showLegend: true
      tooltip:
        hideZeros: false
        mode: multi
        sort: desc
    pluginVersion: 12.2.0
    type: timeseries
  - title: Datapoints scanned per query
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "histogram_quantile(0.99, sum(rate(vm_rows_scanned_per_query_bucket{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}[$__rate_interval])) by(job,vmrange,%s))" $clusterLabel $clusterLabel }}
      editorMode: code
      format: time_series
      interval: ""
      intervalFactor: 1
      legendFormat: '{{`{{`}}job{{`}}`}}'
      range: true
      refId: A
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: |-
      99th percentile of number of [data samples](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#raw-samples) scanner per query.

      This number can exceed number of RowsReadPerQuery if `step` query arg passed to [/api/v1/query_range](https://prometheus.io/docs/prometheus/latest/querying/api/#range-queries) is smaller than the lookbehind window set in square brackets of [rollup function](https://docs.victoriametrics.com/victoriametrics/metricsql/#rollup-functions). For example, if `increase(some_metric[1h])` is executed with the `step=5m`, then the same [data samples](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#raw-samples) on a hour time range are scanned `1h/5m=12` times. See [this article](https://valyala.medium.com/how-to-optimize-promql-and-metricsql-queries-85a1b75bf986) for details.
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ""
          axisPlacement: auto
          barAlignment: 0
          barWidthFactor: 0.6
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          showValues: false
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: "off"
        decimals: 2
        links: []
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
            value: 0
          - color: red
            value: 80
        unit: short
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 12
      "y": 89
    id: 105
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        hideZeros: false
        mode: multi
        sort: desc
    pluginVersion: 12.2.0
    type: timeseries
  - title: Series read per query
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "histogram_quantile(0.99, sum(rate(vm_series_read_per_query_bucket{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}[$__rate_interval])) by(job,vmrange,%s))" $clusterLabel $clusterLabel }}
      editorMode: code
      format: time_series
      interval: ""
      intervalFactor: 1
      legendFormat: '{{`{{`}}job{{`}}`}}'
      range: true
      refId: A
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: 99th percentile of number of series read per query.
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ""
          axisPlacement: auto
          barAlignment: 0
          barWidthFactor: 0.6
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          showValues: false
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: "off"
        decimals: 2
        links: []
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
            value: 0
          - color: red
            value: 80
        unit: short
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 0
      "y": 97
    id: 99
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        hideZeros: false
        mode: multi
        sort: desc
    pluginVersion: 12.2.0
    type: timeseries
  - title: Datapoints read per query
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "histogram_quantile(0.99, sum(rate(vm_rows_read_per_query_bucket{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}[$__rate_interval])) by(job,vmrange,%s))" $clusterLabel $clusterLabel }}
      editorMode: code
      format: time_series
      interval: ""
      intervalFactor: 1
      legendFormat: '{{`{{`}}job{{`}}`}}'
      range: true
      refId: A
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: 99th percentile of number of [data samples](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#raw-samples) read per query.
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ""
          axisPlacement: auto
          barAlignment: 0
          barWidthFactor: 0.6
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          showValues: false
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: "off"
        decimals: 2
        links: []
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
            value: 0
          - color: red
            value: 80
        unit: short
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 12
      "y": 97
    id: 122
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        hideZeros: false
        mode: multi
        sort: desc
    pluginVersion: 12.2.0
    type: timeseries
  - title: Number of snapshots
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "vm_snapshots{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}" $clusterLabel }}
      editorMode: code
      exemplar: true
      interval: ""
      legendFormat: '{{`{{`}}job{{`}}`}}'
      range: true
      refId: A
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: |-
      Shows number of existing [snapshots](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-work-with-snapshots).

      Recently made snapshots do not occupy disk space. But with time, as snapshots become older, they start to occupy more and more disk space.

      It is recommended deleting old snapshots when they are no longer needed to free up disk space.
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ""
          axisPlacement: auto
          barAlignment: 0
          barWidthFactor: 0.6
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          showValues: false
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: "off"
        mappings: []
        thresholds:
          mode: absolute
          steps:
          - color: green
            value: 0
          - color: red
            value: 80
        unit: short
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 12
      "y": 105
    id: 130
    options:
      legend:
        calcs: []
        displayMode: list
        placement: bottom
        showLegend: true
      tooltip:
        hideZeros: false
        mode: multi
        sort: desc
    pluginVersion: 12.2.0
    type: timeseries
  collapsed: true
  gridPos:
    h: 1
    w: 24
    x: 0
    "y": 39
  id: 14
  type: row
- title: Drilldown
  panels:
  - fieldConfig:
      defaults: {}
      overrides: []
    gridPos:
      h: 2
      w: 17
      x: 0
      "y": 427
    id: 139
    options:
      code:
        language: plaintext
        showLineNumbers: false
        showMiniMap: false
      content: Drilldown row is used by other panels on the dashboard to show more detailed metrics per-instance, or a single instance in cases like memory breakdown.
      mode: markdown
    pluginVersion: 12.2.0
    transparent: true
    type: text
  - title: Major page faults rate
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: ${ds}
      expr: {{ printf "sum(rate(process_major_pagefaults_total{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}[$__rate_interval])) by(job,instance,%s) > 0" $clusterLabel $clusterLabel }}
      editorMode: code
      legendFormat: '{{`{{`}}instance{{`}}`}} ({{`{{`}}job{{`}}`}})'
      range: true
      refId: A
    datasource:
      type: {{ $defaultDatasource }}
      uid: ${ds}
    description: |-
      Major page faults occur when required data is not in physical memory and must be loaded from disk (storage or swap). It usually happens with:
      - insufficient physical memory
      - active use of swap space
      - file mapping access

      It may cause significant performance slowdown if sustained high.

      Possible workaround:
      - Extend RAM to scale up
      - Reduce swap usage by adjusting vm.swappiness
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ""
          axisPlacement: auto
          barAlignment: 0
          barWidthFactor: 0.6
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: auto
          showValues: false
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: "off"
        links: []
        mappings: []
        thresholds:
          mode: absolute
          steps:
          - color: green
            value: 0
          - color: red
            value: 80
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 0
      "y": 429
    id: 140
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
      tooltip:
        hideZeros: false
        mode: multi
        sort: desc
    pluginVersion: 12.2.0
    type: timeseries
  - title: Memory usage breakdown
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: ${ds}
      expr: {{ printf "max(((go_memstats_gc_sys_bytes{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"} + go_memstats_mspan_sys_bytes{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}) + go_memstats_other_sys_bytes{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}) + go_memstats_mcache_sys_bytes{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}) by(%s)" $clusterLabel $clusterLabel $clusterLabel $clusterLabel $clusterLabel }}
      editorMode: code
      legendFormat: 'Go: Other'
      range: true
      refId: A
    - datasource:
        type: {{ $defaultDatasource }}
        uid: ${ds}
      expr: {{ printf "max(go_memstats_stack_inuse_bytes{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}) by(%s)" $clusterLabel $clusterLabel }}
      editorMode: code
      instant: false
      legendFormat: 'Go: Stack'
      range: true
      refId: B
    - datasource:
        type: {{ $defaultDatasource }}
        uid: ${ds}
      expr: {{ printf "max(go_memstats_heap_inuse_bytes{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"} + (go_memstats_heap_idle_bytes{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"} - go_memstats_heap_released_bytes{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"})) by(%s)" $clusterLabel $clusterLabel $clusterLabel $clusterLabel }}
      editorMode: code
      instant: false
      legendFormat: 'Go: Heap'
      range: true
      refId: C
    - datasource:
        type: {{ $defaultDatasource }}
        uid: ${ds}
      expr: {{ printf "max(go_memstats_heap_released_bytes{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}) by(%s)" $clusterLabel $clusterLabel }}
      editorMode: code
      instant: false
      legendFormat: 'Go: Heap Reserved'
      range: true
      refId: D
    - datasource:
        type: {{ $defaultDatasource }}
        uid: ${ds}
      expr: {{ printf "max(sum(vm_cache_size_bytes{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}) by(job,instance,%s)) by(%s)" $clusterLabel $clusterLabel $clusterLabel }}
      editorMode: code
      instant: false
      legendFormat: 'Mmap: VM Cache'
      range: true
      refId: E
    - datasource:
        type: {{ $defaultDatasource }}
        uid: ${ds}
      expr: {{ printf "max(process_resident_memory_file_bytes{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}) by(%s)" $clusterLabel $clusterLabel }}
      editorMode: code
      instant: false
      legendFormat: File cache
      range: true
      refId: F
    datasource:
      type: {{ $defaultDatasource }}
      uid: ${ds}
    description: |-
      This panel breaks down memory usage by type. It’s intended for inspecting a single instance.

      How to use:
      - Start from the high-level RSS panel.
      - Identify an instance with unexpected or abnormal memory growth.
      - Filter to that instance to inspect the detailed breakdown.

      Interpretation
      - A steadily rising Go Heap usually indicates a memory leak. Collect pprof memory profile.
      - A growing Go Stack commonly points to a goroutine leak.
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ""
          axisPlacement: auto
          barAlignment: 0
          barWidthFactor: 0.6
          drawStyle: line
          fillOpacity: 10
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: auto
          showValues: false
          spanNulls: false
          stacking:
            group: A
            mode: normal
          thresholdsStyle:
            mode: "off"
        links: []
        mappings: []
        thresholds:
          mode: absolute
          steps:
          - color: green
            value: 0
          - color: red
            value: 80
        unit: bytes
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 12
      "y": 429
    id: 141
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
      tooltip:
        hideZeros: false
        mode: multi
        sort: desc
    pluginVersion: 12.2.0
    type: timeseries
  - title: RSS memory usage ($instance)
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "max_over_time(process_resident_memory_bytes{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}[$__rate_interval])" $clusterLabel }}
      editorMode: code
      exemplar: true
      interval: ""
      legendFormat: '{{`{{`}}instance{{`}}`}} ({{`{{`}}job{{`}}`}})'
      range: true
      refId: A
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: |-
      Shows the used memory (resident).
      The application's performance will significantly degrade when memory usage is close to 100%.
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ""
          axisPlacement: auto
          barAlignment: 0
          barWidthFactor: 0.6
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          showValues: false
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: "off"
        links: []
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
            value: 0
        unit: decbytes
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 0
      "y": 437
    id: 148
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        hideZeros: false
        mode: multi
        sort: desc
    pluginVersion: 12.2.0
    type: timeseries
  - title: RSS anonymous memory usage ($instance)
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "max_over_time(process_resident_memory_anon_bytes{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}[$__rate_interval])" $clusterLabel }}
      editorMode: code
      exemplar: true
      interval: ""
      legendFormat: '{{`{{`}}instance{{`}}`}} ({{`{{`}}job{{`}}`}})'
      range: true
      refId: A
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: RSS share for memory allocated by the process itself. This share cannot be freed by the OS, so it must be taken into account by OOM killer.
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ""
          axisPlacement: auto
          barAlignment: 0
          barWidthFactor: 0.6
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          showValues: false
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: "off"
        links: []
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
            value: 0
        unit: decbytes
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 12
      "y": 437
    id: 149
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        hideZeros: false
        mode: multi
        sort: desc
    pluginVersion: 12.2.0
    type: timeseries
  - title: Storage full ETA ($instance)
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "((vm_free_disk_space_bytes{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"} - vm_free_disk_space_limit_bytes{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}) / ignoring(path) (((rate(vm_rows_added_to_storage_total{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}[1d]) - sum(rate(vm_deduplicated_samples_total{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}[1d])) without(type)) * (sum(vm_data_size_bytes{job=~\"$job\",instance=~\"$instance\",type!~\"indexdb.*\",%s=~\"$cluster\"}) without(type) / sum(vm_rows{job=~\"$job\",instance=~\"$instance\",type!~\"indexdb.*\",%s=~\"$cluster\"}) without(type))) + (rate(vm_new_timeseries_created_total{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}[1d]) * (sum(vm_data_size_bytes{job=~\"$job\",instance=~\"$instance\",type=\"indexdb/file\",%s=~\"$cluster\"}) without(type) / sum(vm_rows{job=~\"$job\",instance=~\"$instance\",type=\"indexdb/file\",%s=~\"$cluster\"}) without(type))))) > 0" $clusterLabel $clusterLabel $clusterLabel $clusterLabel $clusterLabel $clusterLabel $clusterLabel $clusterLabel $clusterLabel }}
      editorMode: code
      format: time_series
      interval: ""
      intervalFactor: 1
      legendFormat: '{{`{{`}}instance{{`}}`}}'
      range: true
      refId: A
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: |-
      Shows the approx time needed to reach 100% of allowed disk capacity based on the following params:
      * free disk space (after -storage.minFreeDiskSpaceBytes);
      * row ingestion rate;
      * compression.
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ""
          axisPlacement: auto
          barAlignment: 0
          barWidthFactor: 0.6
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          showValues: false
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: "off"
        links: []
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
            value: 0
          - color: red
            value: 80
        unit: s
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 0
      "y": 445
    id: 150
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - min
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        hideZeros: false
        mode: multi
        sort: none
    pluginVersion: 12.2.0
    type: timeseries
  - title: CPU usage ($instance)
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "sum(rate(process_cpu_seconds_total{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}[$__rate_interval])) by(job,instance,%s)" $clusterLabel $clusterLabel }}
      editorMode: code
      exemplar: false
      format: time_series
      interval: ""
      intervalFactor: 1
      legendFormat: '{{`{{`}}instance{{`}}`}} ({{`{{`}}job{{`}}`}})'
      range: true
      refId: A
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ""
          axisPlacement: auto
          barAlignment: 0
          barWidthFactor: 0.6
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          showValues: false
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: "off"
        links: []
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
            value: 0
        unit: short
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 12
      "y": 445
    id: 151
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        hideZeros: false
        mode: multi
        sort: desc
    pluginVersion: 12.2.0
    type: timeseries
  - title: Disk space usage by type ($instance)
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "sum(vm_data_size_bytes{job=~\"$job\",instance=~\"$instance\",type=~\"indexdb.*\",%s=~\"$cluster\"}) by(job,instance,%s)" $clusterLabel $clusterLabel }}
      editorMode: code
      format: time_series
      intervalFactor: 1
      legendFormat: '{{`{{`}}job{{`}}`}}:{{`{{`}}instance{{`}}`}} (indexdb)'
      range: true
      refId: A
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "sum(vm_data_size_bytes{job=~\"$job\",instance=~\"$instance\",type!~\"indexdb.*\",%s=~\"$cluster\"}) by(job,instance,%s)" $clusterLabel $clusterLabel }}
      editorMode: code
      format: time_series
      intervalFactor: 1
      legendFormat: '{{`{{`}}job{{`}}`}}:{{`{{`}}instance{{`}}`}} (datapoints)'
      range: true
      refId: B
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ""
          axisPlacement: auto
          barAlignment: 0
          barWidthFactor: 0.6
          drawStyle: line
          fillOpacity: 10
          gradientMode: opacity
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          showValues: false
          spanNulls: false
          stacking:
            group: A
            mode: percent
          thresholdsStyle:
            mode: line
        links: []
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
            value: 0
        unit: decbytes
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 0
      "y": 453
    id: 152
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        hideZeros: false
        mode: multi
        sort: desc
    pluginVersion: 12.2.0
    type: timeseries
  - title: Disk space usage ($instance)
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "sum(vm_data_size_bytes{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}) by(job,instance,%s)" $clusterLabel $clusterLabel }}
      editorMode: code
      format: time_series
      intervalFactor: 1
      legendFormat: ""
      range: true
      refId: A
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: Shows the percentage of used disk space. It is recommended to have at least 20% of free disk space for the best performance.
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ""
          axisPlacement: auto
          barAlignment: 0
          barWidthFactor: 0.6
          drawStyle: line
          fillOpacity: 0
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          showValues: false
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: line
        links: []
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
            value: 0
        unit: decbytes
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 12
      "y": 453
    id: 153
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        hideZeros: false
        mode: multi
        sort: desc
    pluginVersion: 12.2.0
    type: timeseries
  - title: Logging rate
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "sum(rate(vm_log_messages_total{job=~\"$job\",instance=~\"$instance\",level!=\"info\",%s=~\"$cluster\"}[$__rate_interval])) by(job,instance,level,app_version,location,%s) > 0" $clusterLabel $clusterLabel }}
      editorMode: code
      exemplar: true
      format: time_series
      interval: 5m
      intervalFactor: 1
      legendFormat: '{{`{{`}}instance{{`}}`}} ({{`{{`}}job{{`}}`}}) - {{`{{`}}level{{`}}`}}: {{`{{`}}location{{`}}`}}'
      range: true
      refId: A
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    description: Shows the rate of logging the messages by their level. Unexpected spike in rate is a good reason to check logs.
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ""
          axisPlacement: auto
          barAlignment: 0
          barWidthFactor: 0.6
          drawStyle: bars
          fillOpacity: 100
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          showValues: false
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: "off"
        links:
        - targetBlank: true
          title: Source Code
          url: https://l2s.victoriametrics.com/?app_version=${__field.labels.app_version}&location=${__field.labels.location}
        mappings: []
        min: 0
        thresholds:
          mode: absolute
          steps:
          - color: green
            value: 0
          - color: red
            value: 80
        unit: short
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 0
      "y": 461
    id: 154
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
        sortBy: Last *
        sortDesc: true
      tooltip:
        hideZeros: false
        mode: multi
        sort: desc
    pluginVersion: 12.2.0
    type: timeseries
  collapsed: true
  gridPos:
    h: 1
    w: 24
    x: 0
    "y": 40
  id: 138
  type: row
condition: {{ ($Values.vmsingle).enabled }}
description: Overview for single-node VictoriaMetrics v1.117.0 or higher
fiscalYearStartMonth: 0
graphTooltip: 1
id: 13
links:
- icon: doc
  tags: []
  targetBlank: true
  title: Docs
  type: link
  url: https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/
- asDropdown: false
  icon: question
  includeVars: false
  keepTime: false
  tags: []
  targetBlank: true
  title: Troubleshooting
  tooltip: ""
  type: link
  url: https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#troubleshooting
- icon: info
  tags: []
  targetBlank: true
  title: Community support
  type: link
  url: https://victoriametrics.com/support/community-support/
- asDropdown: false
  icon: bolt
  includeVars: false
  keepTime: false
  tags: []
  targetBlank: true
  title: Enterprise support
  tooltip: ""
  type: link
  url: https://victoriametrics.com/support/enterprise-support/
- icon: info
  tags: []
  targetBlank: true
  title: Releases
  type: link
  url: https://github.com/VictoriaMetrics/VictoriaMetrics/releases
preload: false
refresh: ""
schemaVersion: 42
time:
  from: now-3h
  to: now
timepicker:
  refresh_intervals:
  - 10s
  - 30s
  - 1m
  - 5m
  - 15m
  - 30m
  - 1h
  - 2h
  - 1d
uid: wNf0q_kZk
version: 1
weekStart: ""
