
{{- $Values := (.helm).Values | default .Values }}
{{- $multicluster := ((($Values.grafana).sidecar).dashboards).multicluster | default false }}
{{- $defaultDatasource := "prometheus" -}}
{{- $clusterLabel := ($Values.global).clusterLabel | default "cluster" }}
{{- range (($Values.defaultDatasources).victoriametrics).datasources | default list }}
  {{- if and .isDefault .type }}{{ $defaultDatasource = .type }}{{- end }}
{{- end }}
title: VictoriaMetrics - operator
templating:
  list:
  - name: ds
    type: datasource
    query: {{ $defaultDatasource }}
    current: {}
    options: []
    refresh: 1
    regex: ""
  - name: job
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    type: query
    query:
      query: label_values(operator_log_messages_total, job)
      refId: StandardVariableQuery
    definition: label_values(operator_log_messages_total, job)
    current: {}
    options: []
    refresh: 2
    regex: ""
  - name: instance
    includeAll: true
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    type: query
    query:
      query: {{ printf "label_values(operator_log_messages_total{job=~\"$job\",%s=~\"$cluster\"}, instance)" $clusterLabel }}
      refId: StandardVariableQuery
    definition: {{ printf "label_values(operator_log_messages_total{job=~\"$job\",%s=~\"$cluster\"}, instance)" $clusterLabel }}
    current: {}
    options: []
    refresh: 2
    regex: ""
  - name: version
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    type: query
    query:
      query: {{ printf "label_values(vm_app_version{job=\"$job\",instance=\"$instance\",%s=~\"$cluster\"}, version)" $clusterLabel }}
      refId: StandardVariableQuery
    definition: {{ printf "label_values(vm_app_version{job=\"$job\",instance=\"$instance\",%s=~\"$cluster\"}, version)" $clusterLabel }}
    hide: 2
    current: {}
    options: []
    refresh: 1
    regex: ""
    sort: 2
  - name: adhoc
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    type: adhoc
    baseFilters: []
    filters: []
  - name: cluster
    label: cluster
    multi: {{ ternary true false $multicluster }}
    includeAll: {{ ternary true false $multicluster }}
    datasource:
      type: {{ $defaultDatasource }}
    type: {{ ternary "query" "constant" $multicluster }}
    query: {{ ternary (printf "label_values(vm_app_version, %s)" $clusterLabel) ".*" $multicluster }}
    hide: {{ ternary 0 2 $multicluster }}
    allValue: .*
editable: false
timezone: {{ default "utc" ($Values.defaultDashboards).defaultTimezone }}
tags:
- victoriametrics
- vm-k8s-stack
annotations:
  list:
  - name: Annotations & Alerts
    datasource:
      type: datasource
      uid: grafana
    builtIn: 1
    enable: true
    hide: true
    iconColor: rgba(0, 211, 255, 1)
    target:
      limit: 100
      matchAny: false
      tags: []
      type: dashboard
    type: dashboard
  - name: restarts
    expr: {{ printf "sum(changes(vm_app_start_timestamp{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}[$__interval])) by(job,%s)" $clusterLabel $clusterLabel }}
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    enable: true
    hide: false
    iconColor: dark-yellow
    textFormat: '{{`{{`}}job{{`}}`}} restarted'
panels:
- title: Overview
  collapsed: false
  gridPos:
    h: 1
    w: 24
    x: 0
    "y": 0
  id: 8
  type: row
- title: Version
  targets:
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    expr: {{ printf "vm_app_version{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}" $clusterLabel }}
    editorMode: code
    exemplar: false
    format: table
    instant: true
    interval: ""
    legendFormat: '{{`{{`}}short_version{{`}}`}}'
    range: false
    refId: A
  datasource:
    type: {{ $defaultDatasource }}
    uid: $ds
  fieldConfig:
    defaults:
      color:
        mode: thresholds
      mappings: []
      thresholds:
        mode: absolute
        steps:
        - color: green
          value: 0
    overrides: []
  gridPos:
    h: 3
    w: 4
    x: 0
    "y": 1
  id: 24
  options:
    colorMode: value
    graphMode: none
    justifyMode: auto
    orientation: auto
    percentChangeColorMode: standard
    reduceOptions:
      calcs:
      - lastNotNull
      fields: /^short_version$/
      values: false
    showPercentChange: false
    textMode: value
    wideLayout: true
  pluginVersion: 12.3.0
  type: stat
- title: CRD Objects count by controller
  targets:
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    expr: {{ printf "max(operator_controller_objects_count{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}) by(controller,%s)" $clusterLabel $clusterLabel }}
    editorMode: code
    legendFormat: '{{`{{`}}controller{{`}}`}}'
    range: true
    refId: A
  datasource:
    type: {{ $defaultDatasource }}
    uid: $ds
  description: Number of objects at kubernetes cluster per each controller
  fieldConfig:
    defaults:
      color:
        mode: thresholds
      mappings: []
      min: 0
      thresholds:
        mode: absolute
        steps:
        - color: green
          value: 0
        - color: red
          value: 80
    overrides: []
  gridPos:
    h: 7
    w: 20
    x: 4
    "y": 1
  id: 14
  options:
    colorMode: none
    graphMode: area
    justifyMode: auto
    orientation: auto
    percentChangeColorMode: standard
    reduceOptions:
      calcs:
      - lastNotNull
      fields: ""
      values: false
    showPercentChange: false
    text: {}
    textMode: auto
    wideLayout: true
  pluginVersion: 12.3.0
  type: stat
- title: Uptime
  targets:
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    expr: {{ printf "vm_app_uptime_seconds{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}" $clusterLabel }}
    editorMode: code
    exemplar: false
    format: table
    instant: true
    interval: ""
    legendFormat: '{{`{{`}}instance{{`}}`}}'
    range: false
    refId: A
  datasource:
    type: {{ $defaultDatasource }}
    uid: $ds
  fieldConfig:
    defaults:
      color:
        mode: thresholds
      mappings: []
      thresholds:
        mode: absolute
        steps:
        - color: green
          value: 0
        - color: red
          value: 80
      unit: s
    overrides: []
  gridPos:
    h: 4
    w: 4
    x: 0
    "y": 4
  id: 22
  options:
    colorMode: value
    graphMode: area
    justifyMode: auto
    orientation: auto
    percentChangeColorMode: standard
    reduceOptions:
      calcs:
      - lastNotNull
      fields: ""
      values: false
    showPercentChange: false
    textMode: auto
    wideLayout: true
  pluginVersion: 12.3.0
  type: stat
- title: Prometheus Objects watchers
  targets:
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    expr: {{ printf "sum(operator_prometheus_converter_active_watchers) by(%s)" $clusterLabel }}
    editorMode: code
    instant: false
    legendFormat: __auto
    range: true
    refId: A
  datasource:
    type: {{ $defaultDatasource }}
    uid: $ds
  description: ' Shows per namespace watchers for Prometheus Operator objects (ServiceMonitors, PodMonitors, etc) '
  fieldConfig:
    defaults:
      color:
        mode: thresholds
      mappings: []
      min: 0
      thresholds:
        mode: absolute
        steps:
        - color: green
          value: 0
        - color: red
          value: 80
    overrides: []
  gridPos:
    h: 5
    w: 4
    x: 0
    "y": 8
  id: 39
  options:
    colorMode: value
    graphMode: area
    justifyMode: auto
    orientation: auto
    percentChangeColorMode: standard
    reduceOptions:
      calcs:
      - lastNotNull
      fields: ""
      values: false
    showPercentChange: false
    textMode: auto
    wideLayout: true
  pluginVersion: 12.3.0
  type: stat
- title: Elected Leaders
  targets:
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    expr: {{ printf "sum(leader_election_master_status{job=~\"$job\",%s=~\"$cluster\"}) by(%s)" $clusterLabel $clusterLabel }}
    editorMode: code
    instant: false
    legendFormat: __auto
    range: true
    refId: A
  datasource:
    type: {{ $defaultDatasource }}
    uid: $ds
  description: " Number of operator instances with obtained leader status. \n Value above 1 indicates that instances with the same job may behave incorrectly.\n It's recommend to check Operator logs. "
  fieldConfig:
    defaults:
      color:
        mode: thresholds
      mappings: []
      min: 0
      thresholds:
        mode: absolute
        steps:
        - color: green
          value: 0
        - color: red
          value: 80
    overrides: []
  gridPos:
    h: 5
    w: 4
    x: 4
    "y": 8
  id: 40
  options:
    colorMode: value
    graphMode: area
    justifyMode: auto
    orientation: auto
    percentChangeColorMode: standard
    reduceOptions:
      calcs:
      - lastNotNull
      fields: ""
      values: false
    showPercentChange: false
    textMode: auto
    wideLayout: true
  pluginVersion: 12.3.0
  type: stat
- title: Active workers
  targets:
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    expr: {{ printf "sum(controller_runtime_active_workers{job=~\"$job\",%s=~\"$cluster\"}) by(%s)" $clusterLabel $clusterLabel }}
    editorMode: code
    instant: false
    legendFormat: __auto
    range: true
    refId: A
  datasource:
    type: {{ $defaultDatasource }}
    uid: $ds
  description: ' Shows number of active reconcile workers'
  fieldConfig:
    defaults:
      color:
        mode: thresholds
      mappings: []
      thresholds:
        min: 0
        mode: absolute
        steps:
        - color: green
          value: 0
        - color: red
          value: 80
    overrides: []
  gridPos:
    h: 5
    w: 4
    x: 8
    "y": 8
  id: 41
  options:
    colorMode: value
    graphMode: area
    justifyMode: auto
    orientation: auto
    percentChangeColorMode: standard
    reduceOptions:
      calcs:
      - lastNotNull
      fields: ""
      values: false
    showPercentChange: false
    textMode: auto
    wideLayout: true
  pluginVersion: 12.3.0
  type: stat
- title: Prometheus Converter Watch events
  targets:
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    expr: {{ printf "sum(rate(operator_prometheus_converter_watch_events_total{job=~\"$job\",%s=~\"$cluster\"}[$__interval])) by(event_type,object_type_name,%s)" $clusterLabel $clusterLabel }}
    editorMode: code
    instant: false
    legendFormat: '{{`{{`}}object_type_name{{`}}`}} {{`{{`}}event_type{{`}}`}}'
    range: true
    refId: A
  datasource:
    type: {{ $defaultDatasource }}
    uid: $ds
  description: ' Shows amount of Prometheus Operator objects processed by Operator.'
  fieldConfig:
    defaults:
      color:
        mode: palette-classic
      custom:
        axisBorderShow: false
        axisCenteredZero: false
        axisColorMode: text
        axisLabel: ""
        axisPlacement: auto
        barAlignment: 0
        barWidthFactor: 0.6
        drawStyle: line
        fillOpacity: 0
        gradientMode: none
        hideFrom:
          legend: false
          tooltip: false
          viz: false
        insertNulls: false
        lineInterpolation: linear
        lineWidth: 1
        pointSize: 5
        scaleDistribution:
          type: linear
        showPoints: auto
        showValues: false
        spanNulls: false
        stacking:
          group: A
          mode: none
        thresholdsStyle:
          mode: "off"
      mappings: []
      thresholds:
        mode: absolute
        steps:
        - color: green
          value: 0
        - color: red
          value: 80
    overrides: []
  gridPos:
    h: 5
    w: 12
    x: 12
    "y": 8
  id: 38
  options:
    legend:
      calcs:
      - max
      - mean
      displayMode: table
      placement: bottom
      showLegend: true
    tooltip:
      hideZeros: false
      mode: single
      sort: none
  pluginVersion: 12.3.0
  type: timeseries
- title: Non-default flags
  targets:
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    expr: {{ printf "sum(flag{is_set=\"true\",job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}) by(job,instance,name,value,%s)" $clusterLabel $clusterLabel }}
    editorMode: code
    exemplar: false
    format: table
    instant: true
    legendFormat: '{{`{{`}}object_type_name{{`}}`}} {{`{{`}}event_type{{`}}`}}'
    range: false
    refId: A
  datasource:
    type: {{ $defaultDatasource }}
    uid: $ds
  description: ""
  fieldConfig:
    defaults:
      color:
        mode: thresholds
      custom:
        align: auto
        cellOptions:
          type: auto
        footer:
          reducers: []
        inspect: false
      mappings: []
      thresholds:
        mode: absolute
        steps:
        - color: green
          value: 0
        - color: red
          value: 80
    overrides: []
  gridPos:
    h: 5
    w: 12
    x: 0
    "y": 13
  id: 43
  options:
    cellHeight: sm
    showHeader: true
  pluginVersion: 12.3.0
  transformations:
  - id: groupBy
    options:
      fields:
        instance:
          aggregations:
          - uniqueValues
          operation: aggregate
        job:
          aggregations: []
          operation: groupby
        name:
          aggregations: []
          operation: groupby
        value:
          aggregations: []
          operation: groupby
  type: table
- title: Global settings
  targets:
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    expr: {{ printf "sum(config_parameter{job=~\"$job\",instance=~\"$instance\",value!=\"\",%s=~\"$cluster\"}) by(job,instance,name,value,%s)" $clusterLabel $clusterLabel }}
    editorMode: code
    exemplar: false
    format: table
    instant: true
    legendFormat: '{{`{{`}}object_type_name{{`}}`}} {{`{{`}}event_type{{`}}`}}'
    range: false
    refId: A
  datasource:
    type: {{ $defaultDatasource }}
    uid: $ds
  description: ""
  fieldConfig:
    defaults:
      color:
        mode: thresholds
      custom:
        align: auto
        cellOptions:
          type: auto
        footer:
          reducers: []
        inspect: false
      mappings: []
      thresholds:
        mode: absolute
        steps:
        - color: green
          value: 0
        - color: red
          value: 80
    overrides: []
  gridPos:
    h: 5
    w: 12
    x: 12
    "y": 13
  id: 45
  options:
    cellHeight: sm
    showHeader: true
  pluginVersion: 12.3.0
  transformations:
  - id: groupBy
    options:
      fields:
        instance:
          aggregations:
          - uniqueValues
          operation: aggregate
        job:
          aggregations: []
        name:
          aggregations: []
          operation: groupby
        value:
          aggregations: []
          operation: groupby
  type: table
- title: Reconciliation rate by controller
  targets:
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    expr: {{ printf "sum(rate(controller_runtime_reconcile_total{job=~\"$job\",instance=~\"$instance\",result=~\"requeue_after|requeue|success\",%s=~\"$cluster\"}[$__rate_interval])) by(controller,%s)" $clusterLabel $clusterLabel }}
    editorMode: code
    legendFormat: '{{`{{`}}controller{{`}}`}}'
    range: true
    refId: A
  datasource:
    type: {{ $defaultDatasource }}
    uid: $ds
  fieldConfig:
    defaults:
      color:
        mode: palette-classic
      custom:
        axisBorderShow: false
        axisCenteredZero: false
        axisColorMode: text
        axisLabel: ""
        axisPlacement: auto
        barAlignment: 0
        barWidthFactor: 0.6
        drawStyle: line
        fillOpacity: 10
        gradientMode: none
        hideFrom:
          legend: false
          tooltip: false
          viz: false
        insertNulls: false
        lineInterpolation: linear
        lineWidth: 1
        pointSize: 5
        scaleDistribution:
          type: linear
        showPoints: never
        showValues: false
        spanNulls: false
        stacking:
          group: A
          mode: none
        thresholdsStyle:
          mode: "off"
      mappings: []
      thresholds:
        mode: absolute
        steps:
        - color: green
          value: 0
        - color: red
          value: 80
      unit: short
    overrides: []
  gridPos:
    h: 13
    w: 12
    x: 0
    "y": 18
  id: 12
  options:
    legend:
      calcs:
      - mean
      - max
      displayMode: table
      placement: bottom
      showLegend: true
    tooltip:
      hideZeros: false
      mode: multi
      sort: none
  pluginVersion: 12.3.0
  type: timeseries
- title: Log message rate
  targets:
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    expr: {{ printf "sum(rate(operator_log_messages_total{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}[$__rate_interval])) by(level,%s)" $clusterLabel $clusterLabel }}
    editorMode: code
    legendFormat: '{{`{{`}}label_name{{`}}`}}'
    range: true
    refId: A
  datasource:
    type: {{ $defaultDatasource }}
    uid: $ds
  description: Shows the rate of logging the messages by their level. Unexpected spike in rate is a good reason to check logs.
  fieldConfig:
    defaults:
      color:
        mode: palette-classic
      custom:
        axisBorderShow: false
        axisCenteredZero: false
        axisColorMode: text
        axisLabel: ""
        axisPlacement: auto
        barAlignment: 0
        barWidthFactor: 0.6
        drawStyle: line
        fillOpacity: 10
        gradientMode: none
        hideFrom:
          legend: false
          tooltip: false
          viz: false
        insertNulls: false
        lineInterpolation: linear
        lineWidth: 1
        pointSize: 5
        scaleDistribution:
          type: linear
        showPoints: never
        showValues: false
        spanNulls: false
        stacking:
          group: A
          mode: none
        thresholdsStyle:
          mode: "off"
      mappings: []
      thresholds:
        mode: absolute
        steps:
        - color: green
          value: 0
        - color: red
          value: 80
      unit: short
    overrides: []
  gridPos:
    h: 13
    w: 12
    x: 12
    "y": 18
  id: 16
  options:
    legend:
      calcs:
      - mean
      - max
      displayMode: table
      placement: bottom
      showLegend: true
    tooltip:
      hideZeros: false
      mode: multi
      sort: none
  pluginVersion: 12.3.0
  type: timeseries
- title: Troubleshooting
  collapsed: false
  gridPos:
    h: 1
    w: 24
    x: 0
    "y": 31
  id: 6
  type: row
- title: Reconcile errors by controller
  targets:
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    expr: {{ printf "sum(rate(controller_runtime_reconcile_errors_total{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}[$__rate_interval])) by(controller,%s) > 0" $clusterLabel $clusterLabel }}
    editorMode: code
    exemplar: false
    instant: false
    legendFormat: '{{`{{`}}controller{{`}}`}}'
    range: true
    refId: A
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    expr: {{ printf "sum(rate(controller_runtime_reconcile_total{job=~\"$job\",instance=~\"$instance\",result=\"error\",%s=~\"$cluster\"}[$__rate_interval])) by(controller,%s) > 0" $clusterLabel $clusterLabel }}
    editorMode: code
    hide: false
    legendFormat: 'result errors: {{`{{`}}controller{{`}}`}}'
    range: true
    refId: B
  datasource:
    type: {{ $defaultDatasource }}
    uid: $ds
  description: Non zero metrics indicates about error with CR object definition (typos or incorrect values) or errors with kubernetes API connection.
  fieldConfig:
    defaults:
      color:
        mode: palette-classic
      custom:
        axisBorderShow: false
        axisCenteredZero: false
        axisColorMode: text
        axisLabel: ""
        axisPlacement: auto
        barAlignment: 0
        barWidthFactor: 0.6
        drawStyle: line
        fillOpacity: 10
        gradientMode: none
        hideFrom:
          legend: false
          tooltip: false
          viz: false
        insertNulls: false
        lineInterpolation: linear
        lineWidth: 1
        pointSize: 5
        scaleDistribution:
          type: linear
        showPoints: never
        showValues: false
        spanNulls: false
        stacking:
          group: A
          mode: none
        thresholdsStyle:
          mode: "off"
      mappings: []
      thresholds:
        mode: absolute
        steps:
        - color: green
          value: 0
        - color: red
          value: 80
      unit: short
    overrides: []
  gridPos:
    h: 8
    w: 12
    x: 0
    "y": 32
  id: 10
  options:
    legend:
      calcs:
      - lastNotNull
      - max
      displayMode: table
      placement: bottom
      showLegend: true
    tooltip:
      hideZeros: false
      mode: multi
      sort: none
  pluginVersion: 12.3.0
  type: timeseries
- title: Throttled reconciliation config events
  targets:
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    expr: {{ printf "sum(rate(operator_reconcile_throttled_events_total[$__rate_interval])) by(controller,%s)" $clusterLabel }}
    editorMode: code
    legendFormat: '{{`{{`}}controller{{`}}`}}'
    range: true
    refId: A
  datasource:
    type: {{ $defaultDatasource }}
    uid: $ds
  description: Operator limits number of reconcile configuration events to 5 events per 2 seconds by default.
  fieldConfig:
    defaults:
      color:
        mode: palette-classic
      custom:
        axisBorderShow: false
        axisCenteredZero: false
        axisColorMode: text
        axisLabel: ""
        axisPlacement: auto
        barAlignment: 0
        barWidthFactor: 0.6
        drawStyle: line
        fillOpacity: 10
        gradientMode: none
        hideFrom:
          legend: false
          tooltip: false
          viz: false
        insertNulls: false
        lineInterpolation: linear
        lineWidth: 1
        pointSize: 5
        scaleDistribution:
          type: linear
        showPoints: never
        showValues: false
        spanNulls: false
        stacking:
          group: A
          mode: none
        thresholdsStyle:
          mode: "off"
      mappings: []
      thresholds:
        mode: absolute
        steps:
        - color: green
          value: 0
        - color: red
          value: 80
      unit: short
    overrides: []
  gridPos:
    h: 8
    w: 12
    x: 12
    "y": 32
  id: 18
  options:
    legend:
      calcs:
      - mean
      - max
      displayMode: table
      placement: bottom
      showLegend: true
    tooltip:
      hideZeros: false
      mode: multi
      sort: none
  pluginVersion: 12.3.0
  type: timeseries
- title: Working queue depth
  targets:
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    expr: {{ printf "max(workqueue_depth{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}) by(name,%s)" $clusterLabel $clusterLabel }}
    editorMode: code
    legendFormat: '{{`{{`}}label_name{{`}}`}}'
    range: true
    refId: A
  datasource:
    type: {{ $defaultDatasource }}
    uid: $ds
  description: Number of objects waiting in the queue for reconciliation. Non-zero values indicate that operator cannot process CR objects changes with the given resources.
  fieldConfig:
    defaults:
      color:
        mode: palette-classic
      custom:
        axisBorderShow: false
        axisCenteredZero: false
        axisColorMode: text
        axisLabel: ""
        axisPlacement: auto
        barAlignment: 0
        barWidthFactor: 0.6
        drawStyle: line
        fillOpacity: 10
        gradientMode: none
        hideFrom:
          legend: false
          tooltip: false
          viz: false
        insertNulls: false
        lineInterpolation: linear
        lineWidth: 1
        pointSize: 5
        scaleDistribution:
          type: linear
        showPoints: never
        showValues: false
        spanNulls: false
        stacking:
          group: A
          mode: none
        thresholdsStyle:
          mode: "off"
      mappings: []
      thresholds:
        mode: absolute
        steps:
        - color: green
          value: 0
        - color: red
          value: 80
      unit: short
    overrides: []
  gridPos:
    h: 11
    w: 12
    x: 0
    "y": 40
  id: 20
  options:
    legend:
      calcs:
      - max
      - mean
      displayMode: table
      placement: bottom
      showLegend: true
    tooltip:
      hideZeros: false
      mode: multi
      sort: none
  pluginVersion: 12.3.0
  type: timeseries
- title: Reconciliation latency by controller
  targets:
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    expr: {{ printf "histogram_quantile(0.99, sum(rate(controller_runtime_reconcile_time_seconds_bucket{job=~\"$job\",%s=~\"$cluster\"}[$__rate_interval])) by(le,controller,%s))" $clusterLabel $clusterLabel }}
    editorMode: code
    legendFormat: q.99 {{`{{`}}controller{{`}}`}}
    range: true
    refId: A
  datasource:
    type: {{ $defaultDatasource }}
    uid: $ds
  description: " For controllers with StatefulSet it's ok to see latency greater then 3 seconds. It could be vmalertmanager,vmcluster or vmagent in statefulMode.\n\n For other controllers, latency greater then 2 second may indicate issues with kubernetes cluster or operator's performance.\n "
  fieldConfig:
    defaults:
      color:
        mode: palette-classic
      custom:
        axisBorderShow: false
        axisCenteredZero: false
        axisColorMode: text
        axisLabel: ""
        axisPlacement: auto
        barAlignment: 0
        barWidthFactor: 0.6
        drawStyle: line
        fillOpacity: 10
        gradientMode: none
        hideFrom:
          legend: false
          tooltip: false
          viz: false
        insertNulls: false
        lineInterpolation: linear
        lineWidth: 1
        pointSize: 5
        scaleDistribution:
          type: linear
        showPoints: never
        showValues: false
        spanNulls: false
        stacking:
          group: A
          mode: none
        thresholdsStyle:
          mode: "off"
      mappings: []
      thresholds:
        mode: absolute
        steps:
        - color: green
          value: 0
        - color: red
          value: 80
      unit: s
    overrides: []
  gridPos:
    h: 11
    w: 12
    x: 12
    "y": 40
  id: 26
  options:
    legend:
      calcs:
      - mean
      - max
      displayMode: table
      placement: bottom
      showLegend: true
      sortBy: Mean
      sortDesc: true
    tooltip:
      hideZeros: false
      mode: multi
      sort: none
  pluginVersion: 12.3.0
  type: timeseries
- title: REST client requests
  targets:
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    expr: {{ printf "sum(rate(rest_client_requests_total{job=~\"$job\",%s=~\"$cluster\"}[$__interval])) by(method,code,%s)" $clusterLabel $clusterLabel }}
    editorMode: code
    instant: false
    legendFormat: '{{`{{`}}method{{`}}`}} {{`{{`}}code{{`}}`}}'
    range: true
    refId: A
  datasource:
    type: {{ $defaultDatasource }}
    uid: $ds
  description: Number of HTTP requests to the Kubernetes API server break down by code and method
  fieldConfig:
    defaults:
      color:
        mode: palette-classic
      custom:
        axisBorderShow: false
        axisCenteredZero: false
        axisColorMode: text
        axisLabel: ""
        axisPlacement: auto
        barAlignment: 0
        barWidthFactor: 0.6
        drawStyle: line
        fillOpacity: 0
        gradientMode: none
        hideFrom:
          legend: false
          tooltip: false
          viz: false
        insertNulls: false
        lineInterpolation: linear
        lineWidth: 1
        pointSize: 5
        scaleDistribution:
          type: linear
        showPoints: auto
        showValues: false
        spanNulls: false
        stacking:
          group: A
          mode: none
        thresholdsStyle:
          mode: "off"
      mappings: []
      thresholds:
        mode: absolute
        steps:
        - color: green
          value: 0
        - color: red
          value: 80
    overrides: []
  gridPos:
    h: 8
    w: 12
    x: 0
    "y": 51
  id: 36
  options:
    legend:
      calcs:
      - max
      - mean
      displayMode: table
      placement: bottom
      showLegend: true
    tooltip:
      hideZeros: false
      mode: single
      sort: none
  pluginVersion: 12.3.0
  type: timeseries
- title: Concurrent reconcile ($instance)
  targets:
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    expr: {{ printf "max(max_over_time(controller_runtime_active_workers{job=~\"$job\",%s=~\"$cluster\"}[$__interval])) by(%s)" $clusterLabel $clusterLabel }}
    editorMode: code
    instant: false
    legendFormat: Current
    range: true
    refId: A
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    expr: {{ printf "min(controller_runtime_max_concurrent_reconciles{job=~\"$job\",%s=~\"$cluster\"}) by(%s)" $clusterLabel $clusterLabel }}
    editorMode: code
    hide: false
    instant: false
    legendFormat: Max
    range: true
    refId: B
  datasource:
    type: {{ $defaultDatasource }}
    uid: $ds
  description: |-
    Shows how many ongoing reconcile events are taking place, where:
    * `max` - equal to the value of flag`-controller.maxConcurrentReconciles`;
    * `current` - current number of reconcile workers processing CRD objects.

    When `current` hits `max` constantly, it means operator cannot process events in time. It should be either increased value for flag `-controller.maxConcurrentReconciles` or allocated  additional CPU resources to the operator.
  fieldConfig:
    defaults:
      color:
        mode: palette-classic
      custom:
        axisBorderShow: false
        axisCenteredZero: false
        axisColorMode: text
        axisLabel: ""
        axisPlacement: auto
        barAlignment: 0
        barWidthFactor: 0.6
        drawStyle: line
        fillOpacity: 0
        gradientMode: none
        hideFrom:
          legend: false
          tooltip: false
          viz: false
        insertNulls: false
        lineInterpolation: linear
        lineWidth: 1
        pointSize: 5
        scaleDistribution:
          type: linear
        showPoints: auto
        showValues: false
        spanNulls: false
        stacking:
          group: A
          mode: none
        thresholdsStyle:
          mode: "off"
      mappings: []
      thresholds:
        mode: absolute
        steps:
        - color: green
          value: 0
        - color: red
          value: 80
    overrides:
    - matcher:
        id: byName
        options: Max
      properties:
      - id: custom.axisColorMode
        value: text
      - id: color
        value:
          fixedColor: dark-red
          mode: fixed
  gridPos:
    h: 8
    w: 12
    x: 12
    "y": 51
  id: 42
  options:
    legend:
      calcs: []
      displayMode: list
      placement: bottom
      showLegend: true
    tooltip:
      hideZeros: false
      mode: single
      sort: none
  pluginVersion: 12.3.0
  type: timeseries
- title: Go scheduling latency
  targets:
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    expr: {{ printf "max(histogram_quantile(0.99, sum(rate(go_sched_latencies_seconds_bucket{job=~\"$job\",%s=~\"$cluster\"}[$__rate_interval])) by(job,instance,le,%s))) by(job,%s)" $clusterLabel $clusterLabel $clusterLabel }}
    editorMode: code
    instant: false
    legendFormat: __auto
    range: true
    refId: A
  datasource:
    type: {{ $defaultDatasource }}
    uid: $ds
  description: "Shows the time goroutines have spent in runnable state before actually running. The lower is better.\n\nHigh values or values exceeding the threshold is usually a sign of insufficient CPU resources or CPU throttling. \n\nVerify that service has enough CPU resources. Otherwise, the service could work unreliably with delays in processing."
  fieldConfig:
    defaults:
      color:
        mode: palette-classic
      custom:
        axisBorderShow: false
        axisCenteredZero: false
        axisColorMode: text
        axisLabel: ""
        axisPlacement: auto
        barAlignment: 0
        barWidthFactor: 0.6
        drawStyle: line
        fillOpacity: 0
        gradientMode: none
        hideFrom:
          legend: false
          tooltip: false
          viz: false
        insertNulls: false
        lineInterpolation: linear
        lineWidth: 1
        pointSize: 5
        scaleDistribution:
          type: linear
        showPoints: auto
        showValues: false
        spanNulls: false
        stacking:
          group: A
          mode: none
        thresholdsStyle:
          mode: "off"
      mappings: []
      thresholds:
        mode: absolute
        steps:
        - color: green
          value: 0
        - color: red
          value: 80
      unit: s
    overrides: []
  gridPos:
    h: 8
    w: 12
    x: 0
    "y": 59
  id: 37
  options:
    legend:
      calcs: []
      displayMode: list
      placement: bottom
      showLegend: true
    tooltip:
      hideZeros: false
      mode: single
      sort: none
  pluginVersion: 12.3.0
  type: timeseries
- title: REST client latency
  targets:
  - datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    expr: {{ printf "histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{job=~\"$job\",%s=~\"$cluster\"}[$__rate_interval])) by(le,method,api,%s))" $clusterLabel $clusterLabel }}
    editorMode: code
    instant: false
    legendFormat: '{{`{{`}}method{{`}}`}} {{`{{`}}api{{`}}`}}'
    range: true
    refId: A
  datasource:
    type: {{ $defaultDatasource }}
    uid: $ds
  description: ' Requests latency to the Kubernetes API server.'
  fieldConfig:
    defaults:
      color:
        mode: palette-classic
      custom:
        axisBorderShow: false
        axisCenteredZero: false
        axisColorMode: text
        axisLabel: ""
        axisPlacement: auto
        barAlignment: 0
        barWidthFactor: 0.6
        drawStyle: line
        fillOpacity: 0
        gradientMode: none
        hideFrom:
          legend: false
          tooltip: false
          viz: false
        insertNulls: false
        lineInterpolation: linear
        lineWidth: 1
        pointSize: 5
        scaleDistribution:
          type: linear
        showPoints: auto
        showValues: false
        spanNulls: false
        stacking:
          group: A
          mode: none
        thresholdsStyle:
          mode: "off"
      mappings: []
      thresholds:
        mode: absolute
        steps:
        - color: green
          value: 0
        - color: red
          value: 80
      unit: s
    overrides: []
  gridPos:
    h: 8
    w: 12
    x: 12
    "y": 59
  id: 35
  options:
    legend:
      calcs:
      - max
      - mean
      displayMode: table
      placement: bottom
      showLegend: true
    tooltip:
      hideZeros: false
      mode: single
      sort: none
  pluginVersion: 12.3.0
  type: timeseries
- title: Resources
  panels:
  - title: Memory usage ($instance)
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "sum(go_memstats_sys_bytes{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}) by(%s)" $clusterLabel $clusterLabel }}
      editorMode: code
      legendFormat: requested from system
      range: true
      refId: A
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "sum(go_memstats_heap_inuse_bytes{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}) by(%s)" $clusterLabel $clusterLabel }}
      editorMode: code
      hide: false
      legendFormat: heap inuse
      range: true
      refId: B
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "sum(go_memstats_stack_inuse_bytes{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}) by(%s)" $clusterLabel $clusterLabel }}
      editorMode: code
      hide: false
      legendFormat: stack inuse
      range: true
      refId: C
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "sum(process_resident_memory_bytes{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}) by(%s)" $clusterLabel $clusterLabel }}
      editorMode: code
      hide: false
      legendFormat: resident
      range: true
      refId: D
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ""
          axisPlacement: auto
          barAlignment: 0
          drawStyle: line
          fillOpacity: 10
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: "off"
        mappings: []
        thresholds:
          mode: absolute
          steps:
          - color: green
          - color: red
            value: 80
        unit: bytes
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 0
      "y": 63
    id: 28
    options:
      legend:
        calcs:
        - mean
        - lastNotNull
        - max
        displayMode: table
        placement: bottom
        showLegend: true
      tooltip:
        mode: multi
        sort: none
    pluginVersion: 10.4.0
    type: timeseries
  - title: CPU ($instance)
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "sum(rate(process_cpu_seconds_total{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}[$__rate_interval])) by(%s)" $clusterLabel $clusterLabel }}
      editorMode: code
      legendFormat: CPU cores used
      range: true
      refId: A
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ""
          axisPlacement: auto
          barAlignment: 0
          drawStyle: line
          fillOpacity: 10
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: "off"
        mappings: []
        thresholds:
          mode: absolute
          steps:
          - color: green
          - color: red
            value: 80
        unit: short
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 12
      "y": 63
    id: 30
    options:
      legend:
        calcs: []
        displayMode: list
        placement: bottom
        showLegend: true
      tooltip:
        mode: multi
        sort: none
    pluginVersion: 10.4.0
    type: timeseries
  - title: Goroutines ($instance)
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "sum(go_goroutines{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}) by(%s)" $clusterLabel $clusterLabel }}
      editorMode: code
      legendFormat: goroutines
      range: true
      refId: A
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ""
          axisPlacement: auto
          barAlignment: 0
          drawStyle: line
          fillOpacity: 10
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: "off"
        mappings: []
        thresholds:
          mode: absolute
          steps:
          - color: green
          - color: red
            value: 80
        unit: short
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 0
      "y": 71
    id: 32
    options:
      legend:
        calcs: []
        displayMode: list
        placement: bottom
        showLegend: true
      tooltip:
        mode: multi
        sort: none
    pluginVersion: 10.4.0
    type: timeseries
  - title: GC duration ($instance)
    targets:
    - datasource:
        type: {{ $defaultDatasource }}
        uid: $ds
      expr: {{ printf "sum(rate(go_gc_duration_seconds_sum{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}[$__rate_interval])) by(%s) / sum(rate(go_gc_duration_seconds_count{job=~\"$job\",instance=~\"$instance\",%s=~\"$cluster\"}[$__rate_interval])) by(%s)" $clusterLabel $clusterLabel $clusterLabel $clusterLabel }}
      editorMode: code
      legendFormat: avg gc duration
      range: true
      refId: A
    datasource:
      type: {{ $defaultDatasource }}
      uid: $ds
    fieldConfig:
      defaults:
        color:
          mode: palette-classic
        custom:
          axisBorderShow: false
          axisCenteredZero: false
          axisColorMode: text
          axisLabel: ""
          axisPlacement: auto
          barAlignment: 0
          drawStyle: line
          fillOpacity: 10
          gradientMode: none
          hideFrom:
            legend: false
            tooltip: false
            viz: false
          insertNulls: false
          lineInterpolation: linear
          lineWidth: 1
          pointSize: 5
          scaleDistribution:
            type: linear
          showPoints: never
          spanNulls: false
          stacking:
            group: A
            mode: none
          thresholdsStyle:
            mode: "off"
        mappings: []
        thresholds:
          mode: absolute
          steps:
          - color: green
          - color: red
            value: 80
        unit: s
      overrides: []
    gridPos:
      h: 8
      w: 12
      x: 12
      "y": 71
    id: 34
    options:
      legend:
        calcs:
        - mean
        - max
        displayMode: table
        placement: bottom
        showLegend: true
      tooltip:
        mode: multi
        sort: none
    pluginVersion: 10.4.0
    type: timeseries
  collapsed: true
  gridPos:
    h: 1
    w: 24
    x: 0
    "y": 67
  id: 4
  type: row
condition: {{ (index $Values "victoria-metrics-operator" "enabled") }}
description: Overview for operator VictoriaMetrics v0.25.0 or higher
fiscalYearStartMonth: 0
graphTooltip: 0
id: 0
links:
- icon: doc
  tags: []
  targetBlank: true
  title: Docs
  type: link
  url: https://docs.victoriametrics.com/operator/
- icon: info
  tags: []
  targetBlank: true
  title: Community support
  type: link
  url: https://victoriametrics.com/support/community-support/
- asDropdown: false
  icon: bolt
  includeVars: false
  keepTime: false
  tags: []
  targetBlank: true
  title: Enterprise support
  tooltip: ""
  type: link
  url: https://victoriametrics.com/support/enterprise-support/
- icon: info
  tags: []
  targetBlank: true
  title: Releases
  type: link
  url: https://github.com/VictoriaMetrics/operator/releases
preload: false
refresh: ""
schemaVersion: 42
time:
  from: now-15m
  to: now
timepicker: {}
uid: 1H179hunk
version: 8
